2019-07-17 16:10:20 +00:00
|
|
|
use glib::prelude::*;
|
|
|
|
use gst::prelude::*;
|
2021-09-13 09:26:56 +00:00
|
|
|
use gst::{gst_debug, gst_error, gst_log, gst_warning};
|
2019-07-17 16:10:20 +00:00
|
|
|
use gst_video::prelude::*;
|
|
|
|
|
|
|
|
use byte_slice_cast::AsMutSliceOf;
|
|
|
|
|
2019-07-18 21:05:31 +00:00
|
|
|
use std::cmp;
|
2019-07-17 16:10:20 +00:00
|
|
|
use std::collections::VecDeque;
|
2020-01-16 09:14:10 +00:00
|
|
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
2019-07-17 16:10:20 +00:00
|
|
|
use std::sync::{Arc, Condvar, Mutex, Weak};
|
|
|
|
use std::thread;
|
|
|
|
|
2021-02-23 10:29:04 +00:00
|
|
|
use once_cell::sync::Lazy;
|
|
|
|
|
2019-07-17 16:10:20 +00:00
|
|
|
use super::*;
|
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
pub struct ReceiverInfo {
|
|
|
|
id: usize,
|
2020-01-16 10:18:57 +00:00
|
|
|
ndi_name: Option<String>,
|
2020-01-16 09:14:10 +00:00
|
|
|
url_address: Option<String>,
|
|
|
|
recv: RecvInstance,
|
|
|
|
video: Option<Weak<ReceiverInner<VideoReceiver>>>,
|
|
|
|
audio: Option<Weak<ReceiverInner<AudioReceiver>>>,
|
|
|
|
observations: Observations,
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
|
2021-02-23 10:29:04 +00:00
|
|
|
static HASHMAP_RECEIVERS: Lazy<Mutex<HashMap<usize, ReceiverInfo>>> = Lazy::new(|| {
|
|
|
|
let m = HashMap::new();
|
|
|
|
Mutex::new(m)
|
|
|
|
});
|
2019-07-17 16:10:20 +00:00
|
|
|
|
|
|
|
static ID_RECEIVER: AtomicUsize = AtomicUsize::new(0);
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
pub trait ReceiverType: 'static {
|
|
|
|
type InfoType: Send + 'static;
|
|
|
|
const IS_VIDEO: bool;
|
|
|
|
}
|
|
|
|
|
|
|
|
pub enum AudioReceiver {}
|
|
|
|
pub enum VideoReceiver {}
|
|
|
|
|
|
|
|
impl ReceiverType for AudioReceiver {
|
|
|
|
type InfoType = gst_audio::AudioInfo;
|
|
|
|
const IS_VIDEO: bool = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
impl ReceiverType for VideoReceiver {
|
|
|
|
type InfoType = gst_video::VideoInfo;
|
|
|
|
const IS_VIDEO: bool = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
pub struct Receiver<T: ReceiverType>(Arc<ReceiverInner<T>>);
|
|
|
|
|
|
|
|
impl<T: ReceiverType> Clone for Receiver<T> {
|
|
|
|
fn clone(&self) -> Self {
|
|
|
|
Receiver(self.0.clone())
|
|
|
|
}
|
|
|
|
}
|
2019-07-17 16:10:20 +00:00
|
|
|
|
|
|
|
#[derive(Debug)]
|
2019-07-19 08:32:04 +00:00
|
|
|
pub enum ReceiverItem<T: ReceiverType> {
|
|
|
|
Buffer(gst::Buffer, T::InfoType),
|
2019-07-17 16:10:20 +00:00
|
|
|
Flushing,
|
|
|
|
Timeout,
|
|
|
|
Error(gst::FlowError),
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
pub struct ReceiverInner<T: ReceiverType> {
|
2019-07-17 16:10:20 +00:00
|
|
|
id: usize,
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
queue: ReceiverQueue<T>,
|
2021-08-02 05:45:32 +00:00
|
|
|
max_queue_length: usize,
|
2019-07-17 16:10:20 +00:00
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
recv: Mutex<RecvInstance>,
|
2019-07-17 16:10:20 +00:00
|
|
|
|
2019-07-18 21:05:31 +00:00
|
|
|
observations: Observations,
|
|
|
|
|
2019-07-17 16:10:20 +00:00
|
|
|
cat: gst::DebugCategory,
|
|
|
|
element: glib::WeakRef<gst_base::BaseSrc>,
|
|
|
|
timestamp_mode: TimestampMode,
|
2020-01-16 09:14:10 +00:00
|
|
|
|
|
|
|
first_frame: AtomicBool,
|
2019-07-17 16:10:20 +00:00
|
|
|
timeout: u32,
|
2020-01-16 09:14:10 +00:00
|
|
|
connect_timeout: u32,
|
2019-07-17 16:10:20 +00:00
|
|
|
|
|
|
|
thread: Mutex<Option<std::thread::JoinHandle<()>>>,
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
struct ReceiverQueue<T: ReceiverType>(Arc<(Mutex<ReceiverQueueInner<T>>, Condvar)>);
|
2019-07-17 16:10:20 +00:00
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
impl<T: ReceiverType> Clone for ReceiverQueue<T> {
|
|
|
|
fn clone(&self) -> Self {
|
|
|
|
ReceiverQueue(self.0.clone())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
struct ReceiverQueueInner<T: ReceiverType> {
|
2019-07-17 16:10:20 +00:00
|
|
|
// If we should be capturing at all or go out of our capture loop
|
|
|
|
//
|
|
|
|
// This is true as long as the source element is in Paused/Playing
|
|
|
|
capturing: bool,
|
|
|
|
|
|
|
|
// If we're flushing right now and all buffers should simply be discarded
|
|
|
|
// and capture() directly returns Flushing
|
|
|
|
flushing: bool,
|
|
|
|
|
|
|
|
// If we're playing right now or not: if not we simply discard everything captured
|
|
|
|
playing: bool,
|
|
|
|
// Queue containing our buffers. This holds at most 5 buffers at a time.
|
|
|
|
//
|
|
|
|
// On timeout/error will contain a single item and then never be filled again
|
2019-07-19 08:32:04 +00:00
|
|
|
buffer_queue: VecDeque<(gst::Buffer, T::InfoType)>,
|
2019-07-17 16:10:20 +00:00
|
|
|
|
|
|
|
error: Option<gst::FlowError>,
|
|
|
|
timeout: bool,
|
|
|
|
}
|
|
|
|
|
2019-07-18 21:05:31 +00:00
|
|
|
// 100 frames observations window over which we calculate the timestamp drift
|
|
|
|
// between sender and receiver. A bigger window allows more smoothing out of
|
|
|
|
// network effects
|
|
|
|
const WINDOW_LENGTH: usize = 100;
|
|
|
|
#[derive(Clone)]
|
|
|
|
struct Observations(Arc<Mutex<ObservationsInner>>);
|
|
|
|
struct ObservationsInner {
|
|
|
|
// NDI timestamp - GStreamer clock time tuples
|
|
|
|
values: Vec<(u64, u64)>,
|
|
|
|
values_tmp: [(u64, u64); WINDOW_LENGTH],
|
|
|
|
current_mapping: TimeMapping,
|
|
|
|
next_mapping: TimeMapping,
|
|
|
|
time_mapping_pending: bool,
|
|
|
|
|
|
|
|
// How many frames we skipped since last observation
|
|
|
|
// we took
|
|
|
|
skip_count: usize,
|
|
|
|
// How many frames we skip in this period. once skip_count
|
|
|
|
// reaches this, we take another observation
|
|
|
|
skip_period: usize,
|
|
|
|
// How many observations are left until we update the skip_period
|
|
|
|
// again. This is always initialized to WINDOW_LENGTH
|
|
|
|
skip_period_update_in: usize,
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone, Copy, Debug)]
|
|
|
|
struct TimeMapping {
|
|
|
|
xbase: u64,
|
|
|
|
b: u64,
|
|
|
|
num: u64,
|
|
|
|
den: u64,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Observations {
|
|
|
|
fn new() -> Self {
|
|
|
|
Self(Arc::new(Mutex::new(ObservationsInner {
|
|
|
|
values: Vec::with_capacity(WINDOW_LENGTH),
|
|
|
|
values_tmp: [(0, 0); WINDOW_LENGTH],
|
|
|
|
current_mapping: TimeMapping::default(),
|
|
|
|
next_mapping: TimeMapping::default(),
|
|
|
|
time_mapping_pending: false,
|
|
|
|
skip_count: 0,
|
|
|
|
skip_period: 1,
|
|
|
|
skip_period_update_in: WINDOW_LENGTH,
|
|
|
|
})))
|
|
|
|
}
|
|
|
|
|
|
|
|
fn process(
|
|
|
|
&self,
|
|
|
|
cat: gst::DebugCategory,
|
|
|
|
element: &gst_base::BaseSrc,
|
2021-09-13 09:26:56 +00:00
|
|
|
time: (Option<gst::ClockTime>, gst::ClockTime),
|
|
|
|
duration: Option<gst::ClockTime>,
|
|
|
|
) -> (gst::ClockTime, Option<gst::ClockTime>) {
|
2019-07-18 21:05:31 +00:00
|
|
|
if time.0.is_none() {
|
|
|
|
return (time.1, duration);
|
|
|
|
}
|
|
|
|
|
2021-09-13 09:26:56 +00:00
|
|
|
let time = (time.0.unwrap(), time.1);
|
2019-07-18 21:05:31 +00:00
|
|
|
|
|
|
|
let mut inner = self.0.lock().unwrap();
|
|
|
|
let ObservationsInner {
|
|
|
|
ref mut values,
|
|
|
|
ref mut values_tmp,
|
|
|
|
ref mut current_mapping,
|
|
|
|
ref mut next_mapping,
|
|
|
|
ref mut time_mapping_pending,
|
|
|
|
ref mut skip_count,
|
|
|
|
ref mut skip_period,
|
|
|
|
ref mut skip_period_update_in,
|
|
|
|
} = *inner;
|
|
|
|
|
|
|
|
if values.is_empty() {
|
2021-09-13 09:26:56 +00:00
|
|
|
current_mapping.xbase = time.0.nseconds();
|
|
|
|
current_mapping.b = time.1.nseconds();
|
2019-07-18 21:05:31 +00:00
|
|
|
current_mapping.num = 1;
|
|
|
|
current_mapping.den = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if *skip_count == 0 {
|
|
|
|
*skip_count += 1;
|
|
|
|
if *skip_count >= *skip_period {
|
|
|
|
*skip_count = 0;
|
|
|
|
}
|
|
|
|
*skip_period_update_in -= 1;
|
|
|
|
if *skip_period_update_in == 0 {
|
|
|
|
*skip_period_update_in = WINDOW_LENGTH;
|
|
|
|
|
|
|
|
// Start by first updating every frame, then every second frame, then every third
|
|
|
|
// frame, etc. until we update once every quarter second
|
2021-09-13 09:26:56 +00:00
|
|
|
let framerate = gst::ClockTime::SECOND
|
|
|
|
.checked_div(duration.unwrap_or(40 * gst::ClockTime::MSECOND).nseconds())
|
|
|
|
.unwrap_or(25) as usize;
|
2019-07-18 21:05:31 +00:00
|
|
|
|
|
|
|
if *skip_period < framerate / 4 + 1 {
|
|
|
|
*skip_period += 1;
|
|
|
|
} else {
|
|
|
|
*skip_period = framerate / 4 + 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
assert!(values.len() <= WINDOW_LENGTH);
|
|
|
|
|
|
|
|
if values.len() == WINDOW_LENGTH {
|
|
|
|
values.remove(0);
|
|
|
|
}
|
2021-09-13 09:26:56 +00:00
|
|
|
values.push((time.0.nseconds(), time.1.nseconds()));
|
2019-07-18 21:05:31 +00:00
|
|
|
|
|
|
|
if let Some((num, den, b, xbase, r_squared)) =
|
2019-08-14 19:49:55 +00:00
|
|
|
gst::calculate_linear_regression(values, Some(values_tmp))
|
2019-07-18 21:05:31 +00:00
|
|
|
{
|
|
|
|
next_mapping.xbase = xbase;
|
|
|
|
next_mapping.b = b;
|
|
|
|
next_mapping.num = num;
|
|
|
|
next_mapping.den = den;
|
|
|
|
*time_mapping_pending = true;
|
|
|
|
gst_debug!(
|
|
|
|
cat,
|
|
|
|
obj: element,
|
|
|
|
"Calculated new time mapping: GStreamer time = {} * (NDI time - {}) + {} ({})",
|
|
|
|
next_mapping.num as f64 / next_mapping.den as f64,
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::ClockTime::from_nseconds(next_mapping.xbase),
|
|
|
|
gst::ClockTime::from_nseconds(next_mapping.b),
|
2019-07-18 21:05:31 +00:00
|
|
|
r_squared,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
*skip_count += 1;
|
|
|
|
if *skip_count >= *skip_period {
|
|
|
|
*skip_count = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if *time_mapping_pending {
|
|
|
|
let expected = gst::Clock::adjust_with_calibration(
|
2021-09-13 09:26:56 +00:00
|
|
|
time.0,
|
|
|
|
gst::ClockTime::from_nseconds(current_mapping.xbase),
|
|
|
|
gst::ClockTime::from_nseconds(current_mapping.b),
|
|
|
|
gst::ClockTime::from_nseconds(current_mapping.num),
|
|
|
|
gst::ClockTime::from_nseconds(current_mapping.den),
|
2019-07-19 09:51:06 +00:00
|
|
|
);
|
2019-07-18 21:05:31 +00:00
|
|
|
let new_calculated = gst::Clock::adjust_with_calibration(
|
2021-09-13 09:26:56 +00:00
|
|
|
time.0,
|
|
|
|
gst::ClockTime::from_nseconds(next_mapping.xbase),
|
|
|
|
gst::ClockTime::from_nseconds(next_mapping.b),
|
|
|
|
gst::ClockTime::from_nseconds(next_mapping.num),
|
|
|
|
gst::ClockTime::from_nseconds(next_mapping.den),
|
2019-07-18 21:05:31 +00:00
|
|
|
);
|
|
|
|
|
2021-09-13 09:26:56 +00:00
|
|
|
let diff = if new_calculated > expected {
|
|
|
|
new_calculated - expected
|
|
|
|
} else {
|
|
|
|
expected - new_calculated
|
|
|
|
};
|
2019-07-19 09:51:06 +00:00
|
|
|
|
2021-09-13 09:26:56 +00:00
|
|
|
// Allow at most 5% frame duration or 2ms difference per frame
|
|
|
|
let max_diff = cmp::max(
|
|
|
|
(duration.map(|d| d / 10)).unwrap_or(2 * gst::ClockTime::MSECOND),
|
|
|
|
2 * gst::ClockTime::MSECOND,
|
|
|
|
);
|
2019-07-18 21:05:31 +00:00
|
|
|
|
2021-09-13 09:26:56 +00:00
|
|
|
if diff > max_diff {
|
|
|
|
gst_debug!(
|
|
|
|
cat,
|
|
|
|
obj: element,
|
|
|
|
"New time mapping causes difference {} but only {} allowed",
|
|
|
|
diff,
|
|
|
|
max_diff,
|
|
|
|
);
|
2019-07-19 09:51:06 +00:00
|
|
|
|
2021-09-13 09:26:56 +00:00
|
|
|
if new_calculated > expected {
|
|
|
|
current_mapping.b = (expected + max_diff).nseconds();
|
|
|
|
current_mapping.xbase = time.0.nseconds();
|
2019-07-18 21:05:31 +00:00
|
|
|
} else {
|
2021-09-13 09:26:56 +00:00
|
|
|
current_mapping.b = (expected - max_diff).nseconds();
|
|
|
|
current_mapping.xbase = time.0.nseconds();
|
2019-07-18 21:05:31 +00:00
|
|
|
}
|
|
|
|
} else {
|
2021-09-13 09:26:56 +00:00
|
|
|
*current_mapping = *next_mapping;
|
2019-07-18 21:05:31 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
let converted_timestamp = gst::Clock::adjust_with_calibration(
|
2021-09-13 09:26:56 +00:00
|
|
|
time.0,
|
|
|
|
gst::ClockTime::from_nseconds(current_mapping.xbase),
|
|
|
|
gst::ClockTime::from_nseconds(current_mapping.b),
|
|
|
|
gst::ClockTime::from_nseconds(current_mapping.num),
|
|
|
|
gst::ClockTime::from_nseconds(current_mapping.den),
|
2019-07-18 21:05:31 +00:00
|
|
|
);
|
2021-09-13 09:26:56 +00:00
|
|
|
let converted_duration =
|
|
|
|
duration.and_then(|d| d.mul_div_floor(current_mapping.num, current_mapping.den));
|
2019-07-18 21:05:31 +00:00
|
|
|
|
|
|
|
gst_debug!(
|
|
|
|
cat,
|
|
|
|
obj: element,
|
|
|
|
"Converted timestamp {}/{} to {}, duration {} to {}",
|
2021-09-13 09:26:56 +00:00
|
|
|
time.0,
|
|
|
|
time.1,
|
|
|
|
converted_timestamp.display(),
|
|
|
|
duration.display(),
|
|
|
|
converted_duration.display(),
|
2019-07-18 21:05:31 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
(converted_timestamp, converted_duration)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Default for TimeMapping {
|
|
|
|
fn default() -> Self {
|
|
|
|
Self {
|
|
|
|
xbase: 0,
|
|
|
|
b: 0,
|
|
|
|
num: 1,
|
|
|
|
den: 1,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
pub struct ReceiverControlHandle<T: ReceiverType> {
|
|
|
|
queue: ReceiverQueue<T>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<T: ReceiverType> Clone for ReceiverControlHandle<T> {
|
|
|
|
fn clone(&self) -> Self {
|
|
|
|
ReceiverControlHandle {
|
|
|
|
queue: self.queue.clone(),
|
|
|
|
}
|
|
|
|
}
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
impl<T: ReceiverType> ReceiverControlHandle<T> {
|
2019-07-17 16:10:20 +00:00
|
|
|
pub fn set_flushing(&self, flushing: bool) {
|
|
|
|
let mut queue = (self.queue.0).0.lock().unwrap();
|
|
|
|
queue.flushing = flushing;
|
|
|
|
(self.queue.0).1.notify_all();
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn set_playing(&self, playing: bool) {
|
|
|
|
let mut queue = (self.queue.0).0.lock().unwrap();
|
|
|
|
queue.playing = playing;
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn shutdown(&self) {
|
|
|
|
let mut queue = (self.queue.0).0.lock().unwrap();
|
|
|
|
queue.capturing = false;
|
|
|
|
(self.queue.0).1.notify_all();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
impl<T: ReceiverType> Receiver<T> {
|
2019-07-17 16:10:20 +00:00
|
|
|
fn new(
|
|
|
|
info: &mut ReceiverInfo,
|
|
|
|
timestamp_mode: TimestampMode,
|
|
|
|
timeout: u32,
|
2020-01-16 09:14:10 +00:00
|
|
|
connect_timeout: u32,
|
2021-08-02 05:45:32 +00:00
|
|
|
max_queue_length: usize,
|
2019-07-17 16:10:20 +00:00
|
|
|
element: &gst_base::BaseSrc,
|
|
|
|
cat: gst::DebugCategory,
|
2019-07-19 08:32:04 +00:00
|
|
|
) -> Self
|
|
|
|
where
|
|
|
|
Receiver<T>: ReceiverCapture<T>,
|
|
|
|
{
|
2019-07-17 16:10:20 +00:00
|
|
|
let receiver = Receiver(Arc::new(ReceiverInner {
|
2020-01-16 09:14:10 +00:00
|
|
|
id: info.id,
|
2019-07-17 16:10:20 +00:00
|
|
|
queue: ReceiverQueue(Arc::new((
|
|
|
|
Mutex::new(ReceiverQueueInner {
|
|
|
|
capturing: true,
|
|
|
|
playing: false,
|
|
|
|
flushing: false,
|
2021-08-02 05:45:32 +00:00
|
|
|
buffer_queue: VecDeque::with_capacity(max_queue_length),
|
2019-07-17 16:10:20 +00:00
|
|
|
error: None,
|
|
|
|
timeout: false,
|
|
|
|
}),
|
|
|
|
Condvar::new(),
|
|
|
|
))),
|
2021-08-02 05:45:32 +00:00
|
|
|
max_queue_length,
|
2020-01-16 09:14:10 +00:00
|
|
|
recv: Mutex::new(info.recv.clone()),
|
|
|
|
observations: info.observations.clone(),
|
2019-07-17 16:10:20 +00:00
|
|
|
cat,
|
|
|
|
element: element.downgrade(),
|
|
|
|
timestamp_mode,
|
2020-01-16 09:14:10 +00:00
|
|
|
first_frame: AtomicBool::new(true),
|
2019-07-17 16:10:20 +00:00
|
|
|
timeout,
|
2020-01-16 09:14:10 +00:00
|
|
|
connect_timeout,
|
2019-07-17 16:10:20 +00:00
|
|
|
thread: Mutex::new(None),
|
|
|
|
}));
|
|
|
|
|
|
|
|
let weak = Arc::downgrade(&receiver.0);
|
|
|
|
let thread = thread::spawn(move || {
|
|
|
|
use std::panic;
|
|
|
|
|
|
|
|
let weak_clone = weak.clone();
|
|
|
|
match panic::catch_unwind(panic::AssertUnwindSafe(move || receive_thread(&weak_clone)))
|
|
|
|
{
|
|
|
|
Ok(_) => (),
|
|
|
|
Err(_) => {
|
|
|
|
if let Some(receiver) = weak.upgrade().map(Receiver) {
|
|
|
|
if let Some(element) = receiver.0.element.upgrade() {
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::element_error!(
|
2019-07-17 16:10:20 +00:00
|
|
|
element,
|
|
|
|
gst::LibraryError::Failed,
|
|
|
|
["Panic while connecting to NDI source"]
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut queue = (receiver.0.queue.0).0.lock().unwrap();
|
|
|
|
queue.error = Some(gst::FlowError::Error);
|
|
|
|
(receiver.0.queue.0).1.notify_one();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
let weak = Arc::downgrade(&receiver.0);
|
2020-01-16 09:14:10 +00:00
|
|
|
Self::store_internal(info, weak);
|
2019-07-17 16:10:20 +00:00
|
|
|
|
|
|
|
*receiver.0.thread.lock().unwrap() = Some(thread);
|
|
|
|
|
|
|
|
receiver
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
pub fn receiver_control_handle(&self) -> ReceiverControlHandle<T> {
|
2019-07-17 16:10:20 +00:00
|
|
|
ReceiverControlHandle {
|
|
|
|
queue: self.0.queue.clone(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn set_flushing(&self, flushing: bool) {
|
|
|
|
let mut queue = (self.0.queue.0).0.lock().unwrap();
|
|
|
|
queue.flushing = flushing;
|
|
|
|
(self.0.queue.0).1.notify_all();
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn set_playing(&self, playing: bool) {
|
|
|
|
let mut queue = (self.0.queue.0).0.lock().unwrap();
|
|
|
|
queue.playing = playing;
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn shutdown(&self) {
|
|
|
|
let mut queue = (self.0.queue.0).0.lock().unwrap();
|
|
|
|
queue.capturing = false;
|
|
|
|
(self.0.queue.0).1.notify_all();
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
pub fn capture(&self) -> ReceiverItem<T> {
|
2019-07-17 16:10:20 +00:00
|
|
|
let mut queue = (self.0.queue.0).0.lock().unwrap();
|
|
|
|
loop {
|
|
|
|
if let Some(err) = queue.error {
|
|
|
|
return ReceiverItem::Error(err);
|
|
|
|
} else if queue.buffer_queue.is_empty() && queue.timeout {
|
|
|
|
return ReceiverItem::Timeout;
|
|
|
|
} else if queue.flushing || !queue.capturing {
|
|
|
|
return ReceiverItem::Flushing;
|
2019-07-19 08:32:04 +00:00
|
|
|
} else if let Some((buffer, info)) = queue.buffer_queue.pop_front() {
|
|
|
|
return ReceiverItem::Buffer(buffer, info);
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
queue = (self.0.queue.0).1.wait(queue).unwrap();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
impl<T: ReceiverType> Drop for ReceiverInner<T> {
|
2019-07-17 16:10:20 +00:00
|
|
|
fn drop(&mut self) {
|
|
|
|
// Will shut down the receiver thread on the next iteration
|
|
|
|
let mut queue = (self.queue.0).0.lock().unwrap();
|
|
|
|
queue.capturing = false;
|
|
|
|
drop(queue);
|
|
|
|
|
|
|
|
let element = self.element.upgrade();
|
|
|
|
|
|
|
|
if let Some(ref element) = element {
|
|
|
|
gst_debug!(self.cat, obj: element, "Closing NDI connection...");
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut receivers = HASHMAP_RECEIVERS.lock().unwrap();
|
|
|
|
{
|
2020-01-16 09:14:10 +00:00
|
|
|
let receiver = receivers.get_mut(&self.id).unwrap();
|
|
|
|
if receiver.audio.is_some() && receiver.video.is_some() {
|
2019-07-19 08:32:04 +00:00
|
|
|
if T::IS_VIDEO {
|
2020-01-16 09:14:10 +00:00
|
|
|
receiver.video = None;
|
2019-07-17 16:10:20 +00:00
|
|
|
} else {
|
2020-01-16 09:14:10 +00:00
|
|
|
receiver.audio = None;
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
receivers.remove(&self.id);
|
|
|
|
|
|
|
|
if let Some(ref element) = element {
|
|
|
|
gst_debug!(self.cat, obj: element, "Closed NDI connection");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
pub fn connect_ndi<T: ReceiverType>(
|
2019-07-17 16:10:20 +00:00
|
|
|
cat: gst::DebugCategory,
|
|
|
|
element: &gst_base::BaseSrc,
|
2020-01-16 10:18:57 +00:00
|
|
|
ndi_name: Option<&str>,
|
2020-01-16 09:14:10 +00:00
|
|
|
url_address: Option<&str>,
|
2019-07-17 16:10:20 +00:00
|
|
|
receiver_ndi_name: &str,
|
|
|
|
connect_timeout: u32,
|
|
|
|
bandwidth: NDIlib_recv_bandwidth_e,
|
|
|
|
timestamp_mode: TimestampMode,
|
|
|
|
timeout: u32,
|
2021-08-02 05:45:32 +00:00
|
|
|
max_queue_length: usize,
|
2019-07-19 08:32:04 +00:00
|
|
|
) -> Option<Receiver<T>>
|
|
|
|
where
|
|
|
|
Receiver<T>: ReceiverCapture<T>,
|
|
|
|
{
|
2019-07-17 16:10:20 +00:00
|
|
|
gst_debug!(cat, obj: element, "Starting NDI connection...");
|
|
|
|
|
2020-01-16 10:18:57 +00:00
|
|
|
assert!(ndi_name.is_some() || url_address.is_some());
|
|
|
|
|
2019-07-17 16:10:20 +00:00
|
|
|
let mut receivers = HASHMAP_RECEIVERS.lock().unwrap();
|
|
|
|
|
|
|
|
// Check if we already have a receiver for this very stream
|
2020-01-16 09:14:10 +00:00
|
|
|
for receiver in receivers.values_mut() {
|
2020-01-16 10:18:57 +00:00
|
|
|
// If both are provided they both must match, if only one is provided
|
|
|
|
// then that one has to match and the other one does not matter
|
|
|
|
if (ndi_name.is_some()
|
|
|
|
&& url_address.is_some()
|
2021-02-23 10:46:27 +00:00
|
|
|
&& receiver.ndi_name.as_deref() == ndi_name
|
|
|
|
&& receiver.url_address.as_deref() == url_address)
|
2020-01-16 10:18:57 +00:00
|
|
|
|| (ndi_name.is_some()
|
|
|
|
&& url_address.is_none()
|
2021-02-23 10:46:27 +00:00
|
|
|
&& receiver.ndi_name.as_deref() == ndi_name)
|
2020-01-16 10:18:57 +00:00
|
|
|
|| (ndi_name.is_none()
|
|
|
|
&& url_address.is_some()
|
2021-02-23 10:46:27 +00:00
|
|
|
&& receiver.url_address.as_deref() == url_address)
|
2020-01-16 09:14:10 +00:00
|
|
|
{
|
|
|
|
if (receiver.video.is_some() || !T::IS_VIDEO)
|
|
|
|
&& (receiver.audio.is_some() || T::IS_VIDEO)
|
|
|
|
{
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::element_error!(
|
2019-07-19 09:51:06 +00:00
|
|
|
element,
|
|
|
|
gst::ResourceError::OpenRead,
|
|
|
|
[
|
2020-01-16 10:18:57 +00:00
|
|
|
"Source with NDI name '{:?}' / URL/address '{:?}' already in use for {}",
|
2020-01-16 09:14:10 +00:00
|
|
|
receiver.ndi_name,
|
2020-01-16 10:18:57 +00:00
|
|
|
receiver.url_address,
|
2019-07-19 09:51:06 +00:00
|
|
|
if T::IS_VIDEO { "video" } else { "audio" }
|
|
|
|
]
|
2019-07-17 16:10:20 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
return None;
|
|
|
|
} else {
|
2020-01-16 09:14:10 +00:00
|
|
|
return Some(Receiver::new(
|
|
|
|
receiver,
|
|
|
|
timestamp_mode,
|
|
|
|
timeout,
|
|
|
|
connect_timeout,
|
2021-08-02 05:45:32 +00:00
|
|
|
max_queue_length,
|
2020-01-16 09:14:10 +00:00
|
|
|
element,
|
2019-07-19 09:15:08 +00:00
|
|
|
cat,
|
2020-01-16 09:14:10 +00:00
|
|
|
));
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
}
|
2020-01-16 09:14:10 +00:00
|
|
|
}
|
2019-07-17 16:10:20 +00:00
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
// Otherwise create a new one and return it to the caller
|
2019-07-17 16:10:20 +00:00
|
|
|
gst_debug!(
|
|
|
|
cat,
|
|
|
|
obj: element,
|
2020-01-16 10:18:57 +00:00
|
|
|
"Connecting to NDI source with NDI name '{:?}' and URL/Address {:?}",
|
2020-01-16 09:14:10 +00:00
|
|
|
ndi_name,
|
|
|
|
url_address,
|
2019-07-17 16:10:20 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
// FIXME: Ideally we would use NDIlib_recv_color_format_fastest here but that seems to be
|
|
|
|
// broken with interlaced content currently
|
2021-09-13 09:26:56 +00:00
|
|
|
let recv = RecvInstance::builder(ndi_name, url_address, receiver_ndi_name)
|
2019-07-17 16:10:20 +00:00
|
|
|
.bandwidth(bandwidth)
|
|
|
|
.color_format(NDIlib_recv_color_format_e::NDIlib_recv_color_format_UYVY_BGRA)
|
|
|
|
.allow_video_fields(true)
|
|
|
|
.build();
|
|
|
|
let recv = match recv {
|
|
|
|
None => {
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::element_error!(
|
2020-01-16 09:14:10 +00:00
|
|
|
element,
|
2019-07-17 16:10:20 +00:00
|
|
|
gst::CoreError::Negotiation,
|
2019-07-19 09:51:06 +00:00
|
|
|
["Failed to connect to source"]
|
2020-01-16 09:14:10 +00:00
|
|
|
);
|
|
|
|
return None;
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
Some(recv) => recv,
|
|
|
|
};
|
|
|
|
|
|
|
|
recv.set_tally(&Tally::default());
|
|
|
|
|
|
|
|
let enable_hw_accel = MetadataFrame::new(0, Some("<ndi_hwaccel enabled=\"true\"/>"));
|
|
|
|
recv.send_metadata(&enable_hw_accel);
|
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
let id_receiver = ID_RECEIVER.fetch_add(1, Ordering::SeqCst);
|
|
|
|
let mut info = ReceiverInfo {
|
2019-07-17 16:10:20 +00:00
|
|
|
id: id_receiver,
|
2020-01-16 10:18:57 +00:00
|
|
|
ndi_name: ndi_name.map(String::from),
|
2020-01-16 09:14:10 +00:00
|
|
|
url_address: url_address.map(String::from),
|
|
|
|
recv,
|
|
|
|
video: None,
|
|
|
|
audio: None,
|
|
|
|
observations: Observations::new(),
|
2019-07-17 16:10:20 +00:00
|
|
|
};
|
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
// This will set info.audio/video accordingly
|
|
|
|
let receiver = Receiver::new(
|
|
|
|
&mut info,
|
|
|
|
timestamp_mode,
|
|
|
|
timeout,
|
|
|
|
connect_timeout,
|
2021-08-02 05:45:32 +00:00
|
|
|
max_queue_length,
|
2020-01-16 09:14:10 +00:00
|
|
|
element,
|
|
|
|
cat,
|
|
|
|
);
|
2019-07-17 16:10:20 +00:00
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
receivers.insert(id_receiver, info);
|
2019-07-17 16:10:20 +00:00
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
Some(receiver)
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
fn receive_thread<T: ReceiverType>(receiver: &Weak<ReceiverInner<T>>)
|
|
|
|
where
|
|
|
|
Receiver<T>: ReceiverCapture<T>,
|
|
|
|
{
|
2019-07-17 16:10:20 +00:00
|
|
|
// Now first capture frames until the queues are empty so that we're sure that we output only
|
|
|
|
// the very latest frame that is available now
|
|
|
|
loop {
|
|
|
|
let receiver = match receiver.upgrade().map(Receiver) {
|
|
|
|
None => return,
|
|
|
|
Some(receiver) => receiver,
|
|
|
|
};
|
|
|
|
|
2019-07-19 09:51:06 +00:00
|
|
|
let element = match receiver.0.element.upgrade() {
|
|
|
|
None => return,
|
|
|
|
Some(element) => element,
|
|
|
|
};
|
2019-07-17 16:10:20 +00:00
|
|
|
|
|
|
|
{
|
|
|
|
let queue = (receiver.0.queue.0).0.lock().unwrap();
|
|
|
|
if !queue.capturing {
|
|
|
|
gst_debug!(receiver.0.cat, obj: &element, "Shutting down");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// If an error happened in the meantime, just go out of here
|
2019-07-30 15:29:42 +00:00
|
|
|
if queue.error.is_some() {
|
2019-07-17 16:10:20 +00:00
|
|
|
gst_error!(
|
|
|
|
receiver.0.cat,
|
|
|
|
obj: &element,
|
|
|
|
"Error while waiting for connection"
|
|
|
|
);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
let recv = receiver.0.recv.lock().unwrap();
|
|
|
|
|
2019-07-17 16:10:20 +00:00
|
|
|
let queue = recv.get_queue();
|
2019-07-19 08:32:04 +00:00
|
|
|
if (!T::IS_VIDEO && queue.audio_frames() <= 1) || (T::IS_VIDEO && queue.video_frames() <= 1)
|
2019-07-17 16:10:20 +00:00
|
|
|
{
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
let _ = recv.capture(T::IS_VIDEO, !T::IS_VIDEO, false, 0);
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// And if that went fine, capture until we're done
|
|
|
|
loop {
|
|
|
|
let receiver = match receiver.upgrade().map(Receiver) {
|
|
|
|
None => break,
|
|
|
|
Some(receiver) => receiver,
|
|
|
|
};
|
|
|
|
|
2019-07-19 09:51:06 +00:00
|
|
|
let element = match receiver.0.element.upgrade() {
|
|
|
|
None => return,
|
|
|
|
Some(element) => element,
|
|
|
|
};
|
2019-07-17 16:10:20 +00:00
|
|
|
|
|
|
|
{
|
|
|
|
let queue = (receiver.0.queue.0).0.lock().unwrap();
|
|
|
|
if !queue.capturing {
|
|
|
|
gst_debug!(receiver.0.cat, obj: &element, "Shutting down");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
let recv = receiver.0.recv.lock().unwrap();
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
let res = receiver.capture_internal(&element, &recv);
|
2019-07-17 16:10:20 +00:00
|
|
|
|
|
|
|
match res {
|
|
|
|
Ok(item) => {
|
|
|
|
let mut queue = (receiver.0.queue.0).0.lock().unwrap();
|
2021-08-02 05:45:32 +00:00
|
|
|
while queue.buffer_queue.len() > receiver.0.max_queue_length {
|
2019-07-17 16:10:20 +00:00
|
|
|
gst_warning!(
|
|
|
|
receiver.0.cat,
|
|
|
|
obj: &element,
|
|
|
|
"Dropping old buffer -- queue has {} items",
|
|
|
|
queue.buffer_queue.len()
|
|
|
|
);
|
|
|
|
queue.buffer_queue.pop_front();
|
|
|
|
}
|
|
|
|
queue.buffer_queue.push_back(item);
|
|
|
|
(receiver.0.queue.0).1.notify_one();
|
|
|
|
}
|
|
|
|
Err(gst::FlowError::Eos) => {
|
|
|
|
gst_debug!(receiver.0.cat, obj: &element, "Signalling EOS");
|
|
|
|
let mut queue = (receiver.0.queue.0).0.lock().unwrap();
|
|
|
|
queue.timeout = true;
|
|
|
|
(receiver.0.queue.0).1.notify_one();
|
|
|
|
}
|
|
|
|
Err(gst::FlowError::CustomError) => {
|
|
|
|
// Flushing, nothing to be done here except for emptying our queue
|
|
|
|
let mut queue = (receiver.0.queue.0).0.lock().unwrap();
|
|
|
|
queue.buffer_queue.clear();
|
|
|
|
(receiver.0.queue.0).1.notify_one();
|
|
|
|
}
|
|
|
|
Err(err) => {
|
|
|
|
gst_error!(receiver.0.cat, obj: &element, "Signalling error");
|
|
|
|
let mut queue = (receiver.0.queue.0).0.lock().unwrap();
|
|
|
|
if queue.error.is_none() {
|
|
|
|
queue.error = Some(err);
|
|
|
|
}
|
|
|
|
(receiver.0.queue.0).1.notify_one();
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
pub trait ReceiverCapture<T: ReceiverType> {
|
|
|
|
fn capture_internal(
|
|
|
|
&self,
|
|
|
|
element: &gst_base::BaseSrc,
|
|
|
|
recv: &RecvInstance,
|
|
|
|
) -> Result<(gst::Buffer, T::InfoType), gst::FlowError>;
|
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
fn store_internal(info: &mut ReceiverInfo, weak: Weak<ReceiverInner<T>>);
|
2019-07-19 08:32:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl ReceiverCapture<VideoReceiver> for Receiver<VideoReceiver> {
|
|
|
|
fn capture_internal(
|
|
|
|
&self,
|
|
|
|
element: &gst_base::BaseSrc,
|
|
|
|
recv: &RecvInstance,
|
|
|
|
) -> Result<(gst::Buffer, gst_video::VideoInfo), gst::FlowError> {
|
|
|
|
self.capture_video(element, recv)
|
|
|
|
}
|
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
fn store_internal(info: &mut ReceiverInfo, weak: Weak<ReceiverInner<VideoReceiver>>) {
|
|
|
|
assert!(info.video.is_none());
|
|
|
|
info.video = Some(weak);
|
2019-07-19 08:32:04 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-19 09:51:06 +00:00
|
|
|
impl<T: ReceiverType> Receiver<T> {
|
|
|
|
fn calculate_timestamp(
|
|
|
|
&self,
|
|
|
|
element: &gst_base::BaseSrc,
|
|
|
|
timestamp: i64,
|
|
|
|
timecode: i64,
|
2021-09-13 09:26:56 +00:00
|
|
|
duration: Option<gst::ClockTime>,
|
|
|
|
) -> Option<(gst::ClockTime, Option<gst::ClockTime>)> {
|
|
|
|
let clock = element.clock()?;
|
2019-07-19 09:51:06 +00:00
|
|
|
|
|
|
|
// For now take the current running time as PTS. At a later time we
|
|
|
|
// will want to work with the timestamp given by the NDI SDK if available
|
2021-09-13 09:26:56 +00:00
|
|
|
let now = clock.time()?;
|
|
|
|
let base_time = element.base_time()?;
|
2019-07-19 09:51:06 +00:00
|
|
|
let receive_time = now - base_time;
|
|
|
|
|
2021-09-13 09:26:56 +00:00
|
|
|
let real_time_now = gst::ClockTime::from_nseconds(glib::real_time() as u64 * 1000);
|
2019-07-19 09:51:06 +00:00
|
|
|
let timestamp = if timestamp == ndisys::NDIlib_recv_timestamp_undefined {
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::ClockTime::NONE
|
2019-07-19 09:51:06 +00:00
|
|
|
} else {
|
2021-09-13 09:26:56 +00:00
|
|
|
Some(gst::ClockTime::from_nseconds(timestamp as u64 * 100))
|
2019-07-19 09:51:06 +00:00
|
|
|
};
|
2021-09-13 09:26:56 +00:00
|
|
|
let timecode = gst::ClockTime::from_nseconds(timecode as u64 * 100);
|
2019-07-19 09:51:06 +00:00
|
|
|
|
|
|
|
gst_log!(
|
|
|
|
self.0.cat,
|
|
|
|
obj: element,
|
|
|
|
"Received frame with timecode {}, timestamp {}, duration {}, receive time {}, local time now {}",
|
|
|
|
timecode,
|
2021-09-13 09:26:56 +00:00
|
|
|
timestamp.display(),
|
|
|
|
duration.display(),
|
|
|
|
receive_time.display(),
|
2019-07-19 09:51:06 +00:00
|
|
|
real_time_now,
|
|
|
|
);
|
|
|
|
|
|
|
|
let (pts, duration) = match self.0.timestamp_mode {
|
2021-09-13 09:26:56 +00:00
|
|
|
TimestampMode::ReceiveTimeTimecode => self.0.observations.process(
|
|
|
|
self.0.cat,
|
|
|
|
element,
|
|
|
|
(Some(timecode), receive_time),
|
|
|
|
duration,
|
|
|
|
),
|
2021-02-25 09:31:32 +00:00
|
|
|
TimestampMode::ReceiveTimeTimestamp => self.0.observations.process(
|
2019-07-19 09:51:06 +00:00
|
|
|
self.0.cat,
|
|
|
|
element,
|
|
|
|
(timestamp, receive_time),
|
|
|
|
duration,
|
|
|
|
),
|
|
|
|
TimestampMode::Timecode => (timecode, duration),
|
|
|
|
TimestampMode::Timestamp if timestamp.is_none() => (receive_time, duration),
|
|
|
|
TimestampMode::Timestamp => {
|
|
|
|
// Timestamps are relative to the UNIX epoch
|
2021-09-13 09:26:56 +00:00
|
|
|
let timestamp = timestamp?;
|
2019-07-19 09:51:06 +00:00
|
|
|
if real_time_now > timestamp {
|
|
|
|
let diff = real_time_now - timestamp;
|
|
|
|
if diff > receive_time {
|
2021-09-13 09:26:56 +00:00
|
|
|
(gst::ClockTime::ZERO, duration)
|
2019-07-19 09:51:06 +00:00
|
|
|
} else {
|
|
|
|
(receive_time - diff, duration)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
let diff = timestamp - real_time_now;
|
|
|
|
(receive_time + diff, duration)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
gst_log!(
|
|
|
|
self.0.cat,
|
|
|
|
obj: element,
|
|
|
|
"Calculated PTS {}, duration {}",
|
2021-09-13 09:26:56 +00:00
|
|
|
pts.display(),
|
|
|
|
duration.display(),
|
2019-07-19 09:51:06 +00:00
|
|
|
);
|
|
|
|
|
2019-07-19 10:00:27 +00:00
|
|
|
Some((pts, duration))
|
2019-07-19 09:51:06 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
impl ReceiverCapture<AudioReceiver> for Receiver<AudioReceiver> {
|
|
|
|
fn capture_internal(
|
|
|
|
&self,
|
|
|
|
element: &gst_base::BaseSrc,
|
|
|
|
recv: &RecvInstance,
|
|
|
|
) -> Result<(gst::Buffer, gst_audio::AudioInfo), gst::FlowError> {
|
|
|
|
self.capture_audio(element, recv)
|
|
|
|
}
|
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
fn store_internal(info: &mut ReceiverInfo, weak: Weak<ReceiverInner<AudioReceiver>>) {
|
|
|
|
assert!(info.audio.is_none());
|
|
|
|
info.audio = Some(weak);
|
2019-07-19 08:32:04 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Receiver<VideoReceiver> {
|
2019-07-17 16:10:20 +00:00
|
|
|
fn capture_video(
|
|
|
|
&self,
|
|
|
|
element: &gst_base::BaseSrc,
|
|
|
|
recv: &RecvInstance,
|
|
|
|
) -> Result<(gst::Buffer, gst_video::VideoInfo), gst::FlowError> {
|
2020-01-16 09:14:10 +00:00
|
|
|
let timer = time::Instant::now();
|
|
|
|
let timeout = if self.0.first_frame.load(Ordering::SeqCst) {
|
|
|
|
self.0.connect_timeout
|
|
|
|
} else {
|
|
|
|
self.0.timeout
|
|
|
|
};
|
2019-07-17 16:10:20 +00:00
|
|
|
let mut flushing;
|
|
|
|
let mut playing;
|
|
|
|
|
|
|
|
let video_frame = loop {
|
|
|
|
{
|
|
|
|
let queue = (self.0.queue.0).0.lock().unwrap();
|
|
|
|
playing = queue.playing;
|
|
|
|
flushing = queue.flushing;
|
|
|
|
if !queue.capturing {
|
|
|
|
gst_debug!(self.0.cat, obj: element, "Shutting down");
|
|
|
|
return Err(gst::FlowError::Flushing);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
let res = match recv.capture(true, false, false, 50) {
|
|
|
|
Err(_) => Err(()),
|
|
|
|
Ok(None) => Ok(None),
|
|
|
|
Ok(Some(Frame::Video(frame))) => Ok(Some(frame)),
|
|
|
|
_ => unreachable!(),
|
|
|
|
};
|
|
|
|
|
|
|
|
let video_frame = match res {
|
|
|
|
Err(_) => {
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::element_error!(
|
2019-07-19 09:51:06 +00:00
|
|
|
element,
|
|
|
|
gst::ResourceError::Read,
|
|
|
|
["Error receiving frame"]
|
|
|
|
);
|
2019-07-17 16:10:20 +00:00
|
|
|
return Err(gst::FlowError::Error);
|
|
|
|
}
|
2020-09-01 18:38:13 +00:00
|
|
|
Ok(None) if timeout > 0 && timer.elapsed().as_millis() >= timeout as u128 => {
|
2019-07-19 09:51:06 +00:00
|
|
|
gst_debug!(self.0.cat, obj: element, "Timed out -- assuming EOS",);
|
2019-07-17 16:10:20 +00:00
|
|
|
return Err(gst::FlowError::Eos);
|
|
|
|
}
|
|
|
|
Ok(None) => {
|
|
|
|
gst_debug!(
|
|
|
|
self.0.cat,
|
|
|
|
obj: element,
|
|
|
|
"No video frame received yet, retry"
|
|
|
|
);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
Ok(Some(frame)) => frame,
|
|
|
|
};
|
|
|
|
|
|
|
|
break video_frame;
|
|
|
|
};
|
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
self.0.first_frame.store(false, Ordering::SeqCst);
|
|
|
|
|
2019-07-19 09:51:06 +00:00
|
|
|
gst_debug!(
|
|
|
|
self.0.cat,
|
|
|
|
obj: element,
|
|
|
|
"Received video frame {:?}",
|
|
|
|
video_frame,
|
|
|
|
);
|
|
|
|
|
2019-07-19 10:00:27 +00:00
|
|
|
let (pts, duration) = self
|
|
|
|
.calculate_video_timestamp(element, &video_frame)
|
|
|
|
.ok_or_else(|| {
|
|
|
|
gst_debug!(self.0.cat, obj: element, "Flushing, dropping buffer");
|
|
|
|
gst::FlowError::CustomError
|
|
|
|
})?;
|
2019-07-18 21:05:31 +00:00
|
|
|
|
2019-07-17 16:10:20 +00:00
|
|
|
// Simply read all video frames while flushing but don't copy them or anything to
|
|
|
|
// make sure that we're not accumulating anything here
|
|
|
|
if !playing || flushing {
|
|
|
|
gst_debug!(self.0.cat, obj: element, "Flushing, dropping buffer");
|
|
|
|
return Err(gst::FlowError::CustomError);
|
|
|
|
}
|
|
|
|
|
|
|
|
let info = self.create_video_info(element, &video_frame)?;
|
|
|
|
|
2021-02-23 10:46:27 +00:00
|
|
|
let buffer = self.create_video_buffer(element, pts, duration, &info, &video_frame);
|
2019-07-17 16:10:20 +00:00
|
|
|
|
|
|
|
gst_log!(self.0.cat, obj: element, "Produced buffer {:?}", buffer);
|
|
|
|
|
|
|
|
Ok((buffer, info))
|
|
|
|
}
|
|
|
|
|
|
|
|
fn calculate_video_timestamp(
|
|
|
|
&self,
|
|
|
|
element: &gst_base::BaseSrc,
|
|
|
|
video_frame: &VideoFrame,
|
2021-09-13 09:26:56 +00:00
|
|
|
) -> Option<(gst::ClockTime, Option<gst::ClockTime>)> {
|
|
|
|
let duration = gst::ClockTime::SECOND.mul_div_floor(
|
|
|
|
video_frame.frame_rate().1 as u64,
|
|
|
|
video_frame.frame_rate().0 as u64,
|
|
|
|
);
|
2019-07-18 21:05:31 +00:00
|
|
|
|
2019-07-19 09:51:06 +00:00
|
|
|
self.calculate_timestamp(
|
|
|
|
element,
|
|
|
|
video_frame.timestamp(),
|
|
|
|
video_frame.timecode(),
|
2019-07-18 21:05:31 +00:00
|
|
|
duration,
|
2019-07-19 09:51:06 +00:00
|
|
|
)
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fn create_video_info(
|
|
|
|
&self,
|
2019-07-19 09:51:06 +00:00
|
|
|
element: &gst_base::BaseSrc,
|
2019-07-17 16:10:20 +00:00
|
|
|
video_frame: &VideoFrame,
|
|
|
|
) -> Result<gst_video::VideoInfo, gst::FlowError> {
|
|
|
|
// YV12 and I420 are swapped in the NDI SDK compared to GStreamer
|
|
|
|
let format = match video_frame.fourcc() {
|
2020-01-02 12:00:47 +00:00
|
|
|
ndisys::NDIlib_FourCC_video_type_UYVY => gst_video::VideoFormat::Uyvy,
|
|
|
|
// FIXME: This drops the alpha plane!
|
|
|
|
ndisys::NDIlib_FourCC_video_type_UYVA => gst_video::VideoFormat::Uyvy,
|
|
|
|
ndisys::NDIlib_FourCC_video_type_YV12 => gst_video::VideoFormat::I420,
|
|
|
|
ndisys::NDIlib_FourCC_video_type_NV12 => gst_video::VideoFormat::Nv12,
|
|
|
|
ndisys::NDIlib_FourCC_video_type_I420 => gst_video::VideoFormat::Yv12,
|
|
|
|
ndisys::NDIlib_FourCC_video_type_BGRA => gst_video::VideoFormat::Bgra,
|
|
|
|
ndisys::NDIlib_FourCC_video_type_BGRX => gst_video::VideoFormat::Bgrx,
|
|
|
|
ndisys::NDIlib_FourCC_video_type_RGBA => gst_video::VideoFormat::Rgba,
|
|
|
|
ndisys::NDIlib_FourCC_video_type_RGBX => gst_video::VideoFormat::Rgbx,
|
|
|
|
_ => {
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::element_error!(
|
2020-01-02 12:00:47 +00:00
|
|
|
element,
|
|
|
|
gst::StreamError::Format,
|
|
|
|
["Unsupported video fourcc {:08x}", video_frame.fourcc()]
|
|
|
|
);
|
|
|
|
|
|
|
|
return Err(gst::FlowError::NotNegotiated);
|
|
|
|
} // TODO: NDIlib_FourCC_video_type_P216 and NDIlib_FourCC_video_type_PA16 not
|
|
|
|
// supported by GStreamer
|
2019-07-17 16:10:20 +00:00
|
|
|
};
|
|
|
|
|
2019-07-19 09:51:06 +00:00
|
|
|
let par = gst::Fraction::approximate_f32(video_frame.picture_aspect_ratio())
|
2020-01-02 11:06:53 +00:00
|
|
|
.unwrap_or_else(|| gst::Fraction::new(1, 1))
|
2019-07-17 16:10:20 +00:00
|
|
|
* gst::Fraction::new(video_frame.yres(), video_frame.xres());
|
|
|
|
|
|
|
|
#[cfg(feature = "interlaced-fields")]
|
|
|
|
{
|
2020-07-27 13:08:56 +00:00
|
|
|
let mut builder = gst_video::VideoInfo::builder(
|
2019-07-17 16:10:20 +00:00
|
|
|
format,
|
|
|
|
video_frame.xres() as u32,
|
|
|
|
video_frame.yres() as u32,
|
|
|
|
)
|
|
|
|
.fps(gst::Fraction::from(video_frame.frame_rate()))
|
|
|
|
.par(par)
|
|
|
|
.interlace_mode(match video_frame.frame_format_type() {
|
|
|
|
ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_progressive => {
|
|
|
|
gst_video::VideoInterlaceMode::Progressive
|
|
|
|
}
|
|
|
|
ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_interleaved => {
|
|
|
|
gst_video::VideoInterlaceMode::Interleaved
|
|
|
|
}
|
|
|
|
_ => gst_video::VideoInterlaceMode::Alternate,
|
|
|
|
});
|
|
|
|
|
|
|
|
if video_frame.frame_format_type()
|
|
|
|
== ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_interleaved
|
|
|
|
{
|
|
|
|
builder = builder.field_order(gst_video::VideoFieldOrder::TopFieldFirst);
|
|
|
|
}
|
|
|
|
|
2019-12-18 17:34:00 +00:00
|
|
|
builder.build().map_err(|_| {
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::element_error!(
|
2019-07-19 09:51:06 +00:00
|
|
|
element,
|
|
|
|
gst::StreamError::Format,
|
|
|
|
["Invalid video format configuration"]
|
|
|
|
);
|
|
|
|
|
|
|
|
gst::FlowError::NotNegotiated
|
|
|
|
})
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(not(feature = "interlaced-fields"))]
|
|
|
|
{
|
|
|
|
if video_frame.frame_format_type()
|
|
|
|
!= ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_progressive
|
|
|
|
&& video_frame.frame_format_type()
|
|
|
|
!= ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_interleaved
|
|
|
|
{
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::element_error!(
|
2019-07-17 16:10:20 +00:00
|
|
|
element,
|
|
|
|
gst::StreamError::Format,
|
|
|
|
["Separate field interlacing not supported"]
|
|
|
|
);
|
|
|
|
return Err(gst::FlowError::NotNegotiated);
|
|
|
|
}
|
|
|
|
|
2020-07-27 13:08:56 +00:00
|
|
|
let mut builder = gst_video::VideoInfo::builder(
|
2019-07-17 16:10:20 +00:00
|
|
|
format,
|
|
|
|
video_frame.xres() as u32,
|
|
|
|
video_frame.yres() as u32,
|
|
|
|
)
|
|
|
|
.fps(gst::Fraction::from(video_frame.frame_rate()))
|
|
|
|
.par(par)
|
|
|
|
.interlace_mode(
|
|
|
|
if video_frame.frame_format_type()
|
|
|
|
== ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_progressive
|
|
|
|
{
|
|
|
|
gst_video::VideoInterlaceMode::Progressive
|
|
|
|
} else {
|
|
|
|
gst_video::VideoInterlaceMode::Interleaved
|
|
|
|
},
|
|
|
|
);
|
2019-07-18 21:05:31 +00:00
|
|
|
|
|
|
|
if video_frame.frame_format_type()
|
|
|
|
== ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_interleaved
|
|
|
|
{
|
|
|
|
builder = builder.field_order(gst_video::VideoFieldOrder::TopFieldFirst);
|
|
|
|
}
|
|
|
|
|
2019-12-18 17:34:00 +00:00
|
|
|
builder.build().map_err(|_| {
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::element_error!(
|
2019-07-19 09:51:06 +00:00
|
|
|
element,
|
|
|
|
gst::StreamError::Format,
|
|
|
|
["Invalid video format configuration"]
|
|
|
|
);
|
|
|
|
|
|
|
|
gst::FlowError::NotNegotiated
|
|
|
|
})
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn create_video_buffer(
|
|
|
|
&self,
|
|
|
|
element: &gst_base::BaseSrc,
|
|
|
|
pts: gst::ClockTime,
|
2021-09-13 09:26:56 +00:00
|
|
|
duration: Option<gst::ClockTime>,
|
2019-07-17 16:10:20 +00:00
|
|
|
info: &gst_video::VideoInfo,
|
|
|
|
video_frame: &VideoFrame,
|
2021-02-23 10:46:27 +00:00
|
|
|
) -> gst::Buffer {
|
2019-07-17 16:10:20 +00:00
|
|
|
let mut buffer = gst::Buffer::with_size(info.size()).unwrap();
|
|
|
|
{
|
|
|
|
let buffer = buffer.get_mut().unwrap();
|
|
|
|
buffer.set_pts(pts);
|
|
|
|
buffer.set_duration(duration);
|
|
|
|
|
|
|
|
#[cfg(feature = "reference-timestamps")]
|
|
|
|
{
|
|
|
|
gst::ReferenceTimestampMeta::add(
|
|
|
|
buffer,
|
|
|
|
&*TIMECODE_CAPS,
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::ClockTime::from_nseconds(video_frame.timecode() as u64 * 100),
|
|
|
|
gst::ClockTime::NONE,
|
2019-07-17 16:10:20 +00:00
|
|
|
);
|
|
|
|
if video_frame.timestamp() != ndisys::NDIlib_recv_timestamp_undefined {
|
|
|
|
gst::ReferenceTimestampMeta::add(
|
|
|
|
buffer,
|
|
|
|
&*TIMESTAMP_CAPS,
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::ClockTime::from_nseconds(video_frame.timestamp() as u64 * 100),
|
|
|
|
gst::ClockTime::NONE,
|
2019-07-17 16:10:20 +00:00
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(feature = "interlaced-fields")]
|
|
|
|
{
|
|
|
|
match video_frame.frame_format_type() {
|
|
|
|
ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_interleaved => {
|
|
|
|
buffer.set_video_flags(
|
|
|
|
gst_video::VideoBufferFlags::INTERLACED
|
|
|
|
| gst_video::VideoBufferFlags::TFF,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_field_0 => {
|
|
|
|
buffer.set_video_flags(
|
|
|
|
gst_video::VideoBufferFlags::INTERLACED
|
|
|
|
| gst_video::VideoBufferFlags::TOP_FIELD,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_field_1 => {
|
|
|
|
buffer.set_video_flags(
|
|
|
|
gst_video::VideoBufferFlags::INTERLACED
|
|
|
|
| gst_video::VideoBufferFlags::BOTTOM_FIELD,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
_ => (),
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(not(feature = "interlaced-fields"))]
|
|
|
|
{
|
|
|
|
if video_frame.frame_format_type()
|
|
|
|
== ndisys::NDIlib_frame_format_type_e::NDIlib_frame_format_type_interleaved
|
|
|
|
{
|
|
|
|
buffer.set_video_flags(
|
|
|
|
gst_video::VideoBufferFlags::INTERLACED | gst_video::VideoBufferFlags::TFF,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
self.copy_video_frame(element, info, buffer, video_frame)
|
|
|
|
}
|
|
|
|
|
|
|
|
fn copy_video_frame(
|
|
|
|
&self,
|
|
|
|
_element: &gst_base::BaseSrc,
|
|
|
|
info: &gst_video::VideoInfo,
|
|
|
|
buffer: gst::Buffer,
|
|
|
|
video_frame: &VideoFrame,
|
2021-02-23 10:46:27 +00:00
|
|
|
) -> gst::Buffer {
|
2019-07-17 16:10:20 +00:00
|
|
|
let mut vframe = gst_video::VideoFrame::from_buffer_writable(buffer, info).unwrap();
|
|
|
|
|
|
|
|
match info.format() {
|
|
|
|
gst_video::VideoFormat::Uyvy
|
|
|
|
| gst_video::VideoFormat::Bgra
|
|
|
|
| gst_video::VideoFormat::Bgrx
|
|
|
|
| gst_video::VideoFormat::Rgba
|
|
|
|
| gst_video::VideoFormat::Rgbx => {
|
|
|
|
let line_bytes = if info.format() == gst_video::VideoFormat::Uyvy {
|
|
|
|
2 * vframe.width() as usize
|
|
|
|
} else {
|
|
|
|
4 * vframe.width() as usize
|
|
|
|
};
|
|
|
|
let dest_stride = vframe.plane_stride()[0] as usize;
|
|
|
|
let dest = vframe.plane_data_mut(0).unwrap();
|
2020-01-02 12:00:47 +00:00
|
|
|
let src_stride = video_frame.line_stride_or_data_size_in_bytes() as usize;
|
2019-07-17 16:10:20 +00:00
|
|
|
let src = video_frame.data();
|
|
|
|
|
|
|
|
for (dest, src) in dest
|
|
|
|
.chunks_exact_mut(dest_stride)
|
|
|
|
.zip(src.chunks_exact(src_stride))
|
|
|
|
{
|
|
|
|
dest.copy_from_slice(src);
|
|
|
|
dest.copy_from_slice(&src[..line_bytes]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
gst_video::VideoFormat::Nv12 => {
|
|
|
|
// First plane
|
|
|
|
{
|
|
|
|
let line_bytes = vframe.width() as usize;
|
|
|
|
let dest_stride = vframe.plane_stride()[0] as usize;
|
|
|
|
let dest = vframe.plane_data_mut(0).unwrap();
|
2020-01-02 12:00:47 +00:00
|
|
|
let src_stride = video_frame.line_stride_or_data_size_in_bytes() as usize;
|
2019-07-17 16:10:20 +00:00
|
|
|
let src = video_frame.data();
|
|
|
|
|
|
|
|
for (dest, src) in dest
|
|
|
|
.chunks_exact_mut(dest_stride)
|
|
|
|
.zip(src.chunks_exact(src_stride))
|
|
|
|
{
|
|
|
|
dest.copy_from_slice(&src[..line_bytes]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Second plane
|
|
|
|
{
|
|
|
|
let line_bytes = vframe.width() as usize;
|
|
|
|
let dest_stride = vframe.plane_stride()[1] as usize;
|
|
|
|
let dest = vframe.plane_data_mut(1).unwrap();
|
2020-01-02 12:00:47 +00:00
|
|
|
let src_stride = video_frame.line_stride_or_data_size_in_bytes() as usize;
|
2019-07-17 16:10:20 +00:00
|
|
|
let src = &video_frame.data()[(video_frame.yres() as usize * src_stride)..];
|
|
|
|
|
|
|
|
for (dest, src) in dest
|
|
|
|
.chunks_exact_mut(dest_stride)
|
|
|
|
.zip(src.chunks_exact(src_stride))
|
|
|
|
{
|
|
|
|
dest.copy_from_slice(&src[..line_bytes]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
gst_video::VideoFormat::Yv12 | gst_video::VideoFormat::I420 => {
|
|
|
|
// First plane
|
|
|
|
{
|
|
|
|
let line_bytes = vframe.width() as usize;
|
|
|
|
let dest_stride = vframe.plane_stride()[0] as usize;
|
|
|
|
let dest = vframe.plane_data_mut(0).unwrap();
|
2020-01-02 12:00:47 +00:00
|
|
|
let src_stride = video_frame.line_stride_or_data_size_in_bytes() as usize;
|
2019-07-17 16:10:20 +00:00
|
|
|
let src = video_frame.data();
|
|
|
|
|
|
|
|
for (dest, src) in dest
|
|
|
|
.chunks_exact_mut(dest_stride)
|
|
|
|
.zip(src.chunks_exact(src_stride))
|
|
|
|
{
|
|
|
|
dest.copy_from_slice(&src[..line_bytes]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Second plane
|
|
|
|
{
|
|
|
|
let line_bytes = (vframe.width() as usize + 1) / 2;
|
|
|
|
let dest_stride = vframe.plane_stride()[1] as usize;
|
|
|
|
let dest = vframe.plane_data_mut(1).unwrap();
|
2020-01-02 12:00:47 +00:00
|
|
|
let src_stride = video_frame.line_stride_or_data_size_in_bytes() as usize;
|
|
|
|
let src_stride1 = video_frame.line_stride_or_data_size_in_bytes() as usize / 2;
|
2019-07-17 16:10:20 +00:00
|
|
|
let src = &video_frame.data()[(video_frame.yres() as usize * src_stride)..];
|
|
|
|
|
|
|
|
for (dest, src) in dest
|
|
|
|
.chunks_exact_mut(dest_stride)
|
|
|
|
.zip(src.chunks_exact(src_stride1))
|
|
|
|
{
|
|
|
|
dest.copy_from_slice(&src[..line_bytes]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Third plane
|
|
|
|
{
|
|
|
|
let line_bytes = (vframe.width() as usize + 1) / 2;
|
|
|
|
let dest_stride = vframe.plane_stride()[2] as usize;
|
|
|
|
let dest = vframe.plane_data_mut(2).unwrap();
|
2020-01-02 12:00:47 +00:00
|
|
|
let src_stride = video_frame.line_stride_or_data_size_in_bytes() as usize;
|
|
|
|
let src_stride1 = video_frame.line_stride_or_data_size_in_bytes() as usize / 2;
|
2019-07-17 16:10:20 +00:00
|
|
|
let src = &video_frame.data()[(video_frame.yres() as usize * src_stride
|
|
|
|
+ (video_frame.yres() as usize + 1) / 2 * src_stride1)..];
|
|
|
|
|
|
|
|
for (dest, src) in dest
|
|
|
|
.chunks_exact_mut(dest_stride)
|
|
|
|
.zip(src.chunks_exact(src_stride1))
|
|
|
|
{
|
|
|
|
dest.copy_from_slice(&src[..line_bytes]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
_ => unreachable!(),
|
|
|
|
}
|
|
|
|
|
2021-02-23 10:46:27 +00:00
|
|
|
vframe.into_buffer()
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
2019-07-19 08:32:04 +00:00
|
|
|
}
|
2019-07-17 16:10:20 +00:00
|
|
|
|
2019-07-19 08:32:04 +00:00
|
|
|
impl Receiver<AudioReceiver> {
|
2019-07-17 16:10:20 +00:00
|
|
|
fn capture_audio(
|
|
|
|
&self,
|
|
|
|
element: &gst_base::BaseSrc,
|
|
|
|
recv: &RecvInstance,
|
|
|
|
) -> Result<(gst::Buffer, gst_audio::AudioInfo), gst::FlowError> {
|
2020-01-16 09:14:10 +00:00
|
|
|
let timer = time::Instant::now();
|
|
|
|
let timeout = if self.0.first_frame.load(Ordering::SeqCst) {
|
|
|
|
self.0.connect_timeout
|
|
|
|
} else {
|
|
|
|
self.0.timeout
|
|
|
|
};
|
2019-07-17 16:10:20 +00:00
|
|
|
let mut flushing;
|
|
|
|
let mut playing;
|
|
|
|
|
|
|
|
let audio_frame = loop {
|
|
|
|
{
|
|
|
|
let queue = (self.0.queue.0).0.lock().unwrap();
|
|
|
|
flushing = queue.flushing;
|
|
|
|
playing = queue.playing;
|
|
|
|
if !queue.capturing {
|
|
|
|
gst_debug!(self.0.cat, obj: element, "Shutting down");
|
|
|
|
return Err(gst::FlowError::Flushing);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
let res = match recv.capture(false, true, false, 50) {
|
|
|
|
Err(_) => Err(()),
|
|
|
|
Ok(None) => Ok(None),
|
|
|
|
Ok(Some(Frame::Audio(frame))) => Ok(Some(frame)),
|
|
|
|
_ => unreachable!(),
|
|
|
|
};
|
|
|
|
|
|
|
|
let audio_frame = match res {
|
|
|
|
Err(_) => {
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::element_error!(
|
2019-07-19 09:51:06 +00:00
|
|
|
element,
|
|
|
|
gst::ResourceError::Read,
|
|
|
|
["Error receiving frame"]
|
|
|
|
);
|
2019-07-17 16:10:20 +00:00
|
|
|
return Err(gst::FlowError::Error);
|
|
|
|
}
|
2020-09-01 18:38:13 +00:00
|
|
|
Ok(None) if timeout > 0 && timer.elapsed().as_millis() >= timeout as u128 => {
|
2019-07-19 09:51:06 +00:00
|
|
|
gst_debug!(self.0.cat, obj: element, "Timed out -- assuming EOS",);
|
2019-07-17 16:10:20 +00:00
|
|
|
return Err(gst::FlowError::Eos);
|
|
|
|
}
|
|
|
|
Ok(None) => {
|
|
|
|
gst_debug!(
|
|
|
|
self.0.cat,
|
|
|
|
obj: element,
|
|
|
|
"No audio frame received yet, retry"
|
|
|
|
);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
Ok(Some(frame)) => frame,
|
|
|
|
};
|
|
|
|
|
|
|
|
break audio_frame;
|
|
|
|
};
|
|
|
|
|
2020-01-16 09:14:10 +00:00
|
|
|
self.0.first_frame.store(false, Ordering::SeqCst);
|
|
|
|
|
2019-07-19 09:51:06 +00:00
|
|
|
gst_debug!(
|
|
|
|
self.0.cat,
|
|
|
|
obj: element,
|
|
|
|
"Received audio frame {:?}",
|
|
|
|
audio_frame,
|
|
|
|
);
|
|
|
|
|
2019-07-19 10:00:27 +00:00
|
|
|
let (pts, duration) = self
|
|
|
|
.calculate_audio_timestamp(element, &audio_frame)
|
|
|
|
.ok_or_else(|| {
|
|
|
|
gst_debug!(self.0.cat, obj: element, "Flushing, dropping buffer");
|
|
|
|
gst::FlowError::CustomError
|
|
|
|
})?;
|
2019-07-18 21:05:31 +00:00
|
|
|
|
2019-07-17 16:10:20 +00:00
|
|
|
// Simply read all video frames while flushing but don't copy them or anything to
|
|
|
|
// make sure that we're not accumulating anything here
|
|
|
|
if !playing || flushing {
|
|
|
|
gst_debug!(self.0.cat, obj: element, "Flushing, dropping buffer");
|
|
|
|
return Err(gst::FlowError::CustomError);
|
|
|
|
}
|
|
|
|
|
|
|
|
let info = self.create_audio_info(element, &audio_frame)?;
|
|
|
|
|
2021-02-23 10:46:27 +00:00
|
|
|
let buffer = self.create_audio_buffer(element, pts, duration, &info, &audio_frame);
|
2019-07-17 16:10:20 +00:00
|
|
|
|
|
|
|
gst_log!(self.0.cat, obj: element, "Produced buffer {:?}", buffer);
|
|
|
|
|
|
|
|
Ok((buffer, info))
|
|
|
|
}
|
|
|
|
|
|
|
|
fn calculate_audio_timestamp(
|
|
|
|
&self,
|
|
|
|
element: &gst_base::BaseSrc,
|
|
|
|
audio_frame: &AudioFrame,
|
2021-09-13 09:26:56 +00:00
|
|
|
) -> Option<(gst::ClockTime, Option<gst::ClockTime>)> {
|
|
|
|
let duration = gst::ClockTime::SECOND.mul_div_floor(
|
|
|
|
audio_frame.no_samples() as u64,
|
|
|
|
audio_frame.sample_rate() as u64,
|
|
|
|
);
|
2019-07-18 21:05:31 +00:00
|
|
|
|
2019-07-19 09:51:06 +00:00
|
|
|
self.calculate_timestamp(
|
|
|
|
element,
|
|
|
|
audio_frame.timestamp(),
|
|
|
|
audio_frame.timecode(),
|
2019-07-18 21:05:31 +00:00
|
|
|
duration,
|
2019-07-19 09:51:06 +00:00
|
|
|
)
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fn create_audio_info(
|
|
|
|
&self,
|
2019-07-19 09:51:06 +00:00
|
|
|
element: &gst_base::BaseSrc,
|
2019-07-17 16:10:20 +00:00
|
|
|
audio_frame: &AudioFrame,
|
|
|
|
) -> Result<gst_audio::AudioInfo, gst::FlowError> {
|
2020-07-27 13:08:56 +00:00
|
|
|
let builder = gst_audio::AudioInfo::builder(
|
2019-07-17 16:10:20 +00:00
|
|
|
gst_audio::AUDIO_FORMAT_S16,
|
|
|
|
audio_frame.sample_rate() as u32,
|
|
|
|
audio_frame.no_channels() as u32,
|
|
|
|
);
|
|
|
|
|
2019-12-18 17:34:00 +00:00
|
|
|
builder.build().map_err(|_| {
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::element_error!(
|
2019-07-19 09:51:06 +00:00
|
|
|
element,
|
|
|
|
gst::StreamError::Format,
|
|
|
|
["Invalid audio format configuration"]
|
|
|
|
);
|
|
|
|
|
|
|
|
gst::FlowError::NotNegotiated
|
|
|
|
})
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fn create_audio_buffer(
|
|
|
|
&self,
|
|
|
|
_element: &gst_base::BaseSrc,
|
|
|
|
pts: gst::ClockTime,
|
2021-09-13 09:26:56 +00:00
|
|
|
duration: Option<gst::ClockTime>,
|
2019-07-17 16:10:20 +00:00
|
|
|
info: &gst_audio::AudioInfo,
|
|
|
|
audio_frame: &AudioFrame,
|
2021-02-23 10:46:27 +00:00
|
|
|
) -> gst::Buffer {
|
2019-07-17 16:10:20 +00:00
|
|
|
// We multiply by 2 because is the size in bytes of an i16 variable
|
|
|
|
let buff_size = (audio_frame.no_samples() as u32 * info.bpf()) as usize;
|
|
|
|
let mut buffer = gst::Buffer::with_size(buff_size).unwrap();
|
|
|
|
{
|
|
|
|
let buffer = buffer.get_mut().unwrap();
|
|
|
|
|
|
|
|
buffer.set_pts(pts);
|
|
|
|
buffer.set_duration(duration);
|
|
|
|
|
|
|
|
#[cfg(feature = "reference-timestamps")]
|
|
|
|
{
|
|
|
|
gst::ReferenceTimestampMeta::add(
|
|
|
|
buffer,
|
|
|
|
&*TIMECODE_CAPS,
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::ClockTime::from_nseconds(audio_frame.timecode() as u64 * 100),
|
|
|
|
gst::ClockTime::NONE,
|
2019-07-17 16:10:20 +00:00
|
|
|
);
|
|
|
|
if audio_frame.timestamp() != ndisys::NDIlib_recv_timestamp_undefined {
|
|
|
|
gst::ReferenceTimestampMeta::add(
|
|
|
|
buffer,
|
|
|
|
&*TIMESTAMP_CAPS,
|
2021-09-13 09:26:56 +00:00
|
|
|
gst::ClockTime::from_nseconds(audio_frame.timestamp() as u64 * 100),
|
|
|
|
gst::ClockTime::NONE,
|
2019-07-17 16:10:20 +00:00
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
audio_frame.copy_to_interleaved_16s(
|
|
|
|
buffer
|
|
|
|
.map_writable()
|
|
|
|
.unwrap()
|
|
|
|
.as_mut_slice_of::<i16>()
|
|
|
|
.unwrap(),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2021-02-23 10:46:27 +00:00
|
|
|
buffer
|
2019-07-17 16:10:20 +00:00
|
|
|
}
|
|
|
|
}
|