diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ea36df3..834f59c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,7 @@ stages: libwayland-egl1-mesa llvm nasm + libcsound64-dev python3-pip python3-setuptools python3-wheel @@ -52,7 +53,6 @@ stages: # FIXME: The feature name should explicitly mention the dav1d plugin but # Cargo currently doesn't support passthrough for that scenario. - export RUSTFLAGS='--cfg feature="build"' - - cd "${CI_PROJECT_DIR}" cache: key: "gst" diff --git a/Cargo.toml b/Cargo.toml index a1539545..55ad99de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "gst-plugin-lewton", "gst-plugin-claxon", "gst-plugin-gif", + "gst-plugin-csound", ] [profile.release] diff --git a/README.md b/README.md index ef2cc2af..c886d9d8 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,10 @@ gst-plugin-togglerecord is licensed under the Lesser General Public License ([LICENSE-LGPLv2](LICENSE-LGPLv2)) version 2.1 or (at your option) any later version. +gst-plugin-csound is licensed under the Lesser General Public License +([LICENSE-LGPLv2](LICENSE-LGPLv2)) version 2.1 or (at your option) any later +version. + GStreamer itself is licensed under the Lesser General Public License version 2.1 or (at your option) any later version: https://www.gnu.org/licenses/lgpl-2.1.html diff --git a/gst-plugin-csound/Cargo.toml b/gst-plugin-csound/Cargo.toml new file mode 100644 index 00000000..ed4419dc --- /dev/null +++ b/gst-plugin-csound/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "gst-plugin-csound" +version = "0.1.0" +authors = ["Natanael Mojica "] +repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" +license = "LGPL-2.1+" +edition = "2018" +description = "An Audio filter plugin based on Csound" + +[dependencies] +glib = { git = "https://github.com/gtk-rs/glib" } +gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst_base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst_audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst_check = { package = "gstreamer-check", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +csound = { git = "https://github.com/neithanmo/csound-rs"} +once_cell = "1.0" +byte-slice-cast = "0.3" + +[lib] +name = "gstcsound" +crate-type = ["cdylib", "rlib"] +path = "src/lib.rs" + +[[example]] +name = "csound-effect" +path = "examples/effect_example.rs" + +[build-dependencies] +gst-plugin-version-helper = { git = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" } diff --git a/gst-plugin-csound/README.md b/gst-plugin-csound/README.md new file mode 100644 index 00000000..1b856362 --- /dev/null +++ b/gst-plugin-csound/README.md @@ -0,0 +1,21 @@ +# gst-plugin-csound + +This is a [GStreamer](https://gstreamer.freedesktop.org/) plugin to interact +with the [Csound](https://csound.com/) sound computing system. + +Currently, there is only a filter element, called, csoundfilter. Two more elements a source and sink would be implemented +later on. + +For more information about dependencies and installation process, please refer to the [csound-rs](https://crates.io/crates/csound) +documentation + +## simple example +The included example constructs the follow pipeline +``` +$ gst-launch-1.0 \ + audiotestsrc ! \ + audioconvert ! \ + csoundfilter location=effect.csd ! \ + audioconvert ! \ + autoaudiosink +``` diff --git a/gst-plugin-csound/build.rs b/gst-plugin-csound/build.rs new file mode 100644 index 00000000..0d1ddb61 --- /dev/null +++ b/gst-plugin-csound/build.rs @@ -0,0 +1,5 @@ +extern crate gst_plugin_version_helper; + +fn main() { + gst_plugin_version_helper::get_info() +} diff --git a/gst-plugin-csound/examples/effect_example.rs b/gst-plugin-csound/examples/effect_example.rs new file mode 100644 index 00000000..b330aa25 --- /dev/null +++ b/gst-plugin-csound/examples/effect_example.rs @@ -0,0 +1,144 @@ +// Copyright (C) 2020 Natanael Mojica +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this library; if not, write to the +// Free Software Foundation, Inc., 51 Franklin Street, Suite 500, +// Boston, MA 02110-1335, USA. + +use glib; +use glib::prelude::*; + +use gst; +use gst::prelude::*; + +use gstcsound; + +use std::error::Error; + +const AUDIO_SRC: &str = "audiotestsrc"; +const AUDIO_SINK: &str = "audioconvert ! autoaudiosink"; + +// This example defines two instruments, the first instrument send to the output that is at its input and accumulates the received audio samples +// into a global variable called gasig. The execution of this instrument last the first 2 seconds. +// The second instrument starts it execution at 1.8 second, This instrument creates two audio buffers with samples that are read +// from the global accumulator(gasig), then reads these buffers at a fixed delay time, creating the adelL, adelM and adelR buffers, +// also, It multiplies the audio samples in the right channel by 0.5 * kdel, being kdel a line of values starting at 0.5 at increments of 0.001. +// Finally, those buffers are mixed with the accumulator, and an audio envelop is applied(aseg) to them. +// The result is similar to an audio echo in which the buffered samples are read at different delay times and also modified in frecuency(right channel), +// this creates an space effect using just one channel audio input. +const CSD: &str = " + + + + + + sr = 44100 + ksmps = 7 + + nchnls_i = 1 + nchnls = 2 + + gasig init 0 + gidel = 1 + + instr 1 + + ain in + outs ain, ain + + vincr gasig, ain + endin + + instr 2 + + ifeedback = p4 + + aseg linseg 1., p3, 0.0 + + abuf2 delayr gidel + adelL deltap .4 + adelM deltap .5 + delayw gasig + (adelL * ifeedback) + + abuf3 delayr gidel + kdel line .5, p3, .001 + adelR deltap .5 * kdel + delayw gasig + (adelR * ifeedback) + outs (adelL + adelM) * aseg, (adelR + adelM) * aseg + clear gasig + endin + + + + + i 1 0 2 + i 2 1.8 5 .8 + e + + "; + +fn create_pipeline() -> Result> { + let pipeline = gst::Pipeline::new(None); + + let audio_src = gst::parse_bin_from_description(AUDIO_SRC, true)?.upcast(); + + let audio_sink = gst::parse_bin_from_description(AUDIO_SINK, true)?.upcast(); + + let csoundfilter = gst::ElementFactory::make("csoundfilter", None)?; + csoundfilter.set_property("csd-text", &CSD)?; + + pipeline.add_many(&[&audio_src, &csoundfilter, &audio_sink])?; + + audio_src.link_pads(Some("src"), &csoundfilter, Some("sink"))?; + csoundfilter.link_pads(Some("src"), &audio_sink, Some("sink"))?; + + Ok(pipeline) +} + +fn main_loop(pipeline: gst::Pipeline) -> Result<(), Box> { + pipeline.set_state(gst::State::Playing)?; + + let bus = pipeline + .get_bus() + .expect("Pipeline without bus. Shouldn't happen!"); + + for msg in bus.iter_timed(gst::CLOCK_TIME_NONE) { + use gst::MessageView; + + match msg.view() { + MessageView::Eos(..) => break, + MessageView::Error(err) => { + println!( + "Error from {:?}: {} ({:?})", + msg.get_src().map(|s| s.get_path_string()), + err.get_error(), + err.get_debug() + ); + break; + } + _ => (), + } + } + + pipeline.set_state(gst::State::Null)?; + + Ok(()) +} + +fn main() -> Result<(), Box> { + gst::init().unwrap(); + + gstcsound::plugin_register_static().expect("Failed to register csound plugin"); + + create_pipeline().and_then(main_loop) +} diff --git a/gst-plugin-csound/src/filter.rs b/gst-plugin-csound/src/filter.rs new file mode 100644 index 00000000..87b64519 --- /dev/null +++ b/gst-plugin-csound/src/filter.rs @@ -0,0 +1,697 @@ +// Copyright (C) 2020 Natanael Mojica +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this library; if not, write to the +// Free Software Foundation, Inc., 51 Franklin Street, Suite 500, +// Boston, MA 02110-1335, USA. + +use glib; +use glib::subclass; +use glib::subclass::prelude::*; +use glib::{glib_object_impl, glib_object_subclass}; +use gst; +use gst::prelude::*; +use gst::subclass::prelude::*; +use gst::{ + gst_debug, gst_element_error, gst_error, gst_error_msg, gst_info, gst_log, gst_loggable_error, + gst_warning, +}; +use gst_audio; +use gst_base; +use gst_base::subclass::base_transform::BaseTransformImplExt; +use gst_base::subclass::base_transform::GeneratedOutput; +use gst_base::subclass::prelude::*; + +use std::sync::atomic::{AtomicBool, Ordering}; + +use std::sync::Mutex; +use std::{f64, i32}; + +use byte_slice_cast::*; + +use csound::{Csound, MessageType}; + +use once_cell::sync::Lazy; + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "csoundfilter", + gst::DebugColorFlags::empty(), + Some("Audio Filter based on Csound"), + ) +}); + +const SCORE_OFFSET_DEFAULT: f64 = 0f64; +const DEFAULT_LOOP: bool = false; + +#[derive(Debug, Clone)] +struct Settings { + pub loop_: bool, + pub location: Option, + pub csd_text: Option, + pub offset: f64, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + loop_: DEFAULT_LOOP, + location: None, + csd_text: None, + offset: SCORE_OFFSET_DEFAULT, + } + } +} + +struct State { + in_info: gst_audio::AudioInfo, + out_info: gst_audio::AudioInfo, + adapter: gst_base::UniqueAdapter, + ksmps: u32, +} + +struct CsoundFilter { + settings: Mutex, + state: Mutex>, + csound: Mutex, + compiled: AtomicBool, +} + +static PROPERTIES: [subclass::Property; 4] = [ + subclass::Property("loop", |name| { + glib::ParamSpec::boolean( + name, + "Loop", + "loop over the score (can be changed in PLAYING or PAUSED state)", + DEFAULT_LOOP, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("location", |name| { + glib::ParamSpec::string( + name, + "Location", + "Location of the csd file to be used by csound. + Use either location or CSD-text but not both at the same time, if so and error would be triggered", + None, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("csd-text", |name| { + glib::ParamSpec::string( + name, + "CSD-text", + "The content of a csd file passed as a String. + Use either location or csd-text but not both at the same time, if so and error would be triggered", + None, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("score_offset", |name| { + glib::ParamSpec::double( + name, + "Score Offset", + "Score offset in seconds to start the performance", + 0.0, + f64::MAX, + SCORE_OFFSET_DEFAULT, + glib::ParamFlags::READWRITE, + ) + }), +]; + +impl State { + // Considering an input of size: input_size and the user's ksmps, + // calculates the equivalent output_size + fn max_output_size(&self, input_size: usize) -> usize { + let in_samples = input_size / self.in_info.bpf() as usize; + let in_process_samples = in_samples - (in_samples % self.ksmps as usize); + in_process_samples * self.out_info.bpf() as usize + } + + fn get_bytes_to_read(&mut self, output_size: usize) -> usize { + // The max amount of bytes at the input that We would need + // for filling an output buffer of size *output_size* + (output_size / self.out_info.bpf() as usize) * self.in_info.bpf() as usize + } + + // returns the spin capacity in bytes + fn spin_capacity(&self) -> usize { + (self.ksmps * self.in_info.bpf()) as _ + } + + fn needs_more_data(&self) -> bool { + self.adapter.available() < self.spin_capacity() + } + + fn samples_to_time(&self, samples: u64) -> gst::ClockTime { + gst::ClockTime(samples.mul_div_round(gst::SECOND_VAL, self.in_info.rate() as u64)) + } + + fn get_current_pts(&self) -> gst::ClockTime { + // get the last seen pts and the amount of bytes + // since then + let (prev_pts, distance) = self.adapter.prev_pts(); + + // Use the distance to get the amount of samples + // and with it calculate the time-offset which + // can be added to the prev_pts to get the + // pts at the beginning of the adapter. + let samples = distance / self.in_info.bpf() as u64; + prev_pts + self.samples_to_time(samples) + } + + fn buffer_duration(&self, buffer_size: u64) -> gst::ClockTime { + let samples = buffer_size / self.out_info.bpf() as u64; + self.samples_to_time(samples) + } +} + +impl CsoundFilter { + fn process(&self, csound: &mut Csound, idata: &[f64], odata: &mut [f64]) -> bool { + let spin = csound.get_spin().unwrap(); + let spout = csound.get_spout().unwrap(); + + let in_chunks = idata.chunks_exact(spin.len()); + let out_chuncks = odata.chunks_exact_mut(spout.len()); + let mut end_score = false; + for (ichunk, ochunk) in in_chunks.zip(out_chuncks) { + spin.copy_from_slice(ichunk); + end_score = csound.perform_ksmps(); + spout.copy_to_slice(ochunk); + } + + end_score + } + + fn compile_score(&self) -> std::result::Result<(), gst::ErrorMessage> { + let csound = self.csound.lock().unwrap(); + let settings = self.settings.lock().unwrap(); + if let Some(ref location) = settings.location { + csound + .compile_csd(location) + .map_err(|e| gst_error_msg!(gst::LibraryError::Failed, [e]))?; + } else if let Some(ref text) = settings.csd_text { + csound + .compile_csd_text(text) + .map_err(|e| gst_error_msg!(gst::LibraryError::Failed, [e]))?; + } else { + return Err(gst_error_msg!( + gst::LibraryError::Failed, + ["No Csound score specified to compile. Use either location or csd-text but not both"] + )); + } + + self.compiled.store(true, Ordering::SeqCst); + Ok(()) + } + + fn message_callback(msg_type: MessageType, msg: &str) { + match msg_type { + MessageType::CSOUNDMSG_ERROR => gst_error!(CAT, "{}", msg), + MessageType::CSOUNDMSG_WARNING => gst_warning!(CAT, "{}", msg), + MessageType::CSOUNDMSG_ORCH => gst_info!(CAT, "{}", msg), + MessageType::CSOUNDMSG_REALTIME => gst_log!(CAT, "{}", msg), + MessageType::CSOUNDMSG_DEFAULT => gst_log!(CAT, "{}", msg), + MessageType::CSOUNDMSG_STDOUT => gst_log!(CAT, "{}", msg), + } + } + + fn drain(&self, element: &gst_base::BaseTransform) -> Result { + let csound = self.csound.lock().unwrap(); + let mut state_lock = self.state.lock().unwrap(); + let state = state_lock.as_mut().unwrap(); + + let avail = state.adapter.available(); + + // Complete processing blocks should have been processed in the transform call + assert!(avail < state.spin_capacity()); + + if avail == 0 { + return Ok(gst::FlowSuccess::Ok); + } + + let mut spin = csound.get_spin().unwrap(); + let spout = csound.get_spout().unwrap(); + + let out_bytes = + (avail / state.in_info.channels() as usize) * state.out_info.channels() as usize; + + let mut buffer = gst::Buffer::with_size(out_bytes).map_err(|e| { + gst_error!( + CAT, + obj: element, + "Failed to allocate buffer at EOS {:?}", + e + ); + gst::FlowError::Flushing + })?; + + let buffer_mut = buffer.get_mut().ok_or(gst::FlowError::NotSupported)?; + + let pts = state.get_current_pts(); + let duration = state.buffer_duration(out_bytes as _); + + buffer_mut.set_pts(pts); + buffer_mut.set_duration(duration); + + let srcpad = element.get_static_pad("src").unwrap(); + + let adapter_map = state.adapter.map(avail).unwrap(); + let data = adapter_map + .as_ref() + .as_slice_of::() + .map_err(|_| gst::FlowError::NotSupported)?; + + let mut omap = buffer_mut + .map_writable() + .map_err(|_| gst::FlowError::NotSupported)?; + let odata = omap + .as_mut_slice_of::() + .map_err(|_| gst::FlowError::NotSupported)?; + + spin.clear(); + spin.copy_from_slice(data); + csound.perform_ksmps(); + spout.copy_to_slice(odata); + + drop(adapter_map); + drop(omap); + + state.adapter.flush(avail); + // Drop the locks before pushing buffers into the srcpad + drop(state_lock); + drop(csound); + + srcpad.push(buffer) + } + + fn generate_output( + &self, + element: &gst_base::BaseTransform, + state: &mut State, + ) -> Result { + let output_size = state.max_output_size(state.adapter.available()); + + let mut output = gst::Buffer::with_size(output_size).map_err(|_| gst::FlowError::Error)?; + let outbuf = output.get_mut().ok_or(gst::FlowError::Error)?; + + let pts = state.get_current_pts(); + let duration = state.buffer_duration(output_size as _); + + outbuf.set_pts(pts); + outbuf.set_duration(duration); + + gst_log!( + CAT, + obj: element, + "Generating output at: {} - duration: {}", + pts, + duration + ); + + // Get the required amount of bytes to be read from + // the adapter to fill an ouput buffer of size output_size + let bytes_to_read = state.get_bytes_to_read(output_size); + + let indata = state + .adapter + .map(bytes_to_read) + .map_err(|_| gst::FlowError::Error)?; + let idata = indata + .as_ref() + .as_slice_of::() + .map_err(|_| gst::FlowError::Error)?; + + let mut omap = outbuf.map_writable().map_err(|_| gst::FlowError::Error)?; + let odata = omap + .as_mut_slice_of::() + .map_err(|_| gst::FlowError::Error)?; + + let mut csound = self.csound.lock().unwrap(); + let end_score = self.process(&mut csound, idata, odata); + + drop(indata); + drop(omap); + state.adapter.flush(bytes_to_read); + + if end_score { + let settings = self.settings.lock().unwrap(); + if settings.loop_ { + csound.set_score_offset_seconds(settings.offset); + csound.rewind_score(); + } else { + // clear the adapter here because our eos event handler + // will try to flush it calling csound.perform() + // which does not make sense since + // the end of score has been reached. + state.adapter.clear(); + return Err(gst::FlowError::Eos); + } + } + + Ok(GeneratedOutput::Buffer(output)) + } +} + +impl ObjectSubclass for CsoundFilter { + const NAME: &'static str = "CsoundFilter"; + type ParentType = gst_base::BaseTransform; + type Instance = gst::subclass::ElementInstanceStruct; + type Class = subclass::simple::ClassStruct; + + glib_object_subclass!(); + + fn new() -> Self { + let csound = Csound::new(); + // create the csound instance and configure + csound.message_string_callback(Self::message_callback); + // Disable all default handling of sound I/O by csound internal library + // by giving to it a hardware buffer size of zero, and setting a state, + // higher than zero. + csound.set_host_implemented_audioIO(1, 0); + // We don't want csound to write samples to our HW + csound.set_option("--nosound").unwrap(); + Self { + settings: Mutex::new(Default::default()), + state: Mutex::new(None), + csound: Mutex::new(csound), + compiled: AtomicBool::new(false), + } + } + + fn class_init(klass: &mut subclass::simple::ClassStruct) { + klass.set_metadata( + "Audio filter", + "Filter/Effect/Audio", + "Implement an audio filter/effects using Csound", + "Natanael Mojica ", + ); + + let caps = gst::Caps::new_simple( + "audio/x-raw", + &[ + ("format", &gst_audio::AUDIO_FORMAT_F64.to_str()), + ("rate", &gst::IntRange::::new(1, i32::MAX)), + ("channels", &gst::IntRange::::new(1, i32::MAX)), + ("layout", &"interleaved"), + ], + ); + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + klass.add_pad_template(src_pad_template); + + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + klass.add_pad_template(sink_pad_template); + + klass.install_properties(&PROPERTIES); + + klass.configure( + gst_base::subclass::BaseTransformMode::NeverInPlace, + false, + false, + ); + } +} + +impl ObjectImpl for CsoundFilter { + glib_object_impl!(); + + fn set_property(&self, _obj: &glib::Object, id: usize, value: &glib::Value) { + let prop = &PROPERTIES[id]; + match *prop { + subclass::Property("loop", ..) => { + let mut settings = self.settings.lock().unwrap(); + settings.loop_ = value.get_some().expect("type checked upstream"); + } + subclass::Property("location", ..) => { + let mut settings = self.settings.lock().unwrap(); + if self.state.lock().unwrap().is_none() { + settings.location = match value.get::() { + Ok(location) => location, + _ => unreachable!("type checked upstream"), + }; + } + } + subclass::Property("csd-text", ..) => { + let mut settings = self.settings.lock().unwrap(); + if self.state.lock().unwrap().is_none() { + settings.csd_text = match value.get::() { + Ok(text) => text, + _ => unreachable!("type checked upstream"), + }; + } + } + subclass::Property("score_offset", ..) => { + let mut settings = self.settings.lock().unwrap(); + settings.offset = value.get_some().expect("type checked upstream"); + } + _ => unimplemented!(), + } + } + + fn get_property(&self, _obj: &glib::Object, id: usize) -> Result { + let prop = &PROPERTIES[id]; + + match *prop { + subclass::Property("loop", ..) => { + let settings = self.settings.lock().unwrap(); + Ok(settings.loop_.to_value()) + } + subclass::Property("location", ..) => { + let settings = self.settings.lock().unwrap(); + Ok(settings.location.to_value()) + } + subclass::Property("csd-text", ..) => { + let settings = self.settings.lock().unwrap(); + Ok(settings.csd_text.to_value()) + } + subclass::Property("score_offset", ..) => { + let settings = self.settings.lock().unwrap(); + Ok(settings.offset.to_value()) + } + _ => unimplemented!(), + } + } +} + +impl ElementImpl for CsoundFilter {} + +impl BaseTransformImpl for CsoundFilter { + fn start( + &self, + _element: &gst_base::BaseTransform, + ) -> std::result::Result<(), gst::ErrorMessage> { + self.compile_score()?; + + let csound = self.csound.lock().unwrap(); + let settings = self.settings.lock().unwrap(); + csound.set_score_offset_seconds(settings.offset); + + if let Err(e) = csound.start() { + return Err(gst_error_msg!(gst::LibraryError::Failed, [e])); + } + + Ok(()) + } + + fn stop(&self, element: &gst_base::BaseTransform) -> Result<(), gst::ErrorMessage> { + let csound = self.csound.lock().unwrap(); + csound.stop(); + csound.reset(); + let _ = self.state.lock().unwrap().take(); + + gst_info!(CAT, obj: element, "Stopped"); + + Ok(()) + } + + fn sink_event(&self, element: &gst_base::BaseTransform, event: gst::Event) -> bool { + use gst::EventView; + + if let EventView::Eos(_) = event.view() { + gst_log!(CAT, obj: element, "Handling Eos"); + if self.drain(element).is_err() { + return false; + } + } + self.parent_sink_event(element, event) + } + + fn transform_caps( + &self, + element: &gst_base::BaseTransform, + direction: gst::PadDirection, + caps: &gst::Caps, + filter: Option<&gst::Caps>, + ) -> Option { + let compiled = self.compiled.load(Ordering::SeqCst); + + let mut other_caps = { + // Our caps proposal + let mut new_caps = caps.clone(); + if compiled { + let csound = self.csound.lock().unwrap(); + // Use the sample rate and channels configured in the csound score + let sr = csound.get_sample_rate() as i32; + let ichannels = csound.input_channels() as i32; + let ochannels = csound.output_channels() as i32; + for s in new_caps.make_mut().iter_mut() { + s.set("format", &gst_audio::AUDIO_FORMAT_F64.to_str()); + s.set("rate", &sr); + + // replace the channel property with our values, + // if they are not supported, the negotiation will fail. + if direction == gst::PadDirection::Src { + s.set("channels", &ichannels); + } else { + s.set("channels", &ochannels); + } + // Csound does not have a concept of channel-mask + s.remove_field("channel-mask"); + } + } + new_caps + }; + + gst_debug!( + CAT, + obj: element, + "Transformed caps from {} to {} in direction {:?}", + caps, + other_caps, + direction + ); + + if let Some(filter) = filter { + other_caps = filter.intersect_with_mode(&other_caps, gst::CapsIntersectMode::First); + } + + Some(other_caps) + } + + fn set_caps( + &self, + element: &gst_base::BaseTransform, + incaps: &gst::Caps, + outcaps: &gst::Caps, + ) -> Result<(), gst::LoggableError> { + // Flush previous state + if self.state.lock().unwrap().is_some() { + self.drain(element).or_else(|e| { + Err(gst_loggable_error!( + CAT, + "Error flusing previous state data {:?}", + e + )) + })?; + } + + let in_info = gst_audio::AudioInfo::from_caps(incaps) + .or_else(|_| Err(gst_loggable_error!(CAT, "Failed to parse input caps")))?; + let out_info = gst_audio::AudioInfo::from_caps(outcaps) + .or_else(|_| Err(gst_loggable_error!(CAT, "Failed to parse output caps")))?; + + let csound = self.csound.lock().unwrap(); + + let ichannels = in_info.channels(); + let ochannels = out_info.channels(); + let rate = in_info.rate(); + + // Check if the negotiated caps are the right ones + if rate != out_info.rate() || rate != csound.get_sample_rate() as _ { + return Err(gst_loggable_error!( + CAT, + "Failed to negotiate caps: invalid sample rate {}", + rate + )); + } else if ichannels != csound.input_channels() { + return Err(gst_loggable_error!( + CAT, + "Failed to negotiate caps: input channels {} not supported", + ichannels + )); + } else if ochannels != csound.output_channels() { + return Err(gst_loggable_error!( + CAT, + "Failed to negotiate caps: output channels {} not supported", + ochannels + )); + } + + let ksmps = csound.get_ksmps(); + + let adapter = gst_base::UniqueAdapter::new(); + + let mut state_lock = self.state.lock().unwrap(); + *state_lock = Some(State { + in_info, + out_info, + adapter, + ksmps, + }); + + Ok(()) + } + + fn generate_output( + &self, + element: &gst_base::BaseTransform, + ) -> Result { + // Check if there are enough data in the queued buffer and adapter, + // if it is not the case, just notify the parent class to not generate + // an output + if let Some(buffer) = self.take_queued_buffer() { + if buffer.get_flags() == gst::BufferFlags::DISCONT { + self.drain(element)?; + } + + let mut state_guard = self.state.lock().unwrap(); + let state = state_guard.as_mut().ok_or_else(|| { + gst_element_error!( + element, + gst::CoreError::Negotiation, + ["Can not generate an output without State"] + ); + gst::FlowError::NotNegotiated + })?; + + state.adapter.push(buffer); + if !state.needs_more_data() { + return self.generate_output(element, state); + } + } + gst_log!(CAT, "No enough data to generate output"); + Ok(GeneratedOutput::NoOutput) + } +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "csoundfilter", + gst::Rank::None, + CsoundFilter::get_type(), + ) +} diff --git a/gst-plugin-csound/src/lib.rs b/gst-plugin-csound/src/lib.rs new file mode 100644 index 00000000..d2b52bf7 --- /dev/null +++ b/gst-plugin-csound/src/lib.rs @@ -0,0 +1,41 @@ +// Copyright (C) 2020 Natanael Mojica +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this library; if not, write to the +// Free Software Foundation, Inc., 51 Franklin Street, Suite 500, +// Boston, MA 02110-1335, USA. + +#![crate_type = "cdylib"] + +use glib; +use gst; +use gst::gst_plugin_define; + +mod filter; + +fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + filter::register(plugin)?; + Ok(()) +} + +gst_plugin_define!( + csound, + env!("CARGO_PKG_DESCRIPTION"), + plugin_init, + concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")), + "MIT/X11", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_REPOSITORY"), + env!("BUILD_REL_DATE") +); diff --git a/gst-plugin-csound/tests/csound_filter.rs b/gst-plugin-csound/tests/csound_filter.rs new file mode 100644 index 00000000..5cbc966c --- /dev/null +++ b/gst-plugin-csound/tests/csound_filter.rs @@ -0,0 +1,423 @@ +// Copyright (C) 2020 Natanael Mojica +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this library; if not, write to the +// Free Software Foundation, Inc., 51 Franklin Street, Suite 500, +// Boston, MA 02110-1335, USA. +use gstcsound; + +use glib; + +use gst; +use gst_check; + +use glib::prelude::*; +use gst::prelude::*; + +use byte_slice_cast::*; + +// This macro allows us to create a kind of dynamic CSD file, +// we need to pass in the ksmps, channels and input/output +// operations that are going to be done by Csound over input and output +// audio samples +macro_rules! CSD { + ($ksmps:expr, $ichannels:expr, $ochannels:expr, $ins:expr, $out:expr) => { + format!( + " + + + + + sr = 44100 ; default sample rate + ksmps = {} + nchnls_i = {} + nchnls = {} + 0dbfs = 1 + + instr 1 + + {} ;input + {} ; csound output + + endin + + + i 1 0 2 + e + + ", + $ksmps, $ichannels, $ochannels, $ins, $out + ); + }; +} + +fn init() { + use std::sync::Once; + static INIT: Once = Once::new(); + + INIT.call_once(|| { + gst::init().unwrap(); + gstcsound::plugin_register_static().expect("Failed to register csound plugin"); + }); +} + +fn build_harness(src_caps: gst::Caps, sink_caps: gst::Caps, csd: &str) -> gst_check::Harness { + let filter = gst::ElementFactory::make("csoundfilter", None).unwrap(); + filter.set_property("csd-text", &csd).unwrap(); + + let mut h = gst_check::Harness::new_with_element(&filter, Some("sink"), Some("src")); + + h.set_caps(src_caps, sink_caps); + h +} + +fn duration_from_samples(num_samples: u64, rate: u64) -> gst::ClockTime { + gst::ClockTime(num_samples.mul_div_round(gst::SECOND_VAL, rate)) +} + +// This test verifies the well functioning of the EOS logic, +// we generate EOS_NUM_BUFFERS=10 buffers with EOS_NUM_SAMPLES=62 samples each one, +// for a total of 10 * 62 = 620 samples, but 620%32(ksmps)= 12 will be leftover and should be processed when +// the eos event is received, which generates another buffer, so that, the total amount of buffers that +// the harness would have at its sinkpad should be EOS_NUM_BUFFERS + 1, being the total amount of processed samples +// equals to EOS_NUM_BUFFERS * EOS_NUM_SAMPLES = 620 samples.It is important to mention that the created buffers have silenced samples(being 0), +// but csoundfilter would add 1.0 to each incoming sample. +// at the end, all of the output samples should have a value of 1.0. +const EOS_NUM_BUFFERS: usize = 10; +const EOS_NUM_SAMPLES: usize = 62; +#[test] +fn csound_filter_eos() { + init(); + + // Sets the ksmps to 32, + // input = output channels = 1 + let ksmps: usize = 32; + let num_channels = 1; + let sr: i32 = 44_100; + + let caps = gst::Caps::new_simple( + "audio/x-raw", + &[ + ("format", &gst_audio::AUDIO_FORMAT_F64.to_str()), + ("rate", &sr), + ("channels", &num_channels), + ("layout", &"interleaved"), + ], + ); + + let mut h = build_harness( + caps.clone(), + caps, + // this score instructs Csound to add 1.0 to each input sample + &CSD!(ksmps, num_channels, num_channels, "ain in", "out ain + 1.0"), + ); + h.play(); + + // The input buffer pts and duration + let mut in_pts = gst::ClockTime(Some(0)); + let in_duration = duration_from_samples(EOS_NUM_SAMPLES as _, sr as _); + // The number of samples that were leftover during the previous iteration + let mut samples_offset = 0; + // Output samples and buffers counters + let mut num_samples: usize = 0; + let mut num_buffers = 0; + // The expected pts of output buffers + let mut expected_pts = gst::ClockTime(Some(0)); + + for _ in 0..EOS_NUM_BUFFERS { + let mut buffer = + gst::Buffer::with_size(EOS_NUM_SAMPLES * std::mem::size_of::()).unwrap(); + + buffer.make_mut().set_pts(in_pts); + buffer.make_mut().set_duration(in_duration); + + let in_samples = samples_offset + EOS_NUM_SAMPLES as u64; + // Gets amount of samples that are going to be processed, + // the output buffer must be in_process_samples length + let in_process_samples = in_samples - (in_samples % ksmps as u64); + + // Push an input buffer and pull the result of processing it + let buffer = h.push_and_pull(buffer); + assert!(buffer.is_ok()); + + let buffer = buffer.unwrap(); + + // Checks output buffer timestamp and duration + assert_eq!( + buffer.as_ref().get_duration(), + duration_from_samples(in_process_samples, sr as _) + ); + assert_eq!(buffer.as_ref().get_pts(), expected_pts); + + // Get the number of samples that were not processed + samples_offset = in_samples % ksmps as u64; + // Calculates the next output buffer timestamp + expected_pts = + in_pts + duration_from_samples(EOS_NUM_SAMPLES as u64 - samples_offset, sr as _); + // Calculates the next input buffer timestamp + in_pts += in_duration; + + let map = buffer.into_mapped_buffer_readable().unwrap(); + let output = map.as_slice().as_slice_of::().unwrap(); + + // all samples in the output buffers must value 1 + assert_eq!(output.iter().any(|sample| *sample as u16 != 1u16), false); + + num_samples += output.len(); + num_buffers += 1; + } + + h.push_event(gst::Event::new_eos().build()); + + // pull the buffer produced after the EOS event + let buffer = h.pull().unwrap(); + + let samples_at_eos = (EOS_NUM_BUFFERS * EOS_NUM_SAMPLES) % ksmps; + assert_eq!( + buffer.as_ref().get_pts(), + in_pts - duration_from_samples(samples_at_eos as _, sr as _) + ); + + let map = buffer.into_mapped_buffer_readable().unwrap(); + let output = map.as_slice().as_slice_of::().unwrap(); + num_samples += output.len(); + num_buffers += 1; + + assert_eq!(output.len(), samples_at_eos); + assert_eq!(output.iter().any(|sample| *sample as u16 != 1u16), false); + + // All the generated samples should have been processed at this point + assert_eq!(num_samples, EOS_NUM_SAMPLES * EOS_NUM_BUFFERS); + assert_eq!(num_buffers, EOS_NUM_BUFFERS + 1); +} + +// In this test, we generate UNDERFLOW_NUM_BUFFERS buffers with UNDERFLOW_NUM_SAMPLES samples each one, however, +// Csound is waiting for UNDERFLOW_NUM_SAMPLES * 2 samples per buffer at its input, so that, +// internally, the output will be only generated when enough data is available. +// It happens, after every 2 * UNDERFLOW_NUM_BUFFERS input buffers, after processing, we should have UNDERFLOW_NUM_BUFFERS/2 +// output buffers containing UNDERFLOW_NUM_SAMPLES * 2 samples. +const UNDERFLOW_NUM_BUFFERS: usize = 200; +const UNDERFLOW_NUM_SAMPLES: usize = 2; +#[test] +fn csound_filter_underflow() { + init(); + + let ksmps: usize = UNDERFLOW_NUM_SAMPLES * 2; + let num_channels = 1; + let sr: i32 = 44_100; + + let caps = gst::Caps::new_simple( + "audio/x-raw", + &[ + ("format", &gst_audio::AUDIO_FORMAT_F64.to_str()), + ("rate", &sr), + ("channels", &num_channels), + ("layout", &"interleaved"), + ], + ); + + let mut h = build_harness( + caps.clone(), + caps, + &CSD!(ksmps, num_channels, num_channels, "ain in", "out ain"), + ); + h.play(); + + // Input buffers timestamp + let mut in_pts = gst::ClockTime(Some(0)); + let in_samples_duration = duration_from_samples(UNDERFLOW_NUM_SAMPLES as _, sr as _); + + for _ in 0..UNDERFLOW_NUM_BUFFERS { + let mut buffer = + gst::Buffer::with_size(UNDERFLOW_NUM_SAMPLES * std::mem::size_of::()).unwrap(); + + buffer.make_mut().set_pts(in_pts); + buffer.make_mut().set_duration(in_samples_duration); + + in_pts += in_samples_duration; + + assert!(h.push(buffer).is_ok()); + } + + h.push_event(gst::Event::new_eos().build()); + + // From here we check our output data + let mut num_buffers = 0; + let mut num_samples = 0; + + let expected_duration = duration_from_samples(UNDERFLOW_NUM_SAMPLES as u64 * 2, sr as _); + let expected_buffers = UNDERFLOW_NUM_BUFFERS / 2; + let mut expected_pts = gst::ClockTime(Some(0)); + + for _ in 0..expected_buffers { + let buffer = h.pull().unwrap(); + let samples = buffer.get_size() / std::mem::size_of::(); + + assert_eq!(buffer.as_ref().get_pts(), expected_pts); + assert_eq!(buffer.as_ref().get_duration(), expected_duration); + assert_eq!(samples, UNDERFLOW_NUM_SAMPLES * 2); + // Output data is produced after 2 input buffers + // so that, the next output buffer's PTS should be + // equal to the last PTS plus the duration of 2 input buffers + expected_pts += in_samples_duration * 2; + + num_buffers += 1; + num_samples += samples; + } + + assert_eq!(num_buffers, UNDERFLOW_NUM_BUFFERS / 2); + assert_eq!( + num_samples as usize, + UNDERFLOW_NUM_SAMPLES * UNDERFLOW_NUM_BUFFERS + ); +} + +// Verifies that the caps negotiation is properly done, by pushing buffers whose caps +// are the same as the one configured in csound, into the harness sink pad. Csoundfilter is expecting 2 channels audio +// at a sample rate of 44100. +// the output caps configured in the harness are not fixated but when the caps negotiation ends, +// those caps must be fixated according to the csound output format which is defined once the csd file is compiled +#[test] +fn csound_filter_caps_negotiation() { + init(); + + let ksmps = 4; + let ichannels = 2; + let ochannels = 1; + let sr: i32 = 44_100; + + let src_caps = gst::Caps::new_simple( + "audio/x-raw", + &[ + ("format", &gst_audio::AUDIO_FORMAT_F64.to_str()), + ("rate", &sr), + ("channels", &ichannels), + ("layout", &"interleaved"), + ], + ); + + // Define the output caps which would be fixated + // at the end of the caps negotiation + let sink_caps = gst::Caps::new_simple( + "audio/x-raw", + &[ + ("format", &gst_audio::AUDIO_FORMAT_F64.to_str()), + ("rate", &gst::IntRange::::new(1, 48000)), + ("channels", &gst::IntRange::::new(1, 2)), + ("layout", &"interleaved"), + ], + ); + + // build the harness setting its src and sink caps, + // also passing the csd score to the filter element + let mut h = build_harness( + src_caps, + sink_caps.clone(), + // creates a csd score that defines the input and output formats on the csound side + // the output fomart would be 1 channel audio samples at 44100 + &CSD!(ksmps, ichannels, ochannels, "ain, ain2 ins", "out ain"), + ); + + h.play(); + assert!(h.push(gst::Buffer::with_size(2048).unwrap()).is_ok()); + + h.push_event(gst::Event::new_eos().build()); + + let buffer = h.pull().unwrap(); + + // Pushing a buffer without a timestamp should produce a no timestamp output + assert!(buffer.as_ref().get_pts().is_none()); + // But It should have a duration + assert_eq!( + buffer.as_ref().get_duration(), + duration_from_samples(1024 / std::mem::size_of::() as u64, sr as u64) + ); + + // get the negotiated harness sink caps + let harness_sink_caps = h + .get_sinkpad() + .expect("harness has no sinkpad") + .get_current_caps() + .expect("pad has no caps"); + + // our expected caps at the harness sinkpad + let expected_caps = gst::Caps::new_simple( + "audio/x-raw", + &[ + ("format", &gst_audio::AUDIO_FORMAT_F64.to_str()), + ("rate", &44_100i32), + ("channels", &ochannels), + ("layout", &"interleaved"), + ], + ); + + assert_eq!(harness_sink_caps, expected_caps); +} + +// Similar to caps negotiation, but in this case, we configure a fixated caps in the harness sinkpad, +// such caps are incompatible with the csoundfilter and it leads to an error during the caps negotiation, +// because there is not a common intersection between both caps. +#[test] +fn csound_filter_caps_negotiation_fail() { + init(); + + let ksmps = 4; + let ichannels = 2; + let ochannels = 1; + + let src_caps = gst::Caps::new_simple( + "audio/x-raw", + &[ + ("format", &gst_audio::AUDIO_FORMAT_F64.to_str()), + ("rate", &44_100i32), + ("channels", &ichannels), + ("layout", &"interleaved"), + ], + ); + + // instead of having a range for channels/rate fields + // we fixate them to 2 and 48_000 respectively, which would cause the negotiation error + let sink_caps = gst::Caps::new_simple( + "audio/x-raw", + &[ + ("format", &gst_audio::AUDIO_FORMAT_F64.to_str()), + ("rate", &48_000i32), + ("channels", &ichannels), + ("layout", &"interleaved"), + ], + ); + + let mut h = build_harness( + src_caps, + sink_caps, + // creates a csd score that defines the input and output formats on the csound side + // the output fomart would be 1 channel audio samples at 44100 + &CSD!(ksmps, ichannels, ochannels, "ain, ain2 ins", "out ain"), + ); + + h.play(); + + let buffer = gst::Buffer::with_size(2048).unwrap(); + assert!(h.push(buffer).is_err()); + + h.push_event(gst::Event::new_eos().build()); + + // The harness sinkpad end up not having defined caps + // so, the get_current_caps should be None + let current_caps = h + .get_sinkpad() + .expect("harness has no sinkpad") + .get_current_caps(); + + assert!(current_caps.is_none()); +} diff --git a/meson.build b/meson.build index d0f2bd43..cfe34e31 100644 --- a/meson.build +++ b/meson.build @@ -1,5 +1,6 @@ project('gst-plugins-rs', 'rust', + 'c', version: '0.13.0', meson_version : '>= 0.52') @@ -59,6 +60,28 @@ else exclude += ['gst-plugin-sodium'] endif +cc = meson.get_compiler('c') +csound_option = get_option('csound') +csound_dep = dependency('', required: false) # not-found dependency +if not csound_option.disabled() + csound_dep = cc.find_library('csound64', required: false) + if not csound_dep.found() + python3 = import('python').find_installation('python3') + res = run_command(python3, '-c', 'import os; print(os.environ["CSOUND_LIB_DIR"])') + if res.returncode() == 0 + csound_dep = cc.find_library('csound64', dirs: res.stdout(), required: csound_option) + elif csound_option.enabled() + error('csound option is enabled, but csound64 library could not be found and CSOUND_LIB_DIR was not set') + endif + endif +endif + +if csound_dep.found() + plugins_rep += {'gst-plugin-csound' : 'libgstcsound'} +else + exclude += ['gst-plugin-csound'] +endif + output = [] foreach p, lib : plugins_rep diff --git a/meson_options.txt b/meson_options.txt index 4adb4520..fe261e6d 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -2,3 +2,4 @@ option('dav1d', type : 'feature', value : 'auto', description : 'Build dav1d plu option('sodium', type : 'combo', choices : ['system', 'built-in', 'disabled'], value : 'built-in', description : 'Weither to use libsodium from the system or the built-in version from the sodiumoxide crate') +option('csound', type : 'feature', value : 'auto', description : 'Build csound plugin')