mirrors/gst-plugins-rs
mirrors/gst-plugins-rs
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git synced 2025-02-21 07:06:19 +00:00
Code Issues Projects Releases Wiki Activity

Add csound-based filter plugin

Browse source
This commit is contained in:
Natanael Mojica 2020-01-09 19:52:31 -06:00 committed by Sebastian Dröge
parent 116cf9bd3c
commit cf59318ab4
12 changed files with 1391 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitlab-ci.yml
View file

@ -28,6 +28,7 @@ stages:
libwayland-egl1-mesa
llvm
nasm
libcsound64-dev
python3-pip
python3-setuptools
python3-wheel
@ -52,7 +53,6 @@ stages:
# FIXME: The feature name should explicitly mention the dav1d plugin but
# Cargo currently doesn't support passthrough for that scenario.
- export RUSTFLAGS='--cfg feature="build"'
- cd "${CI_PROJECT_DIR}"
cache:
key: "gst"

1
Cargo.toml
View file

@ -19,6 +19,7 @@ members = [
"gst-plugin-lewton",
"gst-plugin-claxon",
"gst-plugin-gif",
"gst-plugin-csound",
]
[profile.release]

4
README.md
View file

@ -24,6 +24,10 @@ gst-plugin-togglerecord is licensed under the Lesser General Public License
([LICENSE-LGPLv2](LICENSE-LGPLv2)) version 2.1 or (at your option) any later
version.
gst-plugin-csound is licensed under the Lesser General Public License
([LICENSE-LGPLv2](LICENSE-LGPLv2)) version 2.1 or (at your option) any later
version.
GStreamer itself is licensed under the Lesser General Public License version
2.1 or (at your option) any later version:
https://www.gnu.org/licenses/lgpl-2.1.html

30
gst-plugin-csound/Cargo.toml Normal file
View file

@ -0,0 +1,30 @@
[package]
name = "gst-plugin-csound"
version = "0.1.0"
authors = ["Natanael Mojica <neithanmo@gmail.com>"]
repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs"
license = "LGPL-2.1+"
edition = "2018"
description = "An Audio filter plugin based on Csound"
[dependencies]
glib = { git = "https://github.com/gtk-rs/glib" }
gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
gst_base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
gst_audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
gst_check = { package = "gstreamer-check", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
csound = { git = "https://github.com/neithanmo/csound-rs"}
once_cell = "1.0"
byte-slice-cast = "0.3"
[lib]
name = "gstcsound"
crate-type = ["cdylib", "rlib"]
path = "src/lib.rs"
[[example]]
name = "csound-effect"
path = "examples/effect_example.rs"
[build-dependencies]
gst-plugin-version-helper = { git = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" }

21
gst-plugin-csound/README.md Normal file
View file

@ -0,0 +1,21 @@
# gst-plugin-csound
This is a [GStreamer](https://gstreamer.freedesktop.org/) plugin to interact
with the [Csound](https://csound.com/) sound computing system.
Currently, there is only a filter element, called, csoundfilter. Two more elements a source and sink would be implemented
later on.
For more information about dependencies and installation process, please refer to the [csound-rs](https://crates.io/crates/csound)
documentation
## simple example
The included example constructs the follow pipeline
```
$ gst-launch-1.0 \
audiotestsrc ! \
audioconvert ! \
csoundfilter location=effect.csd ! \
audioconvert ! \
autoaudiosink
```

5
gst-plugin-csound/build.rs Normal file
View file

@ -0,0 +1,5 @@
extern crate gst_plugin_version_helper;
fn main() {
gst_plugin_version_helper::get_info()
}

144
gst-plugin-csound/examples/effect_example.rs Normal file
View file

@ -0,0 +1,144 @@
// Copyright (C) 2020 Natanael Mojica <neithanmo@gmail.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.
use glib;
use glib::prelude::*;
use gst;
use gst::prelude::*;
use gstcsound;
use std::error::Error;
const AUDIO_SRC: &str = "audiotestsrc";
const AUDIO_SINK: &str = "audioconvert ! autoaudiosink";
// This example defines two instruments, the first instrument send to the output that is at its input and accumulates the received audio samples
// into a global variable called gasig. The execution of this instrument last the first 2 seconds.
// The second instrument starts it execution at 1.8 second, This instrument creates two audio buffers with samples that are read
// from the global accumulator(gasig), then reads these buffers at a fixed delay time, creating the adelL, adelM and adelR buffers,
// also, It multiplies the audio samples in the right channel by 0.5 * kdel, being kdel a line of values starting at 0.5 at increments of 0.001.
// Finally, those buffers are mixed with the accumulator, and an audio envelop is applied(aseg) to them.
// The result is similar to an audio echo in which the buffered samples are read at different delay times and also modified in frecuency(right channel),
// this creates an space effect using just one channel audio input.
const CSD: &str = "
<CsoundSynthesizer>
<CsOptions>
</CsOptions>
<CsInstruments>
sr = 44100
ksmps = 7
nchnls_i = 1
nchnls = 2
gasig init 0
gidel = 1
instr 1
ain in
outs ain, ain
vincr gasig, ain
endin
instr 2
ifeedback = p4
aseg linseg 1., p3, 0.0
abuf2 delayr gidel
adelL deltap .4
adelM deltap .5
delayw gasig + (adelL * ifeedback)
abuf3 delayr gidel
kdel line .5, p3, .001
adelR deltap .5 * kdel
delayw gasig + (adelR * ifeedback)
outs (adelL + adelM) * aseg, (adelR + adelM) * aseg
clear gasig
endin
</CsInstruments>
<CsScore>
i 1 0 2
i 2 1.8 5 .8
e
</CsScore>
</CsoundSynthesizer>";
fn create_pipeline() -> Result<gst::Pipeline, Box<dyn Error>> {
let pipeline = gst::Pipeline::new(None);
let audio_src = gst::parse_bin_from_description(AUDIO_SRC, true)?.upcast();
let audio_sink = gst::parse_bin_from_description(AUDIO_SINK, true)?.upcast();
let csoundfilter = gst::ElementFactory::make("csoundfilter", None)?;
csoundfilter.set_property("csd-text", &CSD)?;
pipeline.add_many(&[&audio_src, &csoundfilter, &audio_sink])?;
audio_src.link_pads(Some("src"), &csoundfilter, Some("sink"))?;
csoundfilter.link_pads(Some("src"), &audio_sink, Some("sink"))?;
Ok(pipeline)
}
fn main_loop(pipeline: gst::Pipeline) -> Result<(), Box<dyn Error>> {
pipeline.set_state(gst::State::Playing)?;
let bus = pipeline
.get_bus()
.expect("Pipeline without bus. Shouldn't happen!");
for msg in bus.iter_timed(gst::CLOCK_TIME_NONE) {
use gst::MessageView;
match msg.view() {
MessageView::Eos(..) => break,
MessageView::Error(err) => {
println!(
"Error from {:?}: {} ({:?})",
msg.get_src().map(|s| s.get_path_string()),
err.get_error(),
err.get_debug()
);
break;
}
_ => (),
}
}
pipeline.set_state(gst::State::Null)?;
Ok(())
}
fn main() -> Result<(), Box<dyn Error>> {
gst::init().unwrap();
gstcsound::plugin_register_static().expect("Failed to register csound plugin");
create_pipeline().and_then(main_loop)
}

697
gst-plugin-csound/src/filter.rs Normal file
View file

@ -0,0 +1,697 @@
// Copyright (C) 2020 Natanael Mojica <neithanmo@gmail.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.
use glib;
use glib::subclass;
use glib::subclass::prelude::*;
use glib::{glib_object_impl, glib_object_subclass};
use gst;
use gst::prelude::*;
use gst::subclass::prelude::*;
use gst::{
gst_debug, gst_element_error, gst_error, gst_error_msg, gst_info, gst_log, gst_loggable_error,
gst_warning,
};
use gst_audio;
use gst_base;
use gst_base::subclass::base_transform::BaseTransformImplExt;
use gst_base::subclass::base_transform::GeneratedOutput;
use gst_base::subclass::prelude::*;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Mutex;
use std::{f64, i32};
use byte_slice_cast::*;
use csound::{Csound, MessageType};
use once_cell::sync::Lazy;
static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
gst::DebugCategory::new(
"csoundfilter",
gst::DebugColorFlags::empty(),
Some("Audio Filter based on Csound"),
)
});
const SCORE_OFFSET_DEFAULT: f64 = 0f64;
const DEFAULT_LOOP: bool = false;
#[derive(Debug, Clone)]
struct Settings {
pub loop_: bool,
pub location: Option<String>,
pub csd_text: Option<String>,
pub offset: f64,
}
impl Default for Settings {
fn default() -> Self {
Settings {
loop_: DEFAULT_LOOP,
location: None,
csd_text: None,
offset: SCORE_OFFSET_DEFAULT,
}
}
}
struct State {
in_info: gst_audio::AudioInfo,
out_info: gst_audio::AudioInfo,
adapter: gst_base::UniqueAdapter,
ksmps: u32,
}
struct CsoundFilter {
settings: Mutex<Settings>,
state: Mutex<Option<State>>,
csound: Mutex<Csound>,
compiled: AtomicBool,
}
static PROPERTIES: [subclass::Property; 4] = [
subclass::Property("loop", |name| {
glib::ParamSpec::boolean(
name,
"Loop",
"loop over the score (can be changed in PLAYING or PAUSED state)",
DEFAULT_LOOP,
glib::ParamFlags::READWRITE,
)
}),
subclass::Property("location", |name| {
glib::ParamSpec::string(
name,
"Location",
"Location of the csd file to be used by csound.
Use either location or CSD-text but not both at the same time, if so and error would be triggered",
None,
glib::ParamFlags::READWRITE,
)
}),
subclass::Property("csd-text", |name| {
glib::ParamSpec::string(
name,
"CSD-text",
"The content of a csd file passed as a String.
Use either location or csd-text but not both at the same time, if so and error would be triggered",
None,
glib::ParamFlags::READWRITE,
)
}),
subclass::Property("score_offset", |name| {
glib::ParamSpec::double(
name,
"Score Offset",
"Score offset in seconds to start the performance",
0.0,
f64::MAX,
SCORE_OFFSET_DEFAULT,
glib::ParamFlags::READWRITE,
)
}),
];
impl State {
// Considering an input of size: input_size and the user's ksmps,
// calculates the equivalent output_size
fn max_output_size(&self, input_size: usize) -> usize {
let in_samples = input_size / self.in_info.bpf() as usize;
let in_process_samples = in_samples - (in_samples % self.ksmps as usize);
in_process_samples * self.out_info.bpf() as usize
}
fn get_bytes_to_read(&mut self, output_size: usize) -> usize {
// The max amount of bytes at the input that We would need
// for filling an output buffer of size *output_size*
(output_size / self.out_info.bpf() as usize) * self.in_info.bpf() as usize
}
// returns the spin capacity in bytes
fn spin_capacity(&self) -> usize {
(self.ksmps * self.in_info.bpf()) as _
}
fn needs_more_data(&self) -> bool {
self.adapter.available() < self.spin_capacity()
}
fn samples_to_time(&self, samples: u64) -> gst::ClockTime {
gst::ClockTime(samples.mul_div_round(gst::SECOND_VAL, self.in_info.rate() as u64))
}
fn get_current_pts(&self) -> gst::ClockTime {
// get the last seen pts and the amount of bytes
// since then
let (prev_pts, distance) = self.adapter.prev_pts();
// Use the distance to get the amount of samples
// and with it calculate the time-offset which
// can be added to the prev_pts to get the
// pts at the beginning of the adapter.
let samples = distance / self.in_info.bpf() as u64;
prev_pts + self.samples_to_time(samples)
}
fn buffer_duration(&self, buffer_size: u64) -> gst::ClockTime {
let samples = buffer_size / self.out_info.bpf() as u64;
self.samples_to_time(samples)
}
}
impl CsoundFilter {
fn process(&self, csound: &mut Csound, idata: &[f64], odata: &mut [f64]) -> bool {
let spin = csound.get_spin().unwrap();
let spout = csound.get_spout().unwrap();
let in_chunks = idata.chunks_exact(spin.len());
let out_chuncks = odata.chunks_exact_mut(spout.len());
let mut end_score = false;
for (ichunk, ochunk) in in_chunks.zip(out_chuncks) {
spin.copy_from_slice(ichunk);
end_score = csound.perform_ksmps();
spout.copy_to_slice(ochunk);
}
end_score
}
fn compile_score(&self) -> std::result::Result<(), gst::ErrorMessage> {
let csound = self.csound.lock().unwrap();
let settings = self.settings.lock().unwrap();
if let Some(ref location) = settings.location {
csound
.compile_csd(location)
.map_err(|e| gst_error_msg!(gst::LibraryError::Failed, [e]))?;
} else if let Some(ref text) = settings.csd_text {
csound
.compile_csd_text(text)
.map_err(|e| gst_error_msg!(gst::LibraryError::Failed, [e]))?;
} else {
return Err(gst_error_msg!(
gst::LibraryError::Failed,
["No Csound score specified to compile. Use either location or csd-text but not both"]
));
}
self.compiled.store(true, Ordering::SeqCst);
Ok(())
}
fn message_callback(msg_type: MessageType, msg: &str) {
match msg_type {
MessageType::CSOUNDMSG_ERROR => gst_error!(CAT, "{}", msg),
MessageType::CSOUNDMSG_WARNING => gst_warning!(CAT, "{}", msg),
MessageType::CSOUNDMSG_ORCH => gst_info!(CAT, "{}", msg),
MessageType::CSOUNDMSG_REALTIME => gst_log!(CAT, "{}", msg),
MessageType::CSOUNDMSG_DEFAULT => gst_log!(CAT, "{}", msg),
MessageType::CSOUNDMSG_STDOUT => gst_log!(CAT, "{}", msg),
}
}
fn drain(&self, element: &gst_base::BaseTransform) -> Result<gst::FlowSuccess, gst::FlowError> {
let csound = self.csound.lock().unwrap();
let mut state_lock = self.state.lock().unwrap();
let state = state_lock.as_mut().unwrap();
let avail = state.adapter.available();
// Complete processing blocks should have been processed in the transform call
assert!(avail < state.spin_capacity());
if avail == 0 {
return Ok(gst::FlowSuccess::Ok);
}
let mut spin = csound.get_spin().unwrap();
let spout = csound.get_spout().unwrap();
let out_bytes =
(avail / state.in_info.channels() as usize) * state.out_info.channels() as usize;
let mut buffer = gst::Buffer::with_size(out_bytes).map_err(|e| {
gst_error!(
CAT,
obj: element,
"Failed to allocate buffer at EOS {:?}",
e
);
gst::FlowError::Flushing
})?;
let buffer_mut = buffer.get_mut().ok_or(gst::FlowError::NotSupported)?;
let pts = state.get_current_pts();
let duration = state.buffer_duration(out_bytes as _);
buffer_mut.set_pts(pts);
buffer_mut.set_duration(duration);
let srcpad = element.get_static_pad("src").unwrap();
let adapter_map = state.adapter.map(avail).unwrap();
let data = adapter_map
.as_ref()
.as_slice_of::<f64>()
.map_err(|_| gst::FlowError::NotSupported)?;
let mut omap = buffer_mut
.map_writable()
.map_err(|_| gst::FlowError::NotSupported)?;
let odata = omap
.as_mut_slice_of::<f64>()
.map_err(|_| gst::FlowError::NotSupported)?;
spin.clear();
spin.copy_from_slice(data);
csound.perform_ksmps();
spout.copy_to_slice(odata);
drop(adapter_map);
drop(omap);
state.adapter.flush(avail);
// Drop the locks before pushing buffers into the srcpad
drop(state_lock);
drop(csound);
srcpad.push(buffer)
}
fn generate_output(
&self,
element: &gst_base::BaseTransform,
state: &mut State,
) -> Result<GeneratedOutput, gst::FlowError> {
let output_size = state.max_output_size(state.adapter.available());
let mut output = gst::Buffer::with_size(output_size).map_err(|_| gst::FlowError::Error)?;
let outbuf = output.get_mut().ok_or(gst::FlowError::Error)?;
let pts = state.get_current_pts();
let duration = state.buffer_duration(output_size as _);
outbuf.set_pts(pts);
outbuf.set_duration(duration);
gst_log!(
CAT,
obj: element,
"Generating output at: {} - duration: {}",
pts,
duration
);
// Get the required amount of bytes to be read from
// the adapter to fill an ouput buffer of size output_size
let bytes_to_read = state.get_bytes_to_read(output_size);
let indata = state
.adapter
.map(bytes_to_read)
.map_err(|_| gst::FlowError::Error)?;
let idata = indata
.as_ref()
.as_slice_of::<f64>()
.map_err(|_| gst::FlowError::Error)?;
let mut omap = outbuf.map_writable().map_err(|_| gst::FlowError::Error)?;
let odata = omap
.as_mut_slice_of::<f64>()
.map_err(|_| gst::FlowError::Error)?;
let mut csound = self.csound.lock().unwrap();
let end_score = self.process(&mut csound, idata, odata);
drop(indata);
drop(omap);
state.adapter.flush(bytes_to_read);
if end_score {
let settings = self.settings.lock().unwrap();
if settings.loop_ {
csound.set_score_offset_seconds(settings.offset);
csound.rewind_score();
} else {
// clear the adapter here because our eos event handler
// will try to flush it calling csound.perform()
// which does not make sense since
// the end of score has been reached.
state.adapter.clear();
return Err(gst::FlowError::Eos);
}
}
Ok(GeneratedOutput::Buffer(output))
}
}
impl ObjectSubclass for CsoundFilter {
const NAME: &'static str = "CsoundFilter";
type ParentType = gst_base::BaseTransform;
type Instance = gst::subclass::ElementInstanceStruct<Self>;
type Class = subclass::simple::ClassStruct<Self>;
glib_object_subclass!();
fn new() -> Self {
let csound = Csound::new();
// create the csound instance and configure
csound.message_string_callback(Self::message_callback);
// Disable all default handling of sound I/O by csound internal library
// by giving to it a hardware buffer size of zero, and setting a state,
// higher than zero.
csound.set_host_implemented_audioIO(1, 0);
// We don't want csound to write samples to our HW
csound.set_option("--nosound").unwrap();
Self {
settings: Mutex::new(Default::default()),
state: Mutex::new(None),
csound: Mutex::new(csound),
compiled: AtomicBool::new(false),
}
}
fn class_init(klass: &mut subclass::simple::ClassStruct<Self>) {
klass.set_metadata(
"Audio filter",
"Filter/Effect/Audio",
"Implement an audio filter/effects using Csound",
"Natanael Mojica <neithanmo@gmail.com>",
);
let caps = gst::Caps::new_simple(
"audio/x-raw",
&[
("format", &gst_audio::AUDIO_FORMAT_F64.to_str()),
("rate", &gst::IntRange::<i32>::new(1, i32::MAX)),
("channels", &gst::IntRange::<i32>::new(1, i32::MAX)),
("layout", &"interleaved"),
],
);
let src_pad_template = gst::PadTemplate::new(
"src",
gst::PadDirection::Src,
gst::PadPresence::Always,
&caps,
)
.unwrap();
klass.add_pad_template(src_pad_template);
let sink_pad_template = gst::PadTemplate::new(
"sink",
gst::PadDirection::Sink,
gst::PadPresence::Always,
&caps,
)
.unwrap();
klass.add_pad_template(sink_pad_template);
klass.install_properties(&PROPERTIES);
klass.configure(
gst_base::subclass::BaseTransformMode::NeverInPlace,
false,
false,
);
}
}
impl ObjectImpl for CsoundFilter {
glib_object_impl!();
fn set_property(&self, _obj: &glib::Object, id: usize, value: &glib::Value) {
let prop = &PROPERTIES[id];
match *prop {
subclass::Property("loop", ..) => {
let mut settings = self.settings.lock().unwrap();
settings.loop_ = value.get_some().expect("type checked upstream");
}
subclass::Property("location", ..) => {
let mut settings = self.settings.lock().unwrap();
if self.state.lock().unwrap().is_none() {
settings.location = match value.get::<String>() {
Ok(location) => location,
_ => unreachable!("type checked upstream"),
};
}
}
subclass::Property("csd-text", ..) => {
let mut settings = self.settings.lock().unwrap();
if self.state.lock().unwrap().is_none() {
settings.csd_text = match value.get::<String>() {
Ok(text) => text,
_ => unreachable!("type checked upstream"),
};
}
}
subclass::Property("score_offset", ..) => {
let mut settings = self.settings.lock().unwrap();
settings.offset = value.get_some().expect("type checked upstream");
}
_ => unimplemented!(),
}
}
fn get_property(&self, _obj: &glib::Object, id: usize) -> Result<glib::Value, ()> {
let prop = &PROPERTIES[id];
match *prop {
subclass::Property("loop", ..) => {
let settings = self.settings.lock().unwrap();
Ok(settings.loop_.to_value())
}
subclass::Property("location", ..) => {
let settings = self.settings.lock().unwrap();
Ok(settings.location.to_value())
}
subclass::Property("csd-text", ..) => {
let settings = self.settings.lock().unwrap();
Ok(settings.csd_text.to_value())
}
subclass::Property("score_offset", ..) => {
let settings = self.settings.lock().unwrap();
Ok(settings.offset.to_value())
}
_ => unimplemented!(),
}
}
}
impl ElementImpl for CsoundFilter {}
impl BaseTransformImpl for CsoundFilter {
fn start(
&self,
_element: &gst_base::BaseTransform,
) -> std::result::Result<(), gst::ErrorMessage> {
self.compile_score()?;
let csound = self.csound.lock().unwrap();
let settings = self.settings.lock().unwrap();
csound.set_score_offset_seconds(settings.offset);
if let Err(e) = csound.start() {
return Err(gst_error_msg!(gst::LibraryError::Failed, [e]));
}
Ok(())
}
fn stop(&self, element: &gst_base::BaseTransform) -> Result<(), gst::ErrorMessage> {
let csound = self.csound.lock().unwrap();
csound.stop();
csound.reset();
let _ = self.state.lock().unwrap().take();
gst_info!(CAT, obj: element, "Stopped");
Ok(())
}
fn sink_event(&self, element: &gst_base::BaseTransform, event: gst::Event) -> bool {
use gst::EventView;
if let EventView::Eos(_) = event.view() {
gst_log!(CAT, obj: element, "Handling Eos");
if self.drain(element).is_err() {
return false;
}
}
self.parent_sink_event(element, event)
}
fn transform_caps(
&self,
element: &gst_base::BaseTransform,
direction: gst::PadDirection,
caps: &gst::Caps,
filter: Option<&gst::Caps>,
) -> Option<gst::Caps> {
let compiled = self.compiled.load(Ordering::SeqCst);
let mut other_caps = {
// Our caps proposal
let mut new_caps = caps.clone();
if compiled {
let csound = self.csound.lock().unwrap();
// Use the sample rate and channels configured in the csound score
let sr = csound.get_sample_rate() as i32;
let ichannels = csound.input_channels() as i32;
let ochannels = csound.output_channels() as i32;
for s in new_caps.make_mut().iter_mut() {
s.set("format", &gst_audio::AUDIO_FORMAT_F64.to_str());
s.set("rate", &sr);
// replace the channel property with our values,
// if they are not supported, the negotiation will fail.
if direction == gst::PadDirection::Src {
s.set("channels", &ichannels);
} else {
s.set("channels", &ochannels);
}
// Csound does not have a concept of channel-mask
s.remove_field("channel-mask");
}
}
new_caps
};
gst_debug!(
CAT,
obj: element,
"Transformed caps from {} to {} in direction {:?}",
caps,
other_caps,
direction
);
if let Some(filter) = filter {
other_caps = filter.intersect_with_mode(&other_caps, gst::CapsIntersectMode::First);
}
Some(other_caps)
}
fn set_caps(
&self,
element: &gst_base::BaseTransform,
incaps: &gst::Caps,
outcaps: &gst::Caps,
) -> Result<(), gst::LoggableError> {
// Flush previous state
if self.state.lock().unwrap().is_some() {
self.drain(element).or_else(|e| {
Err(gst_loggable_error!(
CAT,
"Error flusing previous state data {:?}",
e
))
})?;
}
let in_info = gst_audio::AudioInfo::from_caps(incaps)
.or_else(|_| Err(gst_loggable_error!(CAT, "Failed to parse input caps")))?;
let out_info = gst_audio::AudioInfo::from_caps(outcaps)
.or_else(|_| Err(gst_loggable_error!(CAT, "Failed to parse output caps")))?;
let csound = self.csound.lock().unwrap();
let ichannels = in_info.channels();
let ochannels = out_info.channels();
let rate = in_info.rate();
// Check if the negotiated caps are the right ones
if rate != out_info.rate() || rate != csound.get_sample_rate() as _ {
return Err(gst_loggable_error!(
CAT,
"Failed to negotiate caps: invalid sample rate {}",
rate
));
} else if ichannels != csound.input_channels() {
return Err(gst_loggable_error!(
CAT,
"Failed to negotiate caps: input channels {} not supported",
ichannels
));
} else if ochannels != csound.output_channels() {
return Err(gst_loggable_error!(
CAT,
"Failed to negotiate caps: output channels {} not supported",
ochannels
));
}
let ksmps = csound.get_ksmps();
let adapter = gst_base::UniqueAdapter::new();
let mut state_lock = self.state.lock().unwrap();
*state_lock = Some(State {
in_info,
out_info,
adapter,
ksmps,
});
Ok(())
}
fn generate_output(
&self,
element: &gst_base::BaseTransform,
) -> Result<GeneratedOutput, gst::FlowError> {
// Check if there are enough data in the queued buffer and adapter,
// if it is not the case, just notify the parent class to not generate
// an output
if let Some(buffer) = self.take_queued_buffer() {
if buffer.get_flags() == gst::BufferFlags::DISCONT {
self.drain(element)?;
}
let mut state_guard = self.state.lock().unwrap();
let state = state_guard.as_mut().ok_or_else(|| {
gst_element_error!(
element,
gst::CoreError::Negotiation,
["Can not generate an output without State"]
);
gst::FlowError::NotNegotiated
})?;
state.adapter.push(buffer);
if !state.needs_more_data() {
return self.generate_output(element, state);
}
}
gst_log!(CAT, "No enough data to generate output");
Ok(GeneratedOutput::NoOutput)
}
}
pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
gst::Element::register(
Some(plugin),
"csoundfilter",
gst::Rank::None,
CsoundFilter::get_type(),
)
}

41
gst-plugin-csound/src/lib.rs Normal file
View file

@ -0,0 +1,41 @@
// Copyright (C) 2020 Natanael Mojica <neithanmo@gmail.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.
#![crate_type = "cdylib"]
use glib;
use gst;
use gst::gst_plugin_define;
mod filter;
fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
filter::register(plugin)?;
Ok(())
}
gst_plugin_define!(
csound,
env!("CARGO_PKG_DESCRIPTION"),
plugin_init,
concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")),
"MIT/X11",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_REPOSITORY"),
env!("BUILD_REL_DATE")
);

423
gst-plugin-csound/tests/csound_filter.rs Normal file
View file

@ -0,0 +1,423 @@
// Copyright (C) 2020 Natanael Mojica <neithanmo@gmail.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.
use gstcsound;
use glib;
use gst;
use gst_check;
use glib::prelude::*;
use gst::prelude::*;
use byte_slice_cast::*;
// This macro allows us to create a kind of dynamic CSD file,
// we need to pass in the ksmps, channels and input/output
// operations that are going to be done by Csound over input and output
// audio samples
macro_rules! CSD {
($ksmps:expr, $ichannels:expr, $ochannels:expr, $ins:expr, $out:expr) => {
format!(
"
<CsoundSynthesizer>
<CsOptions>
</CsOptions>
<CsInstruments>
sr = 44100 ; default sample rate
ksmps = {}
nchnls_i = {}
nchnls = {}
0dbfs = 1
instr 1
{} ;input
{} ; csound output
endin
</CsInstruments>
<CsScore>
i 1 0 2
e
</CsScore>
</CsoundSynthesizer>",
$ksmps, $ichannels, $ochannels, $ins, $out
);
};
}
fn init() {
use std::sync::Once;
static INIT: Once = Once::new();
INIT.call_once(|| {
gst::init().unwrap();
gstcsound::plugin_register_static().expect("Failed to register csound plugin");
});
}
fn build_harness(src_caps: gst::Caps, sink_caps: gst::Caps, csd: &str) -> gst_check::Harness {
let filter = gst::ElementFactory::make("csoundfilter", None).unwrap();
filter.set_property("csd-text", &csd).unwrap();
let mut h = gst_check::Harness::new_with_element(&filter, Some("sink"), Some("src"));
h.set_caps(src_caps, sink_caps);
h
}
fn duration_from_samples(num_samples: u64, rate: u64) -> gst::ClockTime {
gst::ClockTime(num_samples.mul_div_round(gst::SECOND_VAL, rate))
}
// This test verifies the well functioning of the EOS logic,
// we generate EOS_NUM_BUFFERS=10 buffers with EOS_NUM_SAMPLES=62 samples each one,
// for a total of 10 * 62 = 620 samples, but 620%32(ksmps)= 12 will be leftover and should be processed when
// the eos event is received, which generates another buffer, so that, the total amount of buffers that
// the harness would have at its sinkpad should be EOS_NUM_BUFFERS + 1, being the total amount of processed samples
// equals to EOS_NUM_BUFFERS * EOS_NUM_SAMPLES = 620 samples.It is important to mention that the created buffers have silenced samples(being 0),
// but csoundfilter would add 1.0 to each incoming sample.
// at the end, all of the output samples should have a value of 1.0.
const EOS_NUM_BUFFERS: usize = 10;
const EOS_NUM_SAMPLES: usize = 62;
#[test]
fn csound_filter_eos() {
init();
// Sets the ksmps to 32,
// input = output channels = 1
let ksmps: usize = 32;
let num_channels = 1;
let sr: i32 = 44_100;
let caps = gst::Caps::new_simple(
"audio/x-raw",
&[
("format", &gst_audio::AUDIO_FORMAT_F64.to_str()),
("rate", &sr),
("channels", &num_channels),
("layout", &"interleaved"),
],
);
let mut h = build_harness(
caps.clone(),
caps,
// this score instructs Csound to add 1.0 to each input sample
&CSD!(ksmps, num_channels, num_channels, "ain in", "out ain + 1.0"),
);
h.play();
// The input buffer pts and duration
let mut in_pts = gst::ClockTime(Some(0));
let in_duration = duration_from_samples(EOS_NUM_SAMPLES as _, sr as _);
// The number of samples that were leftover during the previous iteration
let mut samples_offset = 0;
// Output samples and buffers counters
let mut num_samples: usize = 0;
let mut num_buffers = 0;
// The expected pts of output buffers
let mut expected_pts = gst::ClockTime(Some(0));
for _ in 0..EOS_NUM_BUFFERS {
let mut buffer =
gst::Buffer::with_size(EOS_NUM_SAMPLES * std::mem::size_of::<f64>()).unwrap();
buffer.make_mut().set_pts(in_pts);
buffer.make_mut().set_duration(in_duration);
let in_samples = samples_offset + EOS_NUM_SAMPLES as u64;
// Gets amount of samples that are going to be processed,
// the output buffer must be in_process_samples length
let in_process_samples = in_samples - (in_samples % ksmps as u64);
// Push an input buffer and pull the result of processing it
let buffer = h.push_and_pull(buffer);
assert!(buffer.is_ok());
let buffer = buffer.unwrap();
// Checks output buffer timestamp and duration
assert_eq!(
buffer.as_ref().get_duration(),
duration_from_samples(in_process_samples, sr as _)
);
assert_eq!(buffer.as_ref().get_pts(), expected_pts);
// Get the number of samples that were not processed
samples_offset = in_samples % ksmps as u64;
// Calculates the next output buffer timestamp
expected_pts =
in_pts + duration_from_samples(EOS_NUM_SAMPLES as u64 - samples_offset, sr as _);
// Calculates the next input buffer timestamp
in_pts += in_duration;
let map = buffer.into_mapped_buffer_readable().unwrap();
let output = map.as_slice().as_slice_of::<f64>().unwrap();
// all samples in the output buffers must value 1
assert_eq!(output.iter().any(|sample| *sample as u16 != 1u16), false);
num_samples += output.len();
num_buffers += 1;
}
h.push_event(gst::Event::new_eos().build());
// pull the buffer produced after the EOS event
let buffer = h.pull().unwrap();
let samples_at_eos = (EOS_NUM_BUFFERS * EOS_NUM_SAMPLES) % ksmps;
assert_eq!(
buffer.as_ref().get_pts(),
in_pts - duration_from_samples(samples_at_eos as _, sr as _)
);
let map = buffer.into_mapped_buffer_readable().unwrap();
let output = map.as_slice().as_slice_of::<f64>().unwrap();
num_samples += output.len();
num_buffers += 1;
assert_eq!(output.len(), samples_at_eos);
assert_eq!(output.iter().any(|sample| *sample as u16 != 1u16), false);
// All the generated samples should have been processed at this point
assert_eq!(num_samples, EOS_NUM_SAMPLES * EOS_NUM_BUFFERS);
assert_eq!(num_buffers, EOS_NUM_BUFFERS + 1);
}
// In this test, we generate UNDERFLOW_NUM_BUFFERS buffers with UNDERFLOW_NUM_SAMPLES samples each one, however,
// Csound is waiting for UNDERFLOW_NUM_SAMPLES * 2 samples per buffer at its input, so that,
// internally, the output will be only generated when enough data is available.
// It happens, after every 2 * UNDERFLOW_NUM_BUFFERS input buffers, after processing, we should have UNDERFLOW_NUM_BUFFERS/2
// output buffers containing UNDERFLOW_NUM_SAMPLES * 2 samples.
const UNDERFLOW_NUM_BUFFERS: usize = 200;
const UNDERFLOW_NUM_SAMPLES: usize = 2;
#[test]
fn csound_filter_underflow() {
init();
let ksmps: usize = UNDERFLOW_NUM_SAMPLES * 2;
let num_channels = 1;
let sr: i32 = 44_100;
let caps = gst::Caps::new_simple(
"audio/x-raw",
&[
("format", &gst_audio::AUDIO_FORMAT_F64.to_str()),
("rate", &sr),
("channels", &num_channels),
("layout", &"interleaved"),
],
);
let mut h = build_harness(
caps.clone(),
caps,
&CSD!(ksmps, num_channels, num_channels, "ain in", "out ain"),
);
h.play();
// Input buffers timestamp
let mut in_pts = gst::ClockTime(Some(0));
let in_samples_duration = duration_from_samples(UNDERFLOW_NUM_SAMPLES as _, sr as _);
for _ in 0..UNDERFLOW_NUM_BUFFERS {
let mut buffer =
gst::Buffer::with_size(UNDERFLOW_NUM_SAMPLES * std::mem::size_of::<f64>()).unwrap();
buffer.make_mut().set_pts(in_pts);
buffer.make_mut().set_duration(in_samples_duration);
in_pts += in_samples_duration;
assert!(h.push(buffer).is_ok());
}
h.push_event(gst::Event::new_eos().build());
// From here we check our output data
let mut num_buffers = 0;
let mut num_samples = 0;
let expected_duration = duration_from_samples(UNDERFLOW_NUM_SAMPLES as u64 * 2, sr as _);
let expected_buffers = UNDERFLOW_NUM_BUFFERS / 2;
let mut expected_pts = gst::ClockTime(Some(0));
for _ in 0..expected_buffers {
let buffer = h.pull().unwrap();
let samples = buffer.get_size() / std::mem::size_of::<f64>();
assert_eq!(buffer.as_ref().get_pts(), expected_pts);
assert_eq!(buffer.as_ref().get_duration(), expected_duration);
assert_eq!(samples, UNDERFLOW_NUM_SAMPLES * 2);
// Output data is produced after 2 input buffers
// so that, the next output buffer's PTS should be
// equal to the last PTS plus the duration of 2 input buffers
expected_pts += in_samples_duration * 2;
num_buffers += 1;
num_samples += samples;
}
assert_eq!(num_buffers, UNDERFLOW_NUM_BUFFERS / 2);
assert_eq!(
num_samples as usize,
UNDERFLOW_NUM_SAMPLES * UNDERFLOW_NUM_BUFFERS
);
}
// Verifies that the caps negotiation is properly done, by pushing buffers whose caps
// are the same as the one configured in csound, into the harness sink pad. Csoundfilter is expecting 2 channels audio
// at a sample rate of 44100.
// the output caps configured in the harness are not fixated but when the caps negotiation ends,
// those caps must be fixated according to the csound output format which is defined once the csd file is compiled
#[test]
fn csound_filter_caps_negotiation() {
init();
let ksmps = 4;
let ichannels = 2;
let ochannels = 1;
let sr: i32 = 44_100;
let src_caps = gst::Caps::new_simple(
"audio/x-raw",
&[
("format", &gst_audio::AUDIO_FORMAT_F64.to_str()),
("rate", &sr),
("channels", &ichannels),
("layout", &"interleaved"),
],
);
// Define the output caps which would be fixated
// at the end of the caps negotiation
let sink_caps = gst::Caps::new_simple(
"audio/x-raw",
&[
("format", &gst_audio::AUDIO_FORMAT_F64.to_str()),
("rate", &gst::IntRange::<i32>::new(1, 48000)),
("channels", &gst::IntRange::<i32>::new(1, 2)),
("layout", &"interleaved"),
],
);
// build the harness setting its src and sink caps,
// also passing the csd score to the filter element
let mut h = build_harness(
src_caps,
sink_caps.clone(),
// creates a csd score that defines the input and output formats on the csound side
// the output fomart would be 1 channel audio samples at 44100
&CSD!(ksmps, ichannels, ochannels, "ain, ain2 ins", "out ain"),
);
h.play();
assert!(h.push(gst::Buffer::with_size(2048).unwrap()).is_ok());
h.push_event(gst::Event::new_eos().build());
let buffer = h.pull().unwrap();
// Pushing a buffer without a timestamp should produce a no timestamp output
assert!(buffer.as_ref().get_pts().is_none());
// But It should have a duration
assert_eq!(
buffer.as_ref().get_duration(),
duration_from_samples(1024 / std::mem::size_of::<f64>() as u64, sr as u64)
);
// get the negotiated harness sink caps
let harness_sink_caps = h
.get_sinkpad()
.expect("harness has no sinkpad")
.get_current_caps()
.expect("pad has no caps");
// our expected caps at the harness sinkpad
let expected_caps = gst::Caps::new_simple(
"audio/x-raw",
&[
("format", &gst_audio::AUDIO_FORMAT_F64.to_str()),
("rate", &44_100i32),
("channels", &ochannels),
("layout", &"interleaved"),
],
);
assert_eq!(harness_sink_caps, expected_caps);
}
// Similar to caps negotiation, but in this case, we configure a fixated caps in the harness sinkpad,
// such caps are incompatible with the csoundfilter and it leads to an error during the caps negotiation,
// because there is not a common intersection between both caps.
#[test]
fn csound_filter_caps_negotiation_fail() {
init();
let ksmps = 4;
let ichannels = 2;
let ochannels = 1;
let src_caps = gst::Caps::new_simple(
"audio/x-raw",
&[
("format", &gst_audio::AUDIO_FORMAT_F64.to_str()),
("rate", &44_100i32),
("channels", &ichannels),
("layout", &"interleaved"),
],
);
// instead of having a range for channels/rate fields
// we fixate them to 2 and 48_000 respectively, which would cause the negotiation error
let sink_caps = gst::Caps::new_simple(
"audio/x-raw",
&[
("format", &gst_audio::AUDIO_FORMAT_F64.to_str()),
("rate", &48_000i32),
("channels", &ichannels),
("layout", &"interleaved"),
],
);
let mut h = build_harness(
src_caps,
sink_caps,
// creates a csd score that defines the input and output formats on the csound side
// the output fomart would be 1 channel audio samples at 44100
&CSD!(ksmps, ichannels, ochannels, "ain, ain2 ins", "out ain"),
);
h.play();
let buffer = gst::Buffer::with_size(2048).unwrap();
assert!(h.push(buffer).is_err());
h.push_event(gst::Event::new_eos().build());
// The harness sinkpad end up not having defined caps
// so, the get_current_caps should be None
let current_caps = h
.get_sinkpad()
.expect("harness has no sinkpad")
.get_current_caps();
assert!(current_caps.is_none());
}

23
meson.build
View file

@ -1,5 +1,6 @@
project('gst-plugins-rs',
'rust',
'c',
version: '0.13.0',
meson_version : '>= 0.52')
@ -59,6 +60,28 @@ else
exclude += ['gst-plugin-sodium']
endif
cc = meson.get_compiler('c')
csound_option = get_option('csound')
csound_dep = dependency('', required: false) # not-found dependency
if not csound_option.disabled()
csound_dep = cc.find_library('csound64', required: false)
if not csound_dep.found()
python3 = import('python').find_installation('python3')
res = run_command(python3, '-c', 'import os; print(os.environ["CSOUND_LIB_DIR"])')
if res.returncode() == 0
csound_dep = cc.find_library('csound64', dirs: res.stdout(), required: csound_option)
elif csound_option.enabled()
error('csound option is enabled, but csound64 library could not be found and CSOUND_LIB_DIR was not set')
endif
endif
endif
if csound_dep.found()
plugins_rep += {'gst-plugin-csound' : 'libgstcsound'}
else
exclude += ['gst-plugin-csound']
endif
output = []
foreach p, lib : plugins_rep

1
meson_options.txt
View file

@ -2,3 +2,4 @@ option('dav1d', type : 'feature', value : 'auto', description : 'Build dav1d plu
option('sodium', type : 'combo',
choices : ['system', 'built-in', 'disabled'], value : 'built-in',
description : 'Weither to use libsodium from the system or the built-in version from the sodiumoxide crate')
option('csound', type : 'feature', value : 'auto', description : 'Build csound plugin')