diff --git a/Cargo.lock b/Cargo.lock index 215aaefa..7c7988da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2976,6 +2976,7 @@ dependencies = [ "gstreamer", "gstreamer-base", "librespot-core", + "librespot-metadata", "librespot-playback", "tokio", "url", diff --git a/audio/spotify/Cargo.toml b/audio/spotify/Cargo.toml index 0a2f0ca1..03d01c72 100644 --- a/audio/spotify/Cargo.toml +++ b/audio/spotify/Cargo.toml @@ -12,6 +12,7 @@ rust-version.workspace = true gst.workspace = true gst-base.workspace = true librespot-core = "0.5" +librespot-metadata = "0.5" librespot-playback = { version = "0.5", features = ['passthrough-decoder'] } tokio = { version = "1.0", features = ["rt-multi-thread"] } futures = "0.3" diff --git a/audio/spotify/README.md b/audio/spotify/README.md index 2e364926..3c02be92 100644 --- a/audio/spotify/README.md +++ b/audio/spotify/README.md @@ -41,4 +41,12 @@ The element also implements an URI handler which accepts credentials and cache s ```console gst-launch-1.0 playbin3 uri=spotify:track:3i3P1mGpV9eRlfKccjDjwi?access-token=$ACCESS_TOKEN\&cache-credentials=cache\&cache-files=cache +``` + +## spotifylyricssrc + +The `spotifylyricssrc` element can be used to retrieve the lyrics of a song from Spotify. + +``` +gst-launch-1.0 spotifylyricssrc access-token=$ACCESS_TOKEN track=spotify:track:3yMFBuIdPBdJkkzaPBDjKY ! txt. videotestsrc pattern=black ! video/x-raw,width=1920,height=1080 ! textoverlay name=txt shaded-background=yes valignment=center halignment=center ! autovideosink ``` \ No newline at end of file diff --git a/audio/spotify/src/lib.rs b/audio/spotify/src/lib.rs index f650ab78..7f1c0521 100644 --- a/audio/spotify/src/lib.rs +++ b/audio/spotify/src/lib.rs @@ -16,9 +16,11 @@ use gst::glib; mod common; mod spotifyaudiosrc; +mod spotifylyricssrc; fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { spotifyaudiosrc::register(plugin)?; + spotifylyricssrc::register(plugin)?; Ok(()) } diff --git a/audio/spotify/src/spotifylyricssrc/imp.rs b/audio/spotify/src/spotifylyricssrc/imp.rs new file mode 100644 index 00000000..8100045d --- /dev/null +++ b/audio/spotify/src/spotifylyricssrc/imp.rs @@ -0,0 +1,395 @@ +// Copyright (C) 2021-2024 Guillaume Desmottes +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use std::sync::{Arc, LazyLock, Mutex}; + +use futures::future::{AbortHandle, Abortable}; +use tokio::runtime; + +use gst::glib; +use gst::subclass::prelude::*; +use gst_base::prelude::*; +use gst_base::subclass::{base_src::CreateSuccess, prelude::*}; + +use librespot_metadata::lyrics; + +use crate::common::SetupThread; + +static CAT: LazyLock = LazyLock::new(|| { + gst::DebugCategory::new( + "spotifylyricssrc", + gst::DebugColorFlags::empty(), + Some("Spotify lyrics source"), + ) +}); + +static RUNTIME: LazyLock = LazyLock::new(|| { + runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(1) + .build() + .unwrap() +}); + +#[derive(Default, Debug)] +struct Settings { + common: crate::common::Settings, + background_color: u32, + highlight_text_color: u32, + text_color: u32, +} + +struct State { + /// pending buffers, in reverse order so pop() produces the next one to push + buffers: Vec, +} + +#[derive(Default)] +pub struct SpotifyLyricsSrc { + setup_thread: Mutex, + state: Arc>>, + settings: Mutex, +} + +#[glib::object_subclass] +impl ObjectSubclass for SpotifyLyricsSrc { + const NAME: &'static str = "GstSpotifyLyricsSrc"; + type Type = super::SpotifyLyricsSrc; + type ParentType = gst_base::PushSrc; +} + +impl ObjectImpl for SpotifyLyricsSrc { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: LazyLock> = LazyLock::new(|| { + let mut props = crate::common::Settings::properties(); + + props.push( + glib::ParamSpecUInt::builder("background-color") + .nick("Background color") + .blurb("The background color of the lyrics, in ARGB") + .default_value(0) + .read_only() + .build(), + ); + + props.push( + glib::ParamSpecUInt::builder("highlight-text-color") + .nick("Highlight Text color") + .blurb("The text color of the highlighted lyrics, in ARGB") + .default_value(0) + .read_only() + .build(), + ); + + props.push( + glib::ParamSpecUInt::builder("text-color") + .nick("Text color") + .blurb("The text color of the lyrics, in ARGB") + .default_value(0) + .read_only() + .build(), + ); + + props + }); + PROPERTIES.as_ref() + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + let mut settings = self.settings.lock().unwrap(); + settings.common.set_property(value, pspec); + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + let settings = self.settings.lock().unwrap(); + + match pspec.name() { + "background-color" => settings.background_color.to_value(), + "highlight-text-color" => settings.highlight_text_color.to_value(), + "text-color" => settings.text_color.to_value(), + _ => settings.common.property(pspec), + } + } + + fn constructed(&self) { + self.obj().set_format(gst::Format::Time); + } +} + +impl GstObjectImpl for SpotifyLyricsSrc {} + +impl ElementImpl for SpotifyLyricsSrc { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: LazyLock = LazyLock::new(|| { + gst::subclass::ElementMetadata::new( + "Spotify lyrics source", + "Source/Text", + "Spotify lyrics source", + "Guillaume Desmottes ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: LazyLock> = LazyLock::new(|| { + let caps = gst::Caps::builder("text/x-raw") + .field("format", "utf8") + .build(); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + + vec![src_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl BaseSrcImpl for SpotifyLyricsSrc { + fn start(&self) -> Result<(), gst::ErrorMessage> { + { + let state = self.state.lock().unwrap(); + if state.is_some() { + // already started + return Ok(()); + } + } + + { + // If not started yet and not cancelled, start the setup + let mut setup_thread = self.setup_thread.lock().unwrap(); + assert!(!matches!(&*setup_thread, SetupThread::Cancelled)); + if matches!(&*setup_thread, SetupThread::None) { + self.start_setup(&mut setup_thread); + } + } + + Ok(()) + } + + fn stop(&self) -> Result<(), gst::ErrorMessage> { + if let Some(_state) = self.state.lock().unwrap().take() { + gst::debug!(CAT, imp = self, "stopping"); + } + + Ok(()) + } + + fn unlock(&self) -> Result<(), gst::ErrorMessage> { + let mut setup_thread = self.setup_thread.lock().unwrap(); + setup_thread.abort(); + Ok(()) + } + + fn unlock_stop(&self) -> Result<(), gst::ErrorMessage> { + let mut setup_thread = self.setup_thread.lock().unwrap(); + if matches!(&*setup_thread, SetupThread::Cancelled) { + *setup_thread = SetupThread::None; + } + Ok(()) + } +} + +impl PushSrcImpl for SpotifyLyricsSrc { + fn create( + &self, + _buffer: Option<&mut gst::BufferRef>, + ) -> Result { + let state_set = { + let state = self.state.lock().unwrap(); + state.is_some() + }; + + if !state_set { + // If not started yet and not cancelled, start the setup + let mut setup_thread = self.setup_thread.lock().unwrap(); + if matches!(&*setup_thread, SetupThread::Cancelled) { + return Err(gst::FlowError::Flushing); + } + + if matches!(&*setup_thread, SetupThread::None) { + self.start_setup(&mut setup_thread); + } + } + + { + // wait for the setup to be completed + let mut setup_thread = self.setup_thread.lock().unwrap(); + if let SetupThread::Pending { + ref mut thread_handle, + .. + } = *setup_thread + { + let thread_handle = thread_handle.take().expect("Waiting multiple times"); + drop(setup_thread); + let res = thread_handle.join().unwrap(); + + match res { + Err(_aborted) => { + gst::debug!(CAT, imp = self, "setup has been cancelled"); + setup_thread = self.setup_thread.lock().unwrap(); + *setup_thread = SetupThread::Cancelled; + return Err(gst::FlowError::Flushing); + } + Ok(Err(err)) => { + gst::error!(CAT, imp = self, "failed to start: {err:?}"); + gst::element_imp_error!(self, gst::ResourceError::Settings, ["{err:?}"]); + setup_thread = self.setup_thread.lock().unwrap(); + *setup_thread = SetupThread::None; + return Err(gst::FlowError::Error); + } + Ok(Ok(_)) => { + setup_thread = self.setup_thread.lock().unwrap(); + *setup_thread = SetupThread::Done; + } + } + } + } + + let mut state = self.state.lock().unwrap(); + let state = state.as_mut().unwrap(); + + match state.buffers.pop() { + Some(buffer) => { + gst::log!(CAT, imp = self, "created {:?}", buffer); + Ok(CreateSuccess::NewBuffer(buffer)) + } + None => { + gst::debug!(CAT, imp = self, "eos"); + Err(gst::FlowError::Eos) + } + } + } +} + +impl SpotifyLyricsSrc { + fn start_setup(&self, setup_thread: &mut SetupThread) { + assert!(matches!(setup_thread, SetupThread::None)); + + let self_ = self.to_owned(); + + // run the runtime from another thread to prevent the "start a runtime from within a runtime" panic + // when the plugin is statically linked. + let (abort_handle, abort_registration) = AbortHandle::new_pair(); + let thread_handle = std::thread::spawn(move || { + RUNTIME.block_on(async move { + let future = Abortable::new(self_.setup(), abort_registration); + future.await + }) + }); + + *setup_thread = SetupThread::Pending { + thread_handle: Some(thread_handle), + abort_handle, + }; + } + async fn setup(&self) -> anyhow::Result<()> { + { + let state = self.state.lock().unwrap(); + + if state.is_some() { + // already setup + return Ok(()); + } + } + + let src = self.obj(); + + let (session, track_id) = { + let common = { + let settings = self.settings.lock().unwrap(); + settings.common.clone() + }; + + let session = common.connect_session(src.clone(), &CAT).await?; + let track_id = common.track_id()?; + + (session, track_id) + }; + + let reply = lyrics::Lyrics::get(&session, &track_id) + .await + .map_err(|e| anyhow::anyhow!("failed to get lyrics for track {} ({})", track_id, e))?; + + gst::debug!(CAT, imp = self, "got lyrics for track {}", track_id); + + match reply.lyrics.sync_type { + lyrics::SyncType::Unsynced => { + // lyrics are not synced so we can't generate timestamps + anyhow::bail!("lyrics are not synced") + } + lyrics::SyncType::LineSynced => {} + } + + let mut buffers = vec![]; + + // create all pending buffers, in reverse order. + let mut lines = reply.lyrics.lines.into_iter().rev(); + + // The last line is always empty as it's meant to calculate the last actual line duration, + // so we can safely ignore it. + if let Some(last_line) = lines.next() { + // ts of the next buffer in chronological order, so the previous one when iterating + let mut next_ts = start_time(&last_line)?; + + for line in lines { + let ts = start_time(&line)?; + // skip consecutive empty lines + if !line.words.is_empty() { + let txt = line.words; + let duration = next_ts - ts; + + gst::trace!(CAT, imp = self, "{}: {} (duration: {})", ts, txt, duration); + + let mut buffer = gst::Buffer::from_slice(txt); + { + let buffer = buffer.get_mut().unwrap(); + + buffer.set_pts(ts); + buffer.set_duration(duration); + } + + buffers.push(buffer); + next_ts = ts; + } + } + } else { + // no lyrics + } + + // update color properties + { + let mut settings = self.settings.lock().unwrap(); + settings.background_color = reply.colors.background as u32; + settings.highlight_text_color = reply.colors.highlight_text as u32; + settings.text_color = reply.colors.text as u32; + } + self.obj().notify("background-color"); + self.obj().notify("highlight-text-color"); + self.obj().notify("text-color"); + + let mut state = self.state.lock().unwrap(); + state.replace(State { buffers }); + + Ok(()) + } +} + +fn start_time(line: &lyrics::Line) -> anyhow::Result { + let ms = line.start_time_ms.parse()?; + let ts = gst::ClockTime::from_mseconds(ms); + Ok(ts) +} diff --git a/audio/spotify/src/spotifylyricssrc/mod.rs b/audio/spotify/src/spotifylyricssrc/mod.rs new file mode 100644 index 00000000..15a73034 --- /dev/null +++ b/audio/spotify/src/spotifylyricssrc/mod.rs @@ -0,0 +1,25 @@ +// Copyright (C) 2021-2024 Guillaume Desmottes +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; + +mod imp; + +glib::wrapper! { + pub struct SpotifyLyricsSrc(ObjectSubclass) @extends gst_base::PushSrc, gst_base::BaseSrc, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "spotifylyricssrc", + gst::Rank::PRIMARY, + SpotifyLyricsSrc::static_type(), + ) +} diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 5409c707..0bf2e55c 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -12209,6 +12209,134 @@ } }, "rank": "primary" + }, + "spotifylyricssrc": { + "author": "Guillaume Desmottes ", + "description": "Spotify lyrics source", + "hierarchy": [ + "GstSpotifyLyricsSrc", + "GstPushSrc", + "GstBaseSrc", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Source/Text", + "pad-templates": { + "src": { + "caps": "text/x-raw:\n format: utf8\n", + "direction": "src", + "presence": "always" + } + }, + "properties": { + "access-token": { + "blurb": "Spotify access token, requires 'streaming' scope", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "", + "mutable": "ready", + "readable": true, + "type": "gchararray", + "writable": true + }, + "background-color": { + "blurb": "The background color of the lyrics, in ARGB", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "-1", + "min": "0", + "mutable": "null", + "readable": true, + "type": "guint", + "writable": false + }, + "cache-credentials": { + "blurb": "Directory where to cache Spotify credentials", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "", + "mutable": "ready", + "readable": true, + "type": "gchararray", + "writable": true + }, + "cache-files": { + "blurb": "Directory where to cache downloaded files from Spotify", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "", + "mutable": "ready", + "readable": true, + "type": "gchararray", + "writable": true + }, + "cache-max-size": { + "blurb": "The max allowed size of the cache, in bytes, or 0 to disable the cache limit", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "18446744073709551615", + "min": "0", + "mutable": "ready", + "readable": true, + "type": "guint64", + "writable": true + }, + "highlight-text-color": { + "blurb": "The text color of the highlighted lyrics, in ARGB", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "-1", + "min": "0", + "mutable": "null", + "readable": true, + "type": "guint", + "writable": false + }, + "text-color": { + "blurb": "The text color of the lyrics, in ARGB", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "-1", + "min": "0", + "mutable": "null", + "readable": true, + "type": "guint", + "writable": false + }, + "track": { + "blurb": "Spotify track URI, in the form 'spotify:track:$SPOTIFY_ID'", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "", + "mutable": "ready", + "readable": true, + "type": "gchararray", + "writable": true + } + }, + "rank": "primary" } }, "filename": "gstspotify",