2020-04-10 17:52:28 +00:00
|
|
|
// Copyright (C) 2020 Mathieu Duponchelle <mathieu@centricular.com>
|
2023-02-24 20:40:54 +00:00
|
|
|
// Copyright (C) 2023 François Laignel <francois@centricular.com>
|
2020-04-10 17:52:28 +00:00
|
|
|
//
|
2022-01-15 18:40:12 +00:00
|
|
|
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
|
|
|
|
// If a copy of the MPL was not distributed with this file, You can obtain one at
|
|
|
|
// <https://mozilla.org/MPL/2.0/>.
|
2020-04-10 17:52:28 +00:00
|
|
|
//
|
2022-01-15 18:40:12 +00:00
|
|
|
// SPDX-License-Identifier: MPL-2.0
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
//! AWS Transcriber element.
|
|
|
|
//!
|
|
|
|
//! This element calls AWS Transcribe to extract transcripts from an audio stream.
|
|
|
|
//! The element can optionally translate the resulting transcripts to one or
|
|
|
|
//! multiple languages.
|
|
|
|
//!
|
2023-03-15 13:59:50 +00:00
|
|
|
//! This module contains the element implementation as well as the `TranslateSrcPad`
|
2023-03-10 13:47:38 +00:00
|
|
|
//! sublcass and its `TranslationPadTask`.
|
|
|
|
//!
|
|
|
|
//! Web service specific code can be found in the `transcribe` and `translate` modules.
|
|
|
|
|
2020-04-09 22:51:03 +00:00
|
|
|
use gst::subclass::prelude::*;
|
2023-03-10 13:47:38 +00:00
|
|
|
use gst::{glib, prelude::*};
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2023-02-24 20:40:54 +00:00
|
|
|
use aws_sdk_transcribestreaming as aws_transcribe;
|
2020-04-09 22:51:03 +00:00
|
|
|
|
|
|
|
use futures::channel::mpsc;
|
2023-03-09 10:50:50 +00:00
|
|
|
use futures::future::AbortHandle;
|
2020-04-09 22:51:03 +00:00
|
|
|
use futures::prelude::*;
|
2023-03-10 13:47:38 +00:00
|
|
|
use tokio::{runtime, sync::broadcast, task};
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
use std::collections::{BTreeSet, VecDeque};
|
2023-03-16 17:20:08 +00:00
|
|
|
use std::sync::{Arc, Mutex};
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2020-04-10 17:52:28 +00:00
|
|
|
use once_cell::sync::Lazy;
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
use super::transcribe::{TranscriberSettings, TranscriberStream, TranscriptEvent, TranscriptItem};
|
|
|
|
use super::translate::{TranslateLoop, TranslatedItem};
|
2023-03-11 15:27:51 +00:00
|
|
|
use super::{
|
|
|
|
AwsTranscriberResultStability, AwsTranscriberVocabularyFilterMethod,
|
|
|
|
TranslationTokenizationMethod, CAT,
|
|
|
|
};
|
2020-04-10 17:52:28 +00:00
|
|
|
|
|
|
|
static RUNTIME: Lazy<runtime::Runtime> = Lazy::new(|| {
|
2021-01-09 10:14:31 +00:00
|
|
|
runtime::Builder::new_multi_thread()
|
2020-04-09 22:51:03 +00:00
|
|
|
.enable_all()
|
|
|
|
.build()
|
2020-04-10 17:52:28 +00:00
|
|
|
.unwrap()
|
|
|
|
});
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2023-02-24 20:40:54 +00:00
|
|
|
const DEFAULT_TRANSCRIBER_REGION: &str = "us-east-1";
|
2023-03-10 13:47:38 +00:00
|
|
|
|
|
|
|
// Deprecated in 0.11.0: due to evolutions of the transcriber element,
|
|
|
|
// this property has been replaced by `TRANSCRIBE_LATENCY_PROPERTY`.
|
|
|
|
const DEPRECATED_LATENCY_PROPERTY: &str = "latency";
|
|
|
|
|
|
|
|
const TRANSCRIBE_LATENCY_PROPERTY: &str = "transcribe-latency";
|
|
|
|
pub const DEFAULT_TRANSCRIBE_LATENCY: gst::ClockTime = gst::ClockTime::from_seconds(8);
|
|
|
|
|
|
|
|
const TRANSLATE_LATENCY_PROPERTY: &str = "translate-latency";
|
|
|
|
pub const DEFAULT_TRANSLATE_LATENCY: gst::ClockTime = gst::ClockTime::from_mseconds(500);
|
|
|
|
|
2023-03-15 13:59:50 +00:00
|
|
|
const TRANSLATE_LOOKAHEAD_PROPERTY: &str = "translate-lookahead";
|
2023-03-16 09:29:24 +00:00
|
|
|
pub const DEFAULT_TRANSLATE_LOOKAHEAD: gst::ClockTime = gst::ClockTime::from_seconds(3);
|
2023-03-10 13:47:38 +00:00
|
|
|
|
2022-10-17 17:48:43 +00:00
|
|
|
const DEFAULT_LATENESS: gst::ClockTime = gst::ClockTime::ZERO;
|
2023-03-10 13:47:38 +00:00
|
|
|
pub const DEFAULT_INPUT_LANG_CODE: &str = "en-US";
|
|
|
|
|
2021-06-19 01:27:42 +00:00
|
|
|
const DEFAULT_STABILITY: AwsTranscriberResultStability = AwsTranscriberResultStability::Low;
|
2022-08-03 22:42:50 +00:00
|
|
|
const DEFAULT_VOCABULARY_FILTER_METHOD: AwsTranscriberVocabularyFilterMethod =
|
|
|
|
AwsTranscriberVocabularyFilterMethod::Mask;
|
2023-03-10 13:47:38 +00:00
|
|
|
|
|
|
|
// The period at which the event loops will check if they need to push
|
|
|
|
// anything downstream when no other events show up.
|
|
|
|
pub const GRANULARITY: gst::ClockTime = gst::ClockTime::from_mseconds(100);
|
|
|
|
|
|
|
|
const OUTPUT_LANG_CODE_PROPERTY: &str = "language-code";
|
|
|
|
const DEFAULT_OUTPUT_LANG_CODE: Option<&str> = None;
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2023-03-11 15:27:51 +00:00
|
|
|
const TRANSLATION_TOKENIZATION_PROPERTY: &str = "tokenization-method";
|
|
|
|
|
2020-04-09 22:51:03 +00:00
|
|
|
#[derive(Debug, Clone)]
|
2023-03-10 13:47:38 +00:00
|
|
|
pub(super) struct Settings {
|
|
|
|
transcribe_latency: gst::ClockTime,
|
|
|
|
translate_latency: gst::ClockTime,
|
2023-03-15 13:59:50 +00:00
|
|
|
translate_lookahead: gst::ClockTime,
|
2021-07-27 00:36:59 +00:00
|
|
|
lateness: gst::ClockTime,
|
2023-03-10 13:47:38 +00:00
|
|
|
pub language_code: String,
|
|
|
|
pub vocabulary: Option<String>,
|
|
|
|
pub session_id: Option<String>,
|
|
|
|
pub results_stability: AwsTranscriberResultStability,
|
2021-09-27 13:49:12 +00:00
|
|
|
access_key: Option<String>,
|
|
|
|
secret_access_key: Option<String>,
|
2022-06-16 07:16:28 +00:00
|
|
|
session_token: Option<String>,
|
2023-03-10 13:47:38 +00:00
|
|
|
pub vocabulary_filter: Option<String>,
|
|
|
|
pub vocabulary_filter_method: AwsTranscriberVocabularyFilterMethod,
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
|
2023-02-28 15:28:13 +00:00
|
|
|
impl Default for Settings {
|
2020-04-09 22:51:03 +00:00
|
|
|
fn default() -> Self {
|
|
|
|
Self {
|
2023-03-10 13:47:38 +00:00
|
|
|
transcribe_latency: DEFAULT_TRANSCRIBE_LATENCY,
|
|
|
|
translate_latency: DEFAULT_TRANSLATE_LATENCY,
|
2023-03-15 13:59:50 +00:00
|
|
|
translate_lookahead: DEFAULT_TRANSLATE_LOOKAHEAD,
|
2021-07-27 00:36:59 +00:00
|
|
|
lateness: DEFAULT_LATENESS,
|
2023-03-10 13:47:38 +00:00
|
|
|
language_code: DEFAULT_INPUT_LANG_CODE.to_string(),
|
2021-02-19 20:54:08 +00:00
|
|
|
vocabulary: None,
|
2021-06-16 22:53:04 +00:00
|
|
|
session_id: None,
|
2021-06-19 01:27:42 +00:00
|
|
|
results_stability: DEFAULT_STABILITY,
|
2021-09-27 13:49:12 +00:00
|
|
|
access_key: None,
|
|
|
|
secret_access_key: None,
|
2022-06-16 07:16:28 +00:00
|
|
|
session_token: None,
|
2022-08-03 22:42:50 +00:00
|
|
|
vocabulary_filter: None,
|
|
|
|
vocabulary_filter_method: DEFAULT_VOCABULARY_FILTER_METHOD,
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-16 11:17:38 +00:00
|
|
|
#[derive(Clone, Debug, Default)]
|
|
|
|
struct OutputItem {
|
|
|
|
pts: gst::ClockTime,
|
|
|
|
duration: gst::ClockTime,
|
|
|
|
content: String,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<&TranscriptItem> for OutputItem {
|
|
|
|
fn from(item: &TranscriptItem) -> Self {
|
|
|
|
OutputItem {
|
|
|
|
pts: item.pts,
|
|
|
|
duration: item.duration,
|
|
|
|
content: item.content.clone(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<TranslatedItem> for OutputItem {
|
|
|
|
fn from(item: TranslatedItem) -> Self {
|
|
|
|
OutputItem {
|
|
|
|
pts: item.pts,
|
|
|
|
duration: item.duration,
|
|
|
|
content: item.content,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-16 09:29:24 +00:00
|
|
|
struct State {
|
2023-02-24 20:40:54 +00:00
|
|
|
buffer_tx: Option<mpsc::Sender<gst::Buffer>>,
|
2023-03-10 13:47:38 +00:00
|
|
|
transcriber_loop_handle: Option<task::JoinHandle<Result<(), gst::ErrorMessage>>>,
|
2023-03-15 13:59:50 +00:00
|
|
|
srcpads: BTreeSet<super::TranslateSrcPad>,
|
2023-03-10 13:47:38 +00:00
|
|
|
pad_serial: u32,
|
2023-03-16 09:29:24 +00:00
|
|
|
seqnum: gst::Seqnum,
|
2023-03-16 17:20:08 +00:00
|
|
|
start_time: Option<gst::ClockTime>,
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
|
2023-02-28 15:28:13 +00:00
|
|
|
impl Default for State {
|
2020-04-09 22:51:03 +00:00
|
|
|
fn default() -> Self {
|
|
|
|
Self {
|
2023-02-24 20:40:54 +00:00
|
|
|
buffer_tx: None,
|
2023-03-10 13:47:38 +00:00
|
|
|
transcriber_loop_handle: None,
|
|
|
|
srcpads: Default::default(),
|
|
|
|
pad_serial: 0,
|
2020-04-09 22:51:03 +00:00
|
|
|
seqnum: gst::Seqnum::next(),
|
2023-03-16 17:20:08 +00:00
|
|
|
start_time: None,
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-11-14 17:24:01 +00:00
|
|
|
pub struct Transcriber {
|
2023-03-15 13:59:50 +00:00
|
|
|
static_srcpad: super::TranslateSrcPad,
|
2020-04-09 22:51:03 +00:00
|
|
|
sinkpad: gst::Pad,
|
|
|
|
settings: Mutex<Settings>,
|
|
|
|
state: Mutex<State>,
|
2023-03-10 13:47:38 +00:00
|
|
|
pub(super) aws_config: Mutex<Option<aws_config::SdkConfig>>,
|
2023-03-16 17:20:08 +00:00
|
|
|
// sender to broadcast transcript items to the src pads for translation.
|
|
|
|
transcript_event_for_translate_tx: broadcast::Sender<TranscriptEvent>,
|
|
|
|
// sender to broadcast transcript items to the src pads, not intended for translation.
|
2023-03-10 13:47:38 +00:00
|
|
|
transcript_event_tx: broadcast::Sender<TranscriptEvent>,
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Transcriber {
|
2023-03-10 13:47:38 +00:00
|
|
|
fn start_srcpad_tasks(&self, state: &State) -> Result<(), gst::LoggableError> {
|
|
|
|
gst::debug!(CAT, imp: self, "Starting tasks");
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
if self.static_srcpad.is_linked() {
|
|
|
|
self.static_srcpad.imp().start_task()?;
|
2021-06-22 18:53:49 +00:00
|
|
|
}
|
2021-06-19 01:27:42 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
for pad in state.srcpads.iter() {
|
|
|
|
pad.imp().start_task()?;
|
|
|
|
}
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
gst::debug!(CAT, imp: self, "Tasks Started");
|
2020-05-28 21:55:00 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
fn stop_tasks(&self, state: &mut State) {
|
|
|
|
gst::debug!(CAT, imp: self, "Stopping tasks");
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
if self.static_srcpad.is_linked() {
|
|
|
|
self.static_srcpad.imp().stop_task();
|
2021-06-22 18:53:49 +00:00
|
|
|
}
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
for pad in state.srcpads.iter() {
|
|
|
|
pad.imp().stop_task();
|
|
|
|
}
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
// Terminate the audio buffer stream
|
|
|
|
state.buffer_tx = None;
|
2020-04-10 17:52:28 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
if let Some(transcriber_loop_handle) = state.transcriber_loop_handle.take() {
|
|
|
|
transcriber_loop_handle.abort();
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
gst::debug!(CAT, imp: self, "Tasks Stopped");
|
|
|
|
}
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
fn sink_event(&self, pad: &gst::Pad, event: gst::Event) -> bool {
|
|
|
|
gst::log!(CAT, obj: pad, "Handling event {event:?}");
|
|
|
|
|
|
|
|
use gst::EventView::*;
|
|
|
|
match event.view() {
|
|
|
|
Eos(_) => {
|
|
|
|
// Terminate the audio buffer stream
|
|
|
|
self.state.lock().unwrap().buffer_tx = None;
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
true
|
|
|
|
}
|
|
|
|
FlushStart(_) => {
|
|
|
|
gst::info!(CAT, imp: self, "Received flush start, disconnecting");
|
|
|
|
let ret = gst::Pad::event_default(pad, Some(&*self.obj()), event);
|
|
|
|
self.stop_tasks(&mut self.state.lock().unwrap());
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
ret
|
|
|
|
}
|
|
|
|
FlushStop(_) => {
|
|
|
|
gst::info!(CAT, imp: self, "Received flush stop, restarting task");
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
if gst::Pad::event_default(pad, Some(&*self.obj()), event) {
|
|
|
|
let state = self.state.lock().unwrap();
|
|
|
|
match self.start_srcpad_tasks(&state) {
|
|
|
|
Err(err) => {
|
|
|
|
gst::error!(CAT, imp: self, "Failed to start srcpad tasks: {err}");
|
|
|
|
false
|
|
|
|
}
|
|
|
|
Ok(_) => true,
|
2021-06-22 18:53:49 +00:00
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
} else {
|
|
|
|
false
|
2021-06-22 18:53:49 +00:00
|
|
|
}
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
Segment(e) => {
|
|
|
|
let format = e.segment().format();
|
|
|
|
if format != gst::Format::Time {
|
|
|
|
gst::element_imp_error!(
|
|
|
|
self,
|
|
|
|
gst::StreamError::Format,
|
|
|
|
["Only Time segments supported, got {format:?}"]
|
|
|
|
);
|
|
|
|
return false;
|
|
|
|
};
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
self.state.lock().unwrap().seqnum = e.seqnum();
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
true
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
Tag(_) => true,
|
|
|
|
Caps(c) => {
|
|
|
|
gst::info!(CAT, "Received caps {c:?}");
|
|
|
|
true
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
StreamStart(_) => true,
|
|
|
|
_ => gst::Pad::event_default(pad, Some(&*self.obj()), event),
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
fn sink_chain(
|
2023-02-28 16:38:20 +00:00
|
|
|
&self,
|
2023-03-10 13:47:38 +00:00
|
|
|
pad: &gst::Pad,
|
|
|
|
buffer: gst::Buffer,
|
|
|
|
) -> Result<gst::FlowSuccess, gst::FlowError> {
|
|
|
|
gst::log!(CAT, obj: pad, "Handling {buffer:?}");
|
2021-06-19 01:27:42 +00:00
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
self.ensure_connection().map_err(|err| {
|
|
|
|
gst::element_imp_error!(self, gst::StreamError::Failed, ["Streaming failed: {err}"]);
|
|
|
|
gst::FlowError::Error
|
|
|
|
})?;
|
2020-05-28 21:55:00 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
let Some(mut buffer_tx) = self.state.lock().unwrap().buffer_tx.take() else {
|
|
|
|
gst::log!(CAT, obj: pad, "Flushing");
|
|
|
|
return Err(gst::FlowError::Flushing);
|
|
|
|
};
|
2021-06-19 01:27:42 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
futures::executor::block_on(buffer_tx.send(buffer)).map_err(|err| {
|
|
|
|
gst::element_imp_error!(self, gst::StreamError::Failed, ["Streaming failed: {err}"]);
|
|
|
|
gst::FlowError::Error
|
|
|
|
})?;
|
2021-06-19 01:27:42 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
self.state.lock().unwrap().buffer_tx = Some(buffer_tx);
|
2023-02-28 15:28:13 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
Ok(gst::FlowSuccess::Ok)
|
|
|
|
}
|
2023-03-16 17:20:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Default)]
|
|
|
|
struct TranslateQueue {
|
|
|
|
items: VecDeque<TranscriptItem>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl TranslateQueue {
|
|
|
|
fn is_empty(&self) -> bool {
|
|
|
|
self.items.is_empty()
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Pushes the provided item.
|
|
|
|
///
|
|
|
|
/// Returns `Some(..)` if items are ready for translation.
|
|
|
|
fn push(&mut self, transcript_item: &TranscriptItem) -> Option<Vec<TranscriptItem>> {
|
|
|
|
// Keep track of the item individually so we can schedule translation precisely.
|
|
|
|
self.items.push_back(transcript_item.clone());
|
|
|
|
|
|
|
|
if transcript_item.is_punctuation {
|
|
|
|
// This makes it a good chunk for translation.
|
|
|
|
// Concatenate as a single item for translation
|
|
|
|
|
|
|
|
return Some(self.items.drain(..).collect());
|
|
|
|
}
|
|
|
|
|
|
|
|
// Regular case: no separator detected, don't push transcript items
|
|
|
|
// to translation now. They will be pushed either if a punctuation
|
|
|
|
// is found or of a `dequeue()` is requested.
|
|
|
|
|
|
|
|
None
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Dequeues items from the specified `deadline` up to `lookahead`.
|
|
|
|
///
|
|
|
|
/// Returns `Some(..)` if some items match the criteria.
|
|
|
|
fn dequeue(
|
|
|
|
&mut self,
|
|
|
|
latency: gst::ClockTime,
|
|
|
|
threshold: gst::ClockTime,
|
|
|
|
lookahead: gst::ClockTime,
|
|
|
|
) -> Option<Vec<TranscriptItem>> {
|
|
|
|
let first_pts = self.items.front()?.pts;
|
|
|
|
if first_pts + latency > threshold {
|
|
|
|
// First item is too early to be sent to translation now
|
|
|
|
// we can wait for more items to accumulate.
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Can't wait any longer to send the first item to translation
|
|
|
|
// Try to get up to lookahead worth of items to improve translation accuracy
|
|
|
|
let limit = first_pts + lookahead;
|
|
|
|
|
|
|
|
let mut items_acc = vec![self.items.pop_front().unwrap()];
|
|
|
|
while let Some(item) = self.items.front() {
|
|
|
|
if item.pts > limit {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
items_acc.push(self.items.pop_front().unwrap());
|
|
|
|
}
|
|
|
|
|
|
|
|
Some(items_acc)
|
|
|
|
}
|
2023-02-28 15:28:13 +00:00
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
fn drain(&mut self) -> impl Iterator<Item = TranscriptItem> + '_ {
|
|
|
|
self.items.drain(..)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Transcriber {
|
|
|
|
fn ensure_connection(&self) -> Result<(), gst::ErrorMessage> {
|
2023-03-10 13:47:38 +00:00
|
|
|
let mut state = self.state.lock().unwrap();
|
2021-06-19 01:27:42 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
if state.buffer_tx.is_some() {
|
2023-03-16 17:20:08 +00:00
|
|
|
return Ok(());
|
2020-05-28 21:55:00 +00:00
|
|
|
}
|
2023-02-28 15:28:13 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
let settings = self.settings.lock().unwrap();
|
2020-05-28 21:55:00 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
let in_caps = self.sinkpad.current_caps().unwrap();
|
|
|
|
let s = in_caps.structure(0).unwrap();
|
|
|
|
let sample_rate = s.get::<i32>("rate").unwrap();
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
let transcription_settings = TranscriberSettings::from(&settings, sample_rate);
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
let (buffer_tx, buffer_rx) = mpsc::channel(1);
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
let _enter = RUNTIME.enter();
|
|
|
|
let mut transcriber_stream = futures::executor::block_on(TranscriberStream::try_new(
|
2023-03-10 13:47:38 +00:00
|
|
|
self,
|
|
|
|
transcription_settings,
|
|
|
|
settings.lateness,
|
|
|
|
buffer_rx,
|
2023-03-16 17:20:08 +00:00
|
|
|
))?;
|
|
|
|
|
|
|
|
// Latency budget for an item to be pushed to stream on time
|
|
|
|
// Margin:
|
|
|
|
// - 2 * GRANULARITY: to make sure we don't push items up to GRANULARITY late.
|
|
|
|
// - 1 * GRANULARITY: extra margin to account for additional overheads.
|
|
|
|
let latency = settings.transcribe_latency.saturating_sub(3 * GRANULARITY);
|
|
|
|
let translate_lookahead = settings.translate_lookahead;
|
|
|
|
let mut translate_queue = TranslateQueue::default();
|
|
|
|
let imp = self.ref_counted();
|
|
|
|
let transcriber_loop_handle = RUNTIME.spawn(async move {
|
|
|
|
loop {
|
|
|
|
// This is to make sure we send items on a timely basis or at least Gap events.
|
|
|
|
let timeout = tokio::time::sleep(GRANULARITY.into()).fuse();
|
|
|
|
futures::pin_mut!(timeout);
|
|
|
|
|
|
|
|
let transcriber_next = transcriber_stream.next().fuse();
|
|
|
|
futures::pin_mut!(transcriber_next);
|
|
|
|
|
|
|
|
// `transcriber_next` takes precedence over `timeout`
|
|
|
|
// because we don't want to loose any incoming items.
|
|
|
|
let res = futures::select_biased! {
|
|
|
|
event = transcriber_next => Some(event?),
|
|
|
|
_ = timeout => None,
|
|
|
|
};
|
|
|
|
|
|
|
|
use TranscriptEvent::*;
|
|
|
|
match res {
|
|
|
|
None => (),
|
|
|
|
Some(Items(items)) => {
|
|
|
|
if imp.transcript_event_tx.receiver_count() > 0 {
|
|
|
|
let _ = imp.transcript_event_tx.send(Items(items.clone()));
|
|
|
|
}
|
|
|
|
|
|
|
|
if imp.transcript_event_for_translate_tx.receiver_count() > 0 {
|
|
|
|
for item in items.iter() {
|
|
|
|
if let Some(items_to_translate) = translate_queue.push(item) {
|
|
|
|
let _ = imp
|
|
|
|
.transcript_event_for_translate_tx
|
|
|
|
.send(Items(items_to_translate.into()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Some(Eos) => {
|
|
|
|
gst::debug!(CAT, imp: imp, "Transcriber loop sending EOS");
|
|
|
|
|
|
|
|
if imp.transcript_event_tx.receiver_count() > 0 {
|
|
|
|
let _ = imp.transcript_event_tx.send(Eos);
|
|
|
|
}
|
|
|
|
|
|
|
|
if imp.transcript_event_for_translate_tx.receiver_count() > 0 {
|
|
|
|
let items_to_translate: Vec<TranscriptItem> =
|
|
|
|
translate_queue.drain().collect();
|
|
|
|
let _ = imp
|
|
|
|
.transcript_event_for_translate_tx
|
|
|
|
.send(Items(items_to_translate.into()));
|
|
|
|
|
|
|
|
let _ = imp.transcript_event_for_translate_tx.send(Eos);
|
|
|
|
}
|
|
|
|
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if imp.transcript_event_for_translate_tx.receiver_count() > 0 {
|
|
|
|
// Check if we need to push items for translation
|
|
|
|
|
|
|
|
let Some((start_time, now)) = imp.get_start_time_and_now() else {
|
|
|
|
continue;
|
|
|
|
};
|
|
|
|
|
|
|
|
if !translate_queue.is_empty() {
|
|
|
|
let threshold = now - start_time;
|
|
|
|
|
|
|
|
if let Some(items_to_translate) =
|
|
|
|
translate_queue.dequeue(latency, threshold, translate_lookahead)
|
|
|
|
{
|
|
|
|
gst::debug!(
|
|
|
|
CAT,
|
|
|
|
imp: imp,
|
|
|
|
"Forcing to translation (threshold {threshold}): {items_to_translate:?}"
|
|
|
|
);
|
|
|
|
let _ = imp
|
|
|
|
.transcript_event_for_translate_tx
|
|
|
|
.send(Items(items_to_translate.into()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
gst::debug!(CAT, imp: imp, "Exiting transcriber loop");
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
});
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
state.transcriber_loop_handle = Some(transcriber_loop_handle);
|
|
|
|
state.buffer_tx = Some(buffer_tx);
|
2023-03-16 17:20:08 +00:00
|
|
|
|
|
|
|
Ok(())
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
fn prepare(&self) -> Result<(), gst::ErrorMessage> {
|
|
|
|
gst::debug!(CAT, imp: self, "Preparing");
|
2023-02-24 20:40:54 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
let (access_key, secret_access_key, session_token);
|
|
|
|
{
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
access_key = settings.access_key.to_owned();
|
|
|
|
secret_access_key = settings.secret_access_key.to_owned();
|
|
|
|
session_token = settings.session_token.to_owned();
|
2021-06-22 18:53:49 +00:00
|
|
|
}
|
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
gst::info!(CAT, imp: self, "Loading aws config...");
|
|
|
|
let _enter_guard = RUNTIME.enter();
|
|
|
|
|
|
|
|
let config_loader = match (access_key, secret_access_key) {
|
|
|
|
(Some(key), Some(secret_key)) => {
|
|
|
|
gst::debug!(CAT, imp: self, "Using settings credentials");
|
|
|
|
aws_config::ConfigLoader::default().credentials_provider(
|
|
|
|
aws_transcribe::Credentials::new(
|
|
|
|
key,
|
|
|
|
secret_key,
|
|
|
|
session_token,
|
|
|
|
None,
|
|
|
|
"translate",
|
|
|
|
),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
_ => {
|
|
|
|
gst::debug!(CAT, imp: self, "Attempting to get credentials from env...");
|
|
|
|
aws_config::from_env()
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
let config_loader = config_loader.region(
|
|
|
|
aws_config::meta::region::RegionProviderChain::default_provider()
|
|
|
|
.or_else(DEFAULT_TRANSCRIBER_REGION),
|
|
|
|
);
|
2023-02-24 20:40:54 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
let config = futures::executor::block_on(config_loader.load());
|
|
|
|
gst::debug!(CAT, imp: self, "Using region {}", config.region().unwrap());
|
2023-02-24 20:40:54 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
*self.aws_config.lock().unwrap() = Some(config);
|
2023-02-24 20:40:54 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
gst::debug!(CAT, imp: self, "Prepared");
|
2023-03-09 10:50:50 +00:00
|
|
|
|
2020-04-09 22:51:03 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
fn disconnect(&self) {
|
|
|
|
gst::info!(CAT, imp: self, "Unpreparing");
|
2023-03-09 10:50:50 +00:00
|
|
|
let mut state = self.state.lock().unwrap();
|
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
self.stop_tasks(&mut state);
|
2023-03-09 10:50:50 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
for pad in state.srcpads.iter() {
|
|
|
|
pad.imp().set_discont();
|
2023-02-24 20:40:54 +00:00
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
gst::info!(CAT, imp: self, "Unprepared");
|
2023-02-24 20:40:54 +00:00
|
|
|
}
|
2023-03-16 17:20:08 +00:00
|
|
|
|
|
|
|
fn get_start_time_and_now(&self) -> Option<(gst::ClockTime, gst::ClockTime)> {
|
|
|
|
let now = self.obj().current_running_time()?;
|
|
|
|
|
|
|
|
let mut state = self.state.lock().unwrap();
|
|
|
|
|
|
|
|
if state.start_time.is_none() {
|
|
|
|
state.start_time = Some(now);
|
|
|
|
}
|
|
|
|
|
|
|
|
Some((state.start_time.unwrap(), now))
|
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
2023-02-24 20:40:54 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
#[glib::object_subclass]
|
|
|
|
impl ObjectSubclass for Transcriber {
|
|
|
|
const NAME: &'static str = "GstAwsTranscriber";
|
|
|
|
type Type = super::Transcriber;
|
|
|
|
type ParentType = gst::Element;
|
|
|
|
type Interfaces = (gst::ChildProxy,);
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2020-11-14 17:24:01 +00:00
|
|
|
fn with_class(klass: &Self::Class) -> Self {
|
2021-04-20 12:58:11 +00:00
|
|
|
let templ = klass.pad_template("sink").unwrap();
|
2020-06-22 08:03:52 +00:00
|
|
|
let sinkpad = gst::Pad::builder_with_template(&templ, Some("sink"))
|
|
|
|
.chain_function(|pad, parent, buffer| {
|
|
|
|
Transcriber::catch_panic_pad_function(
|
|
|
|
parent,
|
|
|
|
|| Err(gst::FlowError::Error),
|
2022-10-09 13:06:59 +00:00
|
|
|
|transcriber| transcriber.sink_chain(pad, buffer),
|
2020-06-22 08:03:52 +00:00
|
|
|
)
|
|
|
|
})
|
|
|
|
.event_function(|pad, parent, event| {
|
|
|
|
Transcriber::catch_panic_pad_function(
|
|
|
|
parent,
|
|
|
|
|| false,
|
2022-10-09 13:06:59 +00:00
|
|
|
|transcriber| transcriber.sink_event(pad, event),
|
2020-06-22 08:03:52 +00:00
|
|
|
)
|
|
|
|
})
|
|
|
|
.build();
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2021-04-20 12:58:11 +00:00
|
|
|
let templ = klass.pad_template("src").unwrap();
|
2023-03-10 13:47:38 +00:00
|
|
|
let static_srcpad =
|
2023-03-15 13:59:50 +00:00
|
|
|
gst::PadBuilder::<super::TranslateSrcPad>::from_template(&templ, Some("src"))
|
2023-03-10 13:47:38 +00:00
|
|
|
.activatemode_function(|pad, parent, mode, active| {
|
|
|
|
Transcriber::catch_panic_pad_function(
|
|
|
|
parent,
|
|
|
|
|| {
|
|
|
|
Err(gst::loggable_error!(
|
|
|
|
CAT,
|
2023-03-15 13:59:50 +00:00
|
|
|
"Panic activating TranslateSrcPad"
|
2023-03-10 13:47:38 +00:00
|
|
|
))
|
|
|
|
},
|
2023-03-15 13:59:50 +00:00
|
|
|
|elem| TranslateSrcPad::activatemode(elem, pad, mode, active),
|
2023-03-10 13:47:38 +00:00
|
|
|
)
|
|
|
|
})
|
|
|
|
.query_function(|pad, parent, query| {
|
|
|
|
Transcriber::catch_panic_pad_function(
|
|
|
|
parent,
|
|
|
|
|| false,
|
2023-03-15 13:59:50 +00:00
|
|
|
|elem| TranslateSrcPad::src_query(elem, pad, query),
|
2023-03-10 13:47:38 +00:00
|
|
|
)
|
|
|
|
})
|
|
|
|
.flags(gst::PadFlags::FIXED_CAPS)
|
|
|
|
.build();
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2023-03-15 13:59:50 +00:00
|
|
|
// Setting the channel capacity so that a TranslateSrcPad that would lag
|
2023-03-10 13:47:38 +00:00
|
|
|
// behind for some reasons get a chance to catch-up without loosing items.
|
|
|
|
// Receiver will be created by subscribing to sender later.
|
2023-03-16 17:20:08 +00:00
|
|
|
let (transcript_event_for_translate_tx, _) = broadcast::channel(128);
|
2023-03-10 13:47:38 +00:00
|
|
|
let (transcript_event_tx, _) = broadcast::channel(128);
|
2020-04-09 22:51:03 +00:00
|
|
|
|
|
|
|
Self {
|
2023-03-10 13:47:38 +00:00
|
|
|
static_srcpad,
|
2020-04-09 22:51:03 +00:00
|
|
|
sinkpad,
|
2023-03-10 13:47:38 +00:00
|
|
|
settings: Default::default(),
|
|
|
|
state: Default::default(),
|
|
|
|
aws_config: Default::default(),
|
2023-03-16 17:20:08 +00:00
|
|
|
transcript_event_for_translate_tx,
|
2023-03-10 13:47:38 +00:00
|
|
|
transcript_event_tx,
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
}
|
2021-01-21 18:21:29 +00:00
|
|
|
}
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2021-01-21 18:21:29 +00:00
|
|
|
impl ObjectImpl for Transcriber {
|
|
|
|
fn properties() -> &'static [glib::ParamSpec] {
|
|
|
|
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
|
|
|
|
vec![
|
2022-08-18 12:04:15 +00:00
|
|
|
glib::ParamSpecString::builder("language-code")
|
|
|
|
.nick("Language Code")
|
|
|
|
.blurb("The Language of the Stream, see \
|
2021-01-21 18:21:29 +00:00
|
|
|
<https://docs.aws.amazon.com/transcribe/latest/dg/how-streaming-transcription.html> \
|
2022-08-18 12:04:15 +00:00
|
|
|
for an up to date list of allowed languages")
|
2023-03-10 13:47:38 +00:00
|
|
|
.default_value(Some(DEFAULT_INPUT_LANG_CODE))
|
2022-08-18 12:04:15 +00:00
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
2023-03-10 13:47:38 +00:00
|
|
|
glib::ParamSpecUInt::builder(DEPRECATED_LATENCY_PROPERTY)
|
2022-08-18 12:04:15 +00:00
|
|
|
.nick("Latency")
|
2023-03-10 13:47:38 +00:00
|
|
|
.blurb("Amount of milliseconds to allow AWS transcribe (Deprecated. Use transcribe-latency)")
|
|
|
|
.default_value(DEFAULT_TRANSCRIBE_LATENCY.mseconds() as u32)
|
|
|
|
.mutable_ready()
|
|
|
|
.deprecated()
|
|
|
|
.build(),
|
|
|
|
glib::ParamSpecUInt::builder(TRANSCRIBE_LATENCY_PROPERTY)
|
|
|
|
.nick("AWS Transcribe Latency")
|
2022-08-18 12:04:15 +00:00
|
|
|
.blurb("Amount of milliseconds to allow AWS transcribe")
|
2023-03-10 13:47:38 +00:00
|
|
|
.default_value(DEFAULT_TRANSCRIBE_LATENCY.mseconds() as u32)
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
|
|
|
glib::ParamSpecUInt::builder(TRANSLATE_LATENCY_PROPERTY)
|
|
|
|
.nick("AWS Translate Latency")
|
|
|
|
.blurb(concat!(
|
|
|
|
"Amount of milliseconds to allow AWS translate ",
|
|
|
|
"(ignored if the input and output languages are the same)",
|
|
|
|
))
|
|
|
|
.default_value(DEFAULT_TRANSLATE_LATENCY.mseconds() as u32)
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
2023-03-15 13:59:50 +00:00
|
|
|
glib::ParamSpecUInt::builder(TRANSLATE_LOOKAHEAD_PROPERTY)
|
|
|
|
.nick("Translate lookahead")
|
2023-03-10 13:47:38 +00:00
|
|
|
.blurb(concat!(
|
|
|
|
"Maximum duration in milliseconds of transcript to lookahead ",
|
|
|
|
"before sending to translation when no separator was encountered",
|
|
|
|
))
|
2023-03-15 13:59:50 +00:00
|
|
|
.default_value(DEFAULT_TRANSLATE_LOOKAHEAD.mseconds() as u32)
|
2022-08-18 12:04:15 +00:00
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
|
|
|
glib::ParamSpecUInt::builder("lateness")
|
|
|
|
.nick("Lateness")
|
|
|
|
.blurb("Amount of milliseconds to introduce as lateness")
|
|
|
|
.default_value(DEFAULT_LATENESS.mseconds() as u32)
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
|
|
|
glib::ParamSpecString::builder("vocabulary-name")
|
|
|
|
.nick("Vocabulary Name")
|
|
|
|
.blurb("The name of a custom vocabulary, see \
|
2021-02-19 20:54:08 +00:00
|
|
|
<https://docs.aws.amazon.com/transcribe/latest/dg/how-vocabulary.html> \
|
2022-08-18 12:04:15 +00:00
|
|
|
for more information")
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
|
|
|
glib::ParamSpecString::builder("session-id")
|
|
|
|
.nick("Session ID")
|
|
|
|
.blurb("The ID of the transcription session, must be length 36")
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
2023-01-21 16:13:48 +00:00
|
|
|
glib::ParamSpecEnum::builder_with_default("results-stability", DEFAULT_STABILITY)
|
2022-08-18 12:04:15 +00:00
|
|
|
.nick("Results stability")
|
|
|
|
.blurb("Defines how fast results should stabilize")
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
|
|
|
glib::ParamSpecString::builder("access-key")
|
|
|
|
.nick("Access Key")
|
|
|
|
.blurb("AWS Access Key")
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
|
|
|
glib::ParamSpecString::builder("secret-access-key")
|
|
|
|
.nick("Secret Access Key")
|
|
|
|
.blurb("AWS Secret Access Key")
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
|
|
|
glib::ParamSpecString::builder("session-token")
|
|
|
|
.nick("Session Token")
|
|
|
|
.blurb("AWS temporary Session Token from STS")
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
|
|
|
glib::ParamSpecString::builder("vocabulary-filter-name")
|
|
|
|
.nick("Vocabulary Filter Name")
|
|
|
|
.blurb("The name of a custom filter vocabulary, see \
|
2022-08-03 22:42:50 +00:00
|
|
|
<https://docs.aws.amazon.com/transcribe/latest/help-panel/vocab-filter.html> \
|
2022-08-18 12:04:15 +00:00
|
|
|
for more information")
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
2023-01-21 16:13:48 +00:00
|
|
|
glib::ParamSpecEnum::builder_with_default("vocabulary-filter-method", DEFAULT_VOCABULARY_FILTER_METHOD)
|
2022-08-18 12:04:15 +00:00
|
|
|
.nick("Vocabulary Filter Method")
|
|
|
|
.blurb("Defines how filtered words will be edited, has no effect when vocabulary-filter-name isn't set")
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
2021-01-21 18:21:29 +00:00
|
|
|
]
|
|
|
|
});
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2021-01-21 18:21:29 +00:00
|
|
|
PROPERTIES.as_ref()
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
|
2022-10-09 13:06:59 +00:00
|
|
|
fn constructed(&self) {
|
|
|
|
self.parent_constructed();
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2022-10-23 20:03:22 +00:00
|
|
|
let obj = self.obj();
|
2020-11-14 17:24:01 +00:00
|
|
|
obj.add_pad(&self.sinkpad).unwrap();
|
2023-03-10 13:47:38 +00:00
|
|
|
obj.add_pad(&self.static_srcpad).unwrap();
|
2020-11-14 17:24:01 +00:00
|
|
|
obj.set_element_flags(gst::ElementFlags::PROVIDE_CLOCK | gst::ElementFlags::REQUIRE_CLOCK);
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
|
2022-10-09 13:06:59 +00:00
|
|
|
fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) {
|
2021-04-12 12:49:54 +00:00
|
|
|
match pspec.name() {
|
2022-09-08 20:07:49 +00:00
|
|
|
"language-code" => {
|
2020-04-09 22:51:03 +00:00
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.language_code = value.get().expect("type checked upstream");
|
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
DEPRECATED_LATENCY_PROPERTY => {
|
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.transcribe_latency = gst::ClockTime::from_mseconds(
|
|
|
|
value.get::<u32>().expect("type checked upstream").into(),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
TRANSCRIBE_LATENCY_PROPERTY => {
|
2020-04-09 22:51:03 +00:00
|
|
|
let mut settings = self.settings.lock().unwrap();
|
2023-03-10 13:47:38 +00:00
|
|
|
settings.transcribe_latency = gst::ClockTime::from_mseconds(
|
2021-05-26 13:22:01 +00:00
|
|
|
value.get::<u32>().expect("type checked upstream").into(),
|
|
|
|
);
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
TRANSLATE_LATENCY_PROPERTY => {
|
|
|
|
self.settings.lock().unwrap().translate_latency =
|
|
|
|
gst::ClockTime::from_mseconds(value.get::<u32>().unwrap().into());
|
|
|
|
}
|
2023-03-15 13:59:50 +00:00
|
|
|
TRANSLATE_LOOKAHEAD_PROPERTY => {
|
|
|
|
self.settings.lock().unwrap().translate_lookahead =
|
2023-03-10 13:47:38 +00:00
|
|
|
gst::ClockTime::from_mseconds(value.get::<u32>().unwrap().into());
|
|
|
|
}
|
2021-07-27 00:36:59 +00:00
|
|
|
"lateness" => {
|
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.lateness = gst::ClockTime::from_mseconds(
|
|
|
|
value.get::<u32>().expect("type checked upstream").into(),
|
|
|
|
);
|
|
|
|
}
|
2021-02-19 20:54:08 +00:00
|
|
|
"vocabulary-name" => {
|
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.vocabulary = value.get().expect("type checked upstream");
|
|
|
|
}
|
2021-06-16 22:53:04 +00:00
|
|
|
"session-id" => {
|
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.session_id = value.get().expect("type checked upstream");
|
|
|
|
}
|
2021-06-19 01:27:42 +00:00
|
|
|
"results-stability" => {
|
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.results_stability = value
|
|
|
|
.get::<AwsTranscriberResultStability>()
|
|
|
|
.expect("type checked upstream");
|
|
|
|
}
|
2021-09-27 13:49:12 +00:00
|
|
|
"access-key" => {
|
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.access_key = value.get().expect("type checked upstream");
|
|
|
|
}
|
|
|
|
"secret-access-key" => {
|
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.secret_access_key = value.get().expect("type checked upstream");
|
|
|
|
}
|
2022-06-16 07:16:28 +00:00
|
|
|
"session-token" => {
|
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.session_token = value.get().expect("type checked upstream");
|
|
|
|
}
|
2022-08-03 22:42:50 +00:00
|
|
|
"vocabulary-filter-name" => {
|
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.vocabulary_filter = value.get().expect("type checked upstream");
|
|
|
|
}
|
|
|
|
"vocabulary-filter-method" => {
|
|
|
|
let mut settings = self.settings.lock().unwrap();
|
|
|
|
settings.vocabulary_filter_method = value
|
|
|
|
.get::<AwsTranscriberVocabularyFilterMethod>()
|
|
|
|
.expect("type checked upstream");
|
|
|
|
}
|
2020-04-09 22:51:03 +00:00
|
|
|
_ => unimplemented!(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-10-09 13:06:59 +00:00
|
|
|
fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value {
|
2021-04-12 12:49:54 +00:00
|
|
|
match pspec.name() {
|
2021-01-21 18:21:29 +00:00
|
|
|
"language-code" => {
|
2020-04-09 22:51:03 +00:00
|
|
|
let settings = self.settings.lock().unwrap();
|
2020-11-19 15:55:57 +00:00
|
|
|
settings.language_code.to_value()
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
DEPRECATED_LATENCY_PROPERTY => {
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
(settings.transcribe_latency.mseconds() as u32).to_value()
|
|
|
|
}
|
|
|
|
TRANSCRIBE_LATENCY_PROPERTY => {
|
2020-04-09 22:51:03 +00:00
|
|
|
let settings = self.settings.lock().unwrap();
|
2023-03-10 13:47:38 +00:00
|
|
|
(settings.transcribe_latency.mseconds() as u32).to_value()
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
TRANSLATE_LATENCY_PROPERTY => {
|
|
|
|
(self.settings.lock().unwrap().translate_latency.mseconds() as u32).to_value()
|
|
|
|
}
|
2023-03-15 13:59:50 +00:00
|
|
|
TRANSLATE_LOOKAHEAD_PROPERTY => {
|
|
|
|
(self.settings.lock().unwrap().translate_lookahead.mseconds() as u32).to_value()
|
|
|
|
}
|
2021-07-27 00:36:59 +00:00
|
|
|
"lateness" => {
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
(settings.lateness.mseconds() as u32).to_value()
|
|
|
|
}
|
2021-02-19 20:54:08 +00:00
|
|
|
"vocabulary-name" => {
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
settings.vocabulary.to_value()
|
|
|
|
}
|
2021-06-16 22:53:04 +00:00
|
|
|
"session-id" => {
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
settings.session_id.to_value()
|
|
|
|
}
|
2021-06-19 01:27:42 +00:00
|
|
|
"results-stability" => {
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
settings.results_stability.to_value()
|
|
|
|
}
|
2021-09-27 13:49:12 +00:00
|
|
|
"access-key" => {
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
settings.access_key.to_value()
|
|
|
|
}
|
|
|
|
"secret-access-key" => {
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
settings.secret_access_key.to_value()
|
|
|
|
}
|
2022-06-16 07:16:28 +00:00
|
|
|
"session-token" => {
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
settings.session_token.to_value()
|
|
|
|
}
|
2022-08-03 22:42:50 +00:00
|
|
|
"vocabulary-filter-name" => {
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
settings.vocabulary_filter.to_value()
|
|
|
|
}
|
|
|
|
"vocabulary-filter-method" => {
|
|
|
|
let settings = self.settings.lock().unwrap();
|
|
|
|
settings.vocabulary_filter_method.to_value()
|
|
|
|
}
|
2020-04-09 22:51:03 +00:00
|
|
|
_ => unimplemented!(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-23 08:57:31 +00:00
|
|
|
impl GstObjectImpl for Transcriber {}
|
|
|
|
|
2020-04-09 22:51:03 +00:00
|
|
|
impl ElementImpl for Transcriber {
|
2021-01-21 18:21:29 +00:00
|
|
|
fn metadata() -> Option<&'static gst::subclass::ElementMetadata> {
|
|
|
|
static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| {
|
|
|
|
gst::subclass::ElementMetadata::new(
|
|
|
|
"Transcriber",
|
|
|
|
"Audio/Text/Filter",
|
|
|
|
"Speech to Text filter, using AWS transcribe",
|
2023-02-24 20:40:54 +00:00
|
|
|
"Jordan Petridis <jordan@centricular.com>, Mathieu Duponchelle <mathieu@centricular.com>, François Laignel <francois@centricular.com>",
|
2021-01-21 18:21:29 +00:00
|
|
|
)
|
|
|
|
});
|
|
|
|
|
|
|
|
Some(&*ELEMENT_METADATA)
|
|
|
|
}
|
|
|
|
|
|
|
|
fn pad_templates() -> &'static [gst::PadTemplate] {
|
|
|
|
static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
|
|
|
|
let src_caps = gst::Caps::builder("text/x-raw")
|
2021-11-06 07:34:10 +00:00
|
|
|
.field("format", "utf8")
|
2021-01-21 18:21:29 +00:00
|
|
|
.build();
|
2023-03-10 13:47:38 +00:00
|
|
|
let src_pad_template = gst::PadTemplate::with_gtype(
|
2021-01-21 18:21:29 +00:00
|
|
|
"src",
|
|
|
|
gst::PadDirection::Src,
|
|
|
|
gst::PadPresence::Always,
|
|
|
|
&src_caps,
|
2023-03-15 13:59:50 +00:00
|
|
|
super::TranslateSrcPad::static_type(),
|
2023-03-10 13:47:38 +00:00
|
|
|
)
|
|
|
|
.unwrap();
|
|
|
|
let req_src_pad_template = gst::PadTemplate::with_gtype(
|
2023-03-15 13:59:50 +00:00
|
|
|
"translate_src_%u",
|
2023-03-10 13:47:38 +00:00
|
|
|
gst::PadDirection::Src,
|
|
|
|
gst::PadPresence::Request,
|
|
|
|
&src_caps,
|
2023-03-15 13:59:50 +00:00
|
|
|
super::TranslateSrcPad::static_type(),
|
2021-01-21 18:21:29 +00:00
|
|
|
)
|
|
|
|
.unwrap();
|
|
|
|
|
2022-10-13 18:02:04 +00:00
|
|
|
let sink_caps = gst_audio::AudioCapsBuilder::new()
|
|
|
|
.format(gst_audio::AudioFormat::S16le)
|
|
|
|
.rate_range(8000..=48000)
|
|
|
|
.channels(1)
|
2021-01-21 18:21:29 +00:00
|
|
|
.build();
|
|
|
|
let sink_pad_template = gst::PadTemplate::new(
|
|
|
|
"sink",
|
|
|
|
gst::PadDirection::Sink,
|
|
|
|
gst::PadPresence::Always,
|
|
|
|
&sink_caps,
|
|
|
|
)
|
|
|
|
.unwrap();
|
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
vec![src_pad_template, req_src_pad_template, sink_pad_template]
|
2021-01-21 18:21:29 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
PAD_TEMPLATES.as_ref()
|
|
|
|
}
|
|
|
|
|
2020-04-09 22:51:03 +00:00
|
|
|
fn change_state(
|
|
|
|
&self,
|
|
|
|
transition: gst::StateChange,
|
|
|
|
) -> Result<gst::StateChangeSuccess, gst::StateChangeError> {
|
2023-02-24 20:40:54 +00:00
|
|
|
gst::info!(CAT, imp: self, "Changing state {transition:?}");
|
2020-04-09 22:51:03 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
if let gst::StateChange::NullToReady = transition {
|
|
|
|
self.prepare().map_err(|err| {
|
|
|
|
self.post_error_message(err);
|
|
|
|
gst::StateChangeError
|
|
|
|
})?;
|
|
|
|
}
|
|
|
|
|
2022-10-09 13:06:59 +00:00
|
|
|
let mut success = self.parent_change_state(transition)?;
|
2021-06-22 18:53:49 +00:00
|
|
|
|
2020-04-09 22:51:03 +00:00
|
|
|
match transition {
|
|
|
|
gst::StateChange::PausedToReady => {
|
2022-10-09 13:06:59 +00:00
|
|
|
self.disconnect();
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
|
|
|
gst::StateChange::ReadyToPaused => {
|
|
|
|
success = gst::StateChangeSuccess::NoPreroll;
|
|
|
|
}
|
|
|
|
gst::StateChange::PlayingToPaused => {
|
|
|
|
success = gst::StateChangeSuccess::NoPreroll;
|
|
|
|
}
|
|
|
|
_ => (),
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(success)
|
|
|
|
}
|
2020-06-02 17:31:58 +00:00
|
|
|
|
2023-03-10 13:47:38 +00:00
|
|
|
fn request_new_pad(
|
|
|
|
&self,
|
|
|
|
templ: &gst::PadTemplate,
|
|
|
|
_name: Option<&str>,
|
|
|
|
_caps: Option<&gst::Caps>,
|
|
|
|
) -> Option<gst::Pad> {
|
|
|
|
let mut state = self.state.lock().unwrap();
|
|
|
|
|
2023-03-15 13:59:50 +00:00
|
|
|
let pad = gst::PadBuilder::<super::TranslateSrcPad>::from_template(
|
2023-03-10 13:47:38 +00:00
|
|
|
templ,
|
2023-03-15 13:59:50 +00:00
|
|
|
Some(format!("translate_src_{}", state.pad_serial).as_str()),
|
2023-03-10 13:47:38 +00:00
|
|
|
)
|
|
|
|
.activatemode_function(|pad, parent, mode, active| {
|
|
|
|
Transcriber::catch_panic_pad_function(
|
|
|
|
parent,
|
|
|
|
|| {
|
|
|
|
Err(gst::loggable_error!(
|
|
|
|
CAT,
|
2023-03-15 13:59:50 +00:00
|
|
|
"Panic activating TranslateSrcPad"
|
2023-03-10 13:47:38 +00:00
|
|
|
))
|
|
|
|
},
|
2023-03-15 13:59:50 +00:00
|
|
|
|elem| TranslateSrcPad::activatemode(elem, pad, mode, active),
|
2023-03-10 13:47:38 +00:00
|
|
|
)
|
|
|
|
})
|
|
|
|
.query_function(|pad, parent, query| {
|
|
|
|
Transcriber::catch_panic_pad_function(
|
|
|
|
parent,
|
|
|
|
|| false,
|
2023-03-15 13:59:50 +00:00
|
|
|
|elem| TranslateSrcPad::src_query(elem, pad, query),
|
2023-03-10 13:47:38 +00:00
|
|
|
)
|
|
|
|
})
|
|
|
|
.flags(gst::PadFlags::FIXED_CAPS)
|
|
|
|
.build();
|
|
|
|
|
|
|
|
state.srcpads.insert(pad.clone());
|
|
|
|
|
|
|
|
state.pad_serial += 1;
|
|
|
|
drop(state);
|
|
|
|
|
|
|
|
self.obj().add_pad(&pad).unwrap();
|
|
|
|
|
|
|
|
let _ = self
|
|
|
|
.obj()
|
|
|
|
.post_message(gst::message::Latency::builder().src(&*self.obj()).build());
|
|
|
|
|
|
|
|
self.obj().child_added(&pad, &pad.name());
|
|
|
|
Some(pad.upcast())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn release_pad(&self, pad: &gst::Pad) {
|
|
|
|
pad.set_active(false).unwrap();
|
|
|
|
self.obj().remove_pad(pad).unwrap();
|
|
|
|
|
|
|
|
self.obj().child_removed(pad, &pad.name());
|
|
|
|
let _ = self
|
|
|
|
.obj()
|
|
|
|
.post_message(gst::message::Latency::builder().src(&*self.obj()).build());
|
|
|
|
}
|
|
|
|
|
2022-10-09 13:06:59 +00:00
|
|
|
fn provide_clock(&self) -> Option<gst::Clock> {
|
2020-06-02 17:31:58 +00:00
|
|
|
Some(gst::SystemClock::obtain())
|
|
|
|
}
|
2020-04-09 22:51:03 +00:00
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
|
|
|
|
// Implementation of gst::ChildProxy virtual methods.
|
|
|
|
//
|
|
|
|
// This allows accessing the pads and their properties from e.g. gst-launch.
|
|
|
|
impl ChildProxyImpl for Transcriber {
|
|
|
|
fn children_count(&self) -> u32 {
|
|
|
|
let object = self.obj();
|
|
|
|
object.num_pads() as u32
|
|
|
|
}
|
|
|
|
|
|
|
|
fn child_by_name(&self, name: &str) -> Option<glib::Object> {
|
|
|
|
let object = self.obj();
|
|
|
|
object
|
|
|
|
.pads()
|
|
|
|
.into_iter()
|
|
|
|
.find(|p| p.name() == name)
|
|
|
|
.map(|p| p.upcast())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn child_by_index(&self, index: u32) -> Option<glib::Object> {
|
|
|
|
let object = self.obj();
|
|
|
|
object
|
|
|
|
.pads()
|
|
|
|
.into_iter()
|
|
|
|
.nth(index as usize)
|
|
|
|
.map(|p| p.upcast())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
struct TranslationPadTask {
|
2023-03-15 13:59:50 +00:00
|
|
|
pad: glib::subclass::ObjectImplRef<TranslateSrcPad>,
|
2023-03-10 13:47:38 +00:00
|
|
|
elem: super::Transcriber,
|
|
|
|
transcript_event_rx: broadcast::Receiver<TranscriptEvent>,
|
|
|
|
needs_translate: bool,
|
2023-03-15 13:59:50 +00:00
|
|
|
translate_loop_handle: Option<task::JoinHandle<Result<(), gst::ErrorMessage>>>,
|
2023-03-16 17:20:08 +00:00
|
|
|
to_translate_tx: Option<mpsc::Sender<Arc<Vec<TranscriptItem>>>>,
|
2023-03-15 13:59:50 +00:00
|
|
|
from_translate_rx: Option<mpsc::Receiver<Vec<TranslatedItem>>>,
|
2023-03-10 13:47:38 +00:00
|
|
|
send_events: bool,
|
2023-03-16 11:17:38 +00:00
|
|
|
output_items: VecDeque<OutputItem>,
|
2023-03-10 13:47:38 +00:00
|
|
|
our_latency: gst::ClockTime,
|
|
|
|
seqnum: gst::Seqnum,
|
|
|
|
send_eos: bool,
|
|
|
|
pending_translations: usize,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl TranslationPadTask {
|
2023-03-16 17:20:08 +00:00
|
|
|
async fn try_new(
|
2023-03-15 13:59:50 +00:00
|
|
|
pad: &TranslateSrcPad,
|
2023-03-10 13:47:38 +00:00
|
|
|
elem: super::Transcriber,
|
|
|
|
) -> Result<TranslationPadTask, gst::ErrorMessage> {
|
2023-03-16 17:20:08 +00:00
|
|
|
let mut translation_loop = None;
|
|
|
|
let mut translate_loop_handle = None;
|
|
|
|
let mut to_translate_tx = None;
|
|
|
|
let mut from_translate_rx = None;
|
|
|
|
|
|
|
|
let (our_latency, transcript_event_rx, needs_translate);
|
|
|
|
|
|
|
|
{
|
|
|
|
let elem_imp = elem.imp();
|
|
|
|
let elem_settings = elem_imp.settings.lock().unwrap();
|
|
|
|
|
|
|
|
let pad_settings = pad.settings.lock().unwrap();
|
|
|
|
|
|
|
|
our_latency = TranslateSrcPad::our_latency(&elem_settings, &pad_settings);
|
|
|
|
if our_latency + elem_settings.lateness <= 2 * GRANULARITY {
|
|
|
|
let err = format!(
|
|
|
|
"total latency + lateness must be greater than {}",
|
|
|
|
2 * GRANULARITY
|
|
|
|
);
|
|
|
|
gst::error!(CAT, imp: pad, "{err}");
|
|
|
|
return Err(gst::error_msg!(gst::LibraryError::Settings, ["{err}"]));
|
|
|
|
}
|
|
|
|
|
|
|
|
needs_translate = TranslateSrcPad::needs_translation(
|
|
|
|
&elem_settings.language_code,
|
|
|
|
pad_settings.language_code.as_deref(),
|
|
|
|
);
|
|
|
|
|
|
|
|
if needs_translate {
|
|
|
|
let (to_loop_tx, to_loop_rx) = mpsc::channel(64);
|
|
|
|
let (from_loop_tx, from_loop_rx) = mpsc::channel(64);
|
|
|
|
|
|
|
|
translation_loop = Some(TranslateLoop::new(
|
|
|
|
elem_imp,
|
|
|
|
pad,
|
|
|
|
&elem_settings.language_code,
|
|
|
|
pad_settings.language_code.as_deref().unwrap(),
|
|
|
|
pad_settings.tokenization_method,
|
|
|
|
to_loop_rx,
|
|
|
|
from_loop_tx,
|
|
|
|
));
|
|
|
|
|
|
|
|
to_translate_tx = Some(to_loop_tx);
|
|
|
|
from_translate_rx = Some(from_loop_rx);
|
|
|
|
|
|
|
|
transcript_event_rx = elem_imp.transcript_event_for_translate_tx.subscribe();
|
|
|
|
} else {
|
|
|
|
transcript_event_rx = elem_imp.transcript_event_tx.subscribe();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if let Some(translation_loop) = translation_loop {
|
|
|
|
translation_loop.check_language().await?;
|
|
|
|
translate_loop_handle = Some(RUNTIME.spawn(translation_loop.run()));
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(TranslationPadTask {
|
2023-03-10 13:47:38 +00:00
|
|
|
pad: pad.ref_counted(),
|
|
|
|
elem,
|
|
|
|
transcript_event_rx,
|
2023-03-16 17:20:08 +00:00
|
|
|
needs_translate,
|
|
|
|
translate_loop_handle,
|
|
|
|
to_translate_tx,
|
|
|
|
from_translate_rx,
|
2023-03-10 13:47:38 +00:00
|
|
|
send_events: true,
|
2023-03-16 11:17:38 +00:00
|
|
|
output_items: VecDeque::new(),
|
2023-03-16 17:20:08 +00:00
|
|
|
our_latency,
|
2023-03-10 13:47:38 +00:00
|
|
|
seqnum: gst::Seqnum::next(),
|
|
|
|
send_eos: false,
|
|
|
|
pending_translations: 0,
|
2023-03-16 17:20:08 +00:00
|
|
|
})
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Drop for TranslationPadTask {
|
|
|
|
fn drop(&mut self) {
|
2023-03-15 13:59:50 +00:00
|
|
|
if let Some(translate_loop_handle) = self.translate_loop_handle.take() {
|
|
|
|
translate_loop_handle.abort();
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl TranslationPadTask {
|
|
|
|
async fn run_iter(&mut self) -> Result<(), gst::ErrorMessage> {
|
|
|
|
self.ensure_init_events()?;
|
|
|
|
|
|
|
|
if self.needs_translate {
|
|
|
|
self.translate_iter().await?;
|
|
|
|
} else {
|
|
|
|
self.passthrough_iter().await?;
|
|
|
|
}
|
|
|
|
|
|
|
|
if !self.dequeue().await {
|
|
|
|
gst::info!(CAT, imp: self.pad, "Failed to dequeue buffer, pausing");
|
|
|
|
let _ = self.pad.obj().pause_task();
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn passthrough_iter(&mut self) -> Result<(), gst::ErrorMessage> {
|
|
|
|
// This is to make sure we send items on a timely basis or at least Gap events.
|
|
|
|
let timeout = tokio::time::sleep(GRANULARITY.into()).fuse();
|
|
|
|
futures::pin_mut!(timeout);
|
|
|
|
|
|
|
|
let transcript_event_rx = self.transcript_event_rx.recv().fuse();
|
|
|
|
futures::pin_mut!(transcript_event_rx);
|
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
// `transcript_event_rx` takes precedence over `timeout`
|
|
|
|
// because we don't want to loose any incoming items.
|
2023-03-10 13:47:38 +00:00
|
|
|
futures::select_biased! {
|
|
|
|
items_res = transcript_event_rx => {
|
|
|
|
use TranscriptEvent::*;
|
|
|
|
use broadcast::error::RecvError;
|
|
|
|
match items_res {
|
|
|
|
Ok(Items(transcript_items)) => {
|
2023-03-16 11:17:38 +00:00
|
|
|
self.output_items.extend(transcript_items.iter().map(Into::into));
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
Ok(Eos) => {
|
|
|
|
gst::debug!(CAT, imp: self.pad, "Got eos");
|
|
|
|
self.send_eos = true;
|
|
|
|
}
|
|
|
|
Err(RecvError::Lagged(nb_msg)) => {
|
|
|
|
gst::warning!(CAT, imp: self.pad, "Missed {nb_msg} transcript sets");
|
|
|
|
}
|
|
|
|
Err(RecvError::Closed) => {
|
|
|
|
gst::debug!(CAT, imp: self.pad, "Transcript chan terminated: setting eos");
|
|
|
|
self.send_eos = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-03-16 17:20:08 +00:00
|
|
|
_ = timeout => (),
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn translate_iter(&mut self) -> Result<(), gst::ErrorMessage> {
|
|
|
|
if self
|
2023-03-15 13:59:50 +00:00
|
|
|
.translate_loop_handle
|
2023-03-10 13:47:38 +00:00
|
|
|
.as_ref()
|
|
|
|
.map_or(true, task::JoinHandle::is_finished)
|
|
|
|
{
|
2023-03-15 13:59:50 +00:00
|
|
|
const ERR: &str = "Translate loop is not running";
|
2023-03-10 13:47:38 +00:00
|
|
|
gst::error!(CAT, imp: self.pad, "{ERR}");
|
|
|
|
return Err(gst::error_msg!(gst::StreamError::Failed, ["{ERR}"]));
|
|
|
|
}
|
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
let items_to_translate = {
|
2023-03-10 13:47:38 +00:00
|
|
|
// This is to make sure we send items on a timely basis or at least Gap events.
|
|
|
|
let timeout = tokio::time::sleep(GRANULARITY.into()).fuse();
|
|
|
|
futures::pin_mut!(timeout);
|
|
|
|
|
|
|
|
let transcript_event_rx = self.transcript_event_rx.recv().fuse();
|
|
|
|
futures::pin_mut!(transcript_event_rx);
|
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
// `transcript_event_rx` takes precedence over `timeout`
|
|
|
|
// because we don't want to loose any incoming items.
|
2023-03-10 13:47:38 +00:00
|
|
|
futures::select_biased! {
|
|
|
|
items_res = transcript_event_rx => {
|
|
|
|
use TranscriptEvent::*;
|
|
|
|
use broadcast::error::RecvError;
|
|
|
|
match items_res {
|
2023-03-16 17:20:08 +00:00
|
|
|
Ok(Items(items_to_translate)) => Some(items_to_translate),
|
2023-03-10 13:47:38 +00:00
|
|
|
Ok(Eos) => {
|
|
|
|
gst::debug!(CAT, imp: self.pad, "Got eos");
|
|
|
|
self.send_eos = true;
|
2023-03-16 17:20:08 +00:00
|
|
|
None
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
Err(RecvError::Lagged(nb_msg)) => {
|
|
|
|
gst::warning!(CAT, imp: self.pad, "Missed {nb_msg} transcript sets");
|
2023-03-16 17:20:08 +00:00
|
|
|
None
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
Err(RecvError::Closed) => {
|
|
|
|
gst::debug!(CAT, imp: self.pad, "Transcript chan terminated: setting eos");
|
|
|
|
self.send_eos = true;
|
2023-03-16 17:20:08 +00:00
|
|
|
None
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-03-16 17:20:08 +00:00
|
|
|
_ = timeout => None,
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
if let Some(items_to_translate) = items_to_translate {
|
|
|
|
if !items_to_translate.is_empty() {
|
|
|
|
let res = self
|
|
|
|
.to_translate_tx
|
|
|
|
.as_mut()
|
|
|
|
.expect("to_translation chan must be available in translation mode")
|
|
|
|
.send(items_to_translate)
|
|
|
|
.await;
|
|
|
|
|
|
|
|
if res.is_err() {
|
|
|
|
const ERR: &str = "to_translation chan terminated";
|
|
|
|
gst::debug!(CAT, imp: self.pad, "{ERR}");
|
|
|
|
return Err(gst::error_msg!(gst::StreamError::Failed, ["{ERR}"]));
|
|
|
|
}
|
|
|
|
|
|
|
|
self.pending_translations += 1;
|
2023-03-16 11:17:38 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
// Check pending translated items
|
|
|
|
let from_translate_rx = self
|
|
|
|
.from_translate_rx
|
|
|
|
.as_mut()
|
|
|
|
.expect("from_translation chan must be available in translation mode");
|
2023-03-16 11:17:38 +00:00
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
while let Ok(translated_items) = from_translate_rx.try_next() {
|
|
|
|
let Some(translated_items) = translated_items else {
|
|
|
|
const ERR: &str = "translation chan terminated";
|
|
|
|
gst::debug!(CAT, imp: self.pad, "{ERR}");
|
|
|
|
return Err(gst::error_msg!(gst::StreamError::Failed, ["{ERR}"]));
|
|
|
|
};
|
|
|
|
|
|
|
|
self.output_items
|
|
|
|
.extend(translated_items.into_iter().map(Into::into));
|
|
|
|
self.pending_translations = self.pending_translations.saturating_sub(1);
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn dequeue(&mut self) -> bool {
|
2023-03-16 17:20:08 +00:00
|
|
|
let Some((start_time, now)) = self.elem.imp().get_start_time_and_now() else {
|
|
|
|
// Wait for the clock to be available
|
|
|
|
return true;
|
|
|
|
};
|
2023-03-10 13:47:38 +00:00
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
let (mut last_position, mut discont_pending) = {
|
|
|
|
let mut state = self.pad.state.lock().unwrap();
|
2023-03-10 13:47:38 +00:00
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
let last_position = if let Some(pos) = state.out_segment.position() {
|
|
|
|
pos
|
|
|
|
} else {
|
|
|
|
state.out_segment.set_position(start_time);
|
|
|
|
start_time
|
|
|
|
};
|
2023-03-10 13:47:38 +00:00
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
(last_position, state.discont_pending)
|
|
|
|
};
|
2023-03-10 13:47:38 +00:00
|
|
|
|
|
|
|
/* First, check our pending buffers */
|
2023-03-16 11:17:38 +00:00
|
|
|
while let Some(item) = self.output_items.front() {
|
2023-03-10 13:47:38 +00:00
|
|
|
// Note: items pts start from 0 + lateness
|
|
|
|
gst::trace!(
|
|
|
|
CAT,
|
|
|
|
imp: self.pad,
|
|
|
|
"Checking now {now} if item is ready for dequeuing, PTS {}, threshold {} vs {}",
|
|
|
|
item.pts,
|
|
|
|
item.pts + self.our_latency.saturating_sub(3 * GRANULARITY),
|
|
|
|
now - start_time
|
|
|
|
);
|
|
|
|
|
|
|
|
// Margin:
|
2023-03-16 09:29:24 +00:00
|
|
|
// - 2 * GRANULARITY: to make sure we don't push items up to GRANULARITY late.
|
|
|
|
// - 1 * GRANULARITY: extra margin to account for additional overheads.
|
2023-03-10 13:47:38 +00:00
|
|
|
if item.pts + self.our_latency.saturating_sub(3 * GRANULARITY) < now - start_time {
|
|
|
|
/* Safe unwrap, we know we have an item */
|
2023-03-16 11:17:38 +00:00
|
|
|
let OutputItem {
|
2023-03-10 13:47:38 +00:00
|
|
|
pts: item_pts,
|
|
|
|
mut duration,
|
|
|
|
content,
|
2023-03-16 11:17:38 +00:00
|
|
|
} = self.output_items.pop_front().unwrap();
|
2023-03-10 13:47:38 +00:00
|
|
|
|
|
|
|
let mut pts = start_time + item_pts;
|
|
|
|
|
|
|
|
let mut buf = gst::Buffer::from_mut_slice(content.into_bytes());
|
|
|
|
{
|
|
|
|
let buf = buf.get_mut().unwrap();
|
|
|
|
|
|
|
|
if discont_pending {
|
|
|
|
buf.set_flags(gst::BufferFlags::DISCONT);
|
|
|
|
discont_pending = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
buf.set_pts(pts);
|
|
|
|
buf.set_duration(duration);
|
|
|
|
}
|
|
|
|
|
|
|
|
use std::cmp::Ordering::*;
|
|
|
|
match pts.cmp(&last_position) {
|
|
|
|
Greater => {
|
|
|
|
// The buffer we are about to push starts after the end of
|
|
|
|
// last item previously pushed to the stream.
|
|
|
|
let gap_event = gst::event::Gap::builder(last_position)
|
|
|
|
.duration(pts - last_position)
|
|
|
|
.seqnum(self.seqnum)
|
|
|
|
.build();
|
|
|
|
gst::log!(CAT, imp: self.pad, "Pushing gap: {last_position} -> {pts}");
|
|
|
|
if !self.pad.obj().push_event(gap_event) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Less => {
|
|
|
|
// The buffer we are about to push was expected to start
|
|
|
|
// before the end of last item previously pushed to the stream.
|
|
|
|
// => update it to fit in stream.
|
|
|
|
let delta = last_position - pts;
|
|
|
|
|
|
|
|
gst::warning!(
|
|
|
|
CAT,
|
|
|
|
imp: self.pad,
|
|
|
|
"Updating item PTS ({pts} < {last_position}), consider increasing latency",
|
|
|
|
);
|
|
|
|
|
|
|
|
pts = last_position;
|
|
|
|
// FIXME if the resulting duration is zero, we might as well not push it.
|
|
|
|
duration = duration.saturating_sub(delta);
|
|
|
|
|
|
|
|
{
|
|
|
|
let buf_mut = buf.get_mut().unwrap();
|
|
|
|
|
|
|
|
buf_mut.set_pts(pts);
|
|
|
|
buf_mut.set_duration(duration);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
_ => (),
|
|
|
|
}
|
|
|
|
|
|
|
|
last_position = pts + duration;
|
|
|
|
|
|
|
|
gst::debug!(CAT, imp: self.pad, "Pushing buffer: {pts} -> {}", pts + duration);
|
|
|
|
|
|
|
|
if self.pad.obj().push(buf).is_err() {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// Current and subsequent items are not ready to be pushed
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-16 17:20:08 +00:00
|
|
|
if self.send_eos && self.pending_translations == 0 && self.output_items.is_empty() {
|
2023-03-10 13:47:38 +00:00
|
|
|
/* We're EOS, we can pause and exit early */
|
|
|
|
let _ = self.pad.obj().pause_task();
|
|
|
|
|
|
|
|
gst::info!(CAT, imp: self.pad, "Sending eos");
|
|
|
|
return self
|
|
|
|
.pad
|
|
|
|
.obj()
|
|
|
|
.push_event(gst::event::Eos::builder().seqnum(self.seqnum).build());
|
|
|
|
}
|
|
|
|
|
|
|
|
/* next, push a gap if we're lagging behind the target position */
|
|
|
|
gst::trace!(
|
|
|
|
CAT,
|
|
|
|
imp: self.pad,
|
|
|
|
"Checking now: {now} if we need to push a gap, last_position: {last_position}, threshold: {}",
|
|
|
|
last_position + self.our_latency.saturating_sub(GRANULARITY)
|
|
|
|
);
|
|
|
|
|
|
|
|
if now > last_position + self.our_latency.saturating_sub(GRANULARITY) {
|
|
|
|
// We are running out of latency budget since last time we pushed downstream,
|
|
|
|
// so push a Gap long enough to keep continuity before we dequeue again:
|
|
|
|
// worse case scenario, this is GRANULARITY ms from now.
|
|
|
|
let duration = now - last_position - self.our_latency.saturating_sub(GRANULARITY);
|
|
|
|
|
|
|
|
let gap_event = gst::event::Gap::builder(last_position)
|
|
|
|
.duration(duration)
|
|
|
|
.seqnum(self.seqnum)
|
|
|
|
.build();
|
|
|
|
|
|
|
|
gst::log!(
|
|
|
|
CAT,
|
|
|
|
imp: self.pad,
|
|
|
|
"Pushing gap: {last_position} -> {}",
|
|
|
|
last_position + duration
|
|
|
|
);
|
|
|
|
|
|
|
|
last_position += duration;
|
|
|
|
|
|
|
|
if !self.pad.obj().push_event(gap_event) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut pad_state = self.pad.state.lock().unwrap();
|
|
|
|
pad_state.out_segment.set_position(last_position);
|
|
|
|
pad_state.discont_pending = discont_pending;
|
|
|
|
|
|
|
|
true
|
|
|
|
}
|
|
|
|
|
|
|
|
fn ensure_init_events(&mut self) -> Result<(), gst::ErrorMessage> {
|
|
|
|
if !self.send_events {
|
|
|
|
return Ok(());
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut events = vec![];
|
|
|
|
|
|
|
|
{
|
|
|
|
let elem_imp = self.elem.imp();
|
|
|
|
let elem_state = elem_imp.state.lock().unwrap();
|
|
|
|
|
|
|
|
let mut pad_state = self.pad.state.lock().unwrap();
|
|
|
|
|
|
|
|
self.seqnum = elem_state.seqnum;
|
|
|
|
pad_state.out_segment = Default::default();
|
|
|
|
|
|
|
|
events.push(
|
|
|
|
gst::event::StreamStart::builder("transcription")
|
|
|
|
.seqnum(self.seqnum)
|
|
|
|
.build(),
|
|
|
|
);
|
|
|
|
|
|
|
|
let caps = gst::Caps::builder("text/x-raw")
|
|
|
|
.field("format", "utf8")
|
|
|
|
.build();
|
|
|
|
events.push(gst::event::Caps::builder(&caps).seqnum(self.seqnum).build());
|
|
|
|
|
|
|
|
events.push(
|
|
|
|
gst::event::Segment::builder(&pad_state.out_segment)
|
|
|
|
.seqnum(self.seqnum)
|
|
|
|
.build(),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
for event in events.drain(..) {
|
|
|
|
gst::info!(CAT, imp: self.pad, "Sending {event:?}");
|
|
|
|
if !self.pad.obj().push_event(event) {
|
|
|
|
const ERR: &str = "Failed to send initial";
|
|
|
|
gst::error!(CAT, imp: self.pad, "{ERR}");
|
|
|
|
return Err(gst::error_msg!(gst::StreamError::Failed, ["{ERR}"]));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
self.send_events = false;
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
struct TranslationPadState {
|
|
|
|
discont_pending: bool,
|
|
|
|
out_segment: gst::FormattedSegment<gst::ClockTime>,
|
|
|
|
task_abort_handle: Option<AbortHandle>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Default for TranslationPadState {
|
|
|
|
fn default() -> TranslationPadState {
|
|
|
|
TranslationPadState {
|
|
|
|
discont_pending: true,
|
|
|
|
out_segment: Default::default(),
|
|
|
|
task_abort_handle: None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Debug, Default, Clone)]
|
2023-03-15 13:59:50 +00:00
|
|
|
struct TranslatePadSettings {
|
2023-03-10 13:47:38 +00:00
|
|
|
language_code: Option<String>,
|
2023-03-11 15:27:51 +00:00
|
|
|
tokenization_method: TranslationTokenizationMethod,
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Debug, Default)]
|
2023-03-15 13:59:50 +00:00
|
|
|
pub struct TranslateSrcPad {
|
2023-03-10 13:47:38 +00:00
|
|
|
state: Mutex<TranslationPadState>,
|
2023-03-15 13:59:50 +00:00
|
|
|
settings: Mutex<TranslatePadSettings>,
|
2023-03-10 13:47:38 +00:00
|
|
|
}
|
|
|
|
|
2023-03-15 13:59:50 +00:00
|
|
|
impl TranslateSrcPad {
|
2023-03-10 13:47:38 +00:00
|
|
|
fn start_task(&self) -> Result<(), gst::LoggableError> {
|
|
|
|
gst::debug!(CAT, imp: self, "Starting task");
|
|
|
|
|
|
|
|
let elem = self.parent();
|
2023-03-16 17:20:08 +00:00
|
|
|
let _enter = RUNTIME.enter();
|
|
|
|
let mut pad_task = futures::executor::block_on(TranslationPadTask::try_new(self, elem))
|
2023-03-10 13:47:38 +00:00
|
|
|
.map_err(|err| gst::loggable_error!(CAT, format!("Failed to start pad task {err}")))?;
|
|
|
|
|
|
|
|
let imp = self.ref_counted();
|
|
|
|
let res = self.obj().start_task(move || {
|
|
|
|
let (abortable_task_iter, abort_handle) = future::abortable(pad_task.run_iter());
|
|
|
|
imp.state.lock().unwrap().task_abort_handle = Some(abort_handle);
|
|
|
|
|
|
|
|
let _enter = RUNTIME.enter();
|
|
|
|
match futures::executor::block_on(abortable_task_iter) {
|
|
|
|
Ok(Ok(())) => (),
|
|
|
|
Ok(Err(err)) => {
|
|
|
|
// Don't bring down the whole element if this Pad fails
|
|
|
|
// FIXME is there a way to mark the Pad in error though?
|
|
|
|
gst::info!(CAT, imp: imp, "Pausing task due to: {err}");
|
|
|
|
let _ = imp.obj().pause_task();
|
|
|
|
}
|
|
|
|
Err(_) => gst::debug!(CAT, imp: imp, "task iter aborted"),
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
if res.is_err() {
|
|
|
|
return Err(gst::loggable_error!(CAT, "Failed to start pad task"));
|
|
|
|
}
|
|
|
|
|
|
|
|
gst::debug!(CAT, imp: self, "Task started");
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn stop_task(&self) {
|
|
|
|
gst::debug!(CAT, imp: self, "Stopping task");
|
|
|
|
|
|
|
|
// See also the note in `start_task()`:
|
|
|
|
// 1. Mark the task as stopped so no further iteration is executed.
|
|
|
|
let _ = self.obj().stop_task();
|
|
|
|
|
|
|
|
// 2. Abort the task iteration if the Future is pending.
|
|
|
|
if let Some(task_abort_handle) = self.state.lock().unwrap().task_abort_handle.take() {
|
|
|
|
task_abort_handle.abort();
|
|
|
|
}
|
|
|
|
|
|
|
|
gst::debug!(CAT, imp: self, "Task stopped");
|
|
|
|
}
|
|
|
|
|
|
|
|
fn set_discont(&self) {
|
|
|
|
self.state.lock().unwrap().discont_pending = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
|
|
|
fn needs_translation(input_lang: &str, output_lang: Option<&str>) -> bool {
|
|
|
|
output_lang.map_or(false, |other| {
|
|
|
|
!input_lang.eq_ignore_ascii_case(other.as_ref())
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
|
|
|
fn our_latency(
|
|
|
|
elem_settings: &Settings,
|
2023-03-15 13:59:50 +00:00
|
|
|
pad_settings: &TranslatePadSettings,
|
2023-03-10 13:47:38 +00:00
|
|
|
) -> gst::ClockTime {
|
|
|
|
if Self::needs_translation(
|
|
|
|
&elem_settings.language_code,
|
|
|
|
pad_settings.language_code.as_deref(),
|
|
|
|
) {
|
2023-03-16 09:29:24 +00:00
|
|
|
elem_settings.transcribe_latency + elem_settings.translate_latency
|
2023-03-10 13:47:38 +00:00
|
|
|
} else {
|
|
|
|
elem_settings.transcribe_latency
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[track_caller]
|
|
|
|
fn parent(&self) -> super::Transcriber {
|
|
|
|
self.obj()
|
|
|
|
.parent()
|
|
|
|
.map(|elem_obj| {
|
|
|
|
elem_obj
|
|
|
|
.downcast::<super::Transcriber>()
|
|
|
|
.expect("Wrong Element type")
|
|
|
|
})
|
|
|
|
.expect("Pad should have a parent at this stage")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-15 13:59:50 +00:00
|
|
|
impl TranslateSrcPad {
|
2023-03-10 13:47:38 +00:00
|
|
|
#[track_caller]
|
|
|
|
pub fn activatemode(
|
|
|
|
_elem: &Transcriber,
|
2023-03-15 13:59:50 +00:00
|
|
|
pad: &super::TranslateSrcPad,
|
2023-03-10 13:47:38 +00:00
|
|
|
_mode: gst::PadMode,
|
|
|
|
active: bool,
|
|
|
|
) -> Result<(), gst::LoggableError> {
|
|
|
|
if active {
|
|
|
|
pad.imp().start_task()?;
|
|
|
|
} else {
|
|
|
|
pad.imp().stop_task();
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn src_query(
|
|
|
|
elem: &Transcriber,
|
2023-03-15 13:59:50 +00:00
|
|
|
pad: &super::TranslateSrcPad,
|
2023-03-10 13:47:38 +00:00
|
|
|
query: &mut gst::QueryRef,
|
|
|
|
) -> bool {
|
|
|
|
gst::log!(CAT, obj: pad, "Handling query {query:?}");
|
|
|
|
|
|
|
|
use gst::QueryViewMut::*;
|
|
|
|
match query.view_mut() {
|
|
|
|
Latency(q) => {
|
|
|
|
let mut peer_query = gst::query::Latency::new();
|
|
|
|
|
|
|
|
let ret = elem.sinkpad.peer_query(&mut peer_query);
|
|
|
|
|
|
|
|
if ret {
|
|
|
|
let (_, min, _) = peer_query.result();
|
|
|
|
|
|
|
|
let our_latency = {
|
|
|
|
let elem_settings = elem.settings.lock().unwrap();
|
|
|
|
let pad_settings = pad.imp().settings.lock().unwrap();
|
|
|
|
|
|
|
|
Self::our_latency(&elem_settings, &pad_settings)
|
|
|
|
};
|
|
|
|
|
|
|
|
gst::info!(CAT, obj: pad, "Our latency {our_latency}");
|
|
|
|
q.set(true, our_latency + min, gst::ClockTime::NONE);
|
|
|
|
}
|
|
|
|
ret
|
|
|
|
}
|
|
|
|
Position(q) => {
|
|
|
|
if q.format() == gst::Format::Time {
|
|
|
|
let stream_time = {
|
|
|
|
let state = pad.imp().state.lock().unwrap();
|
|
|
|
state
|
|
|
|
.out_segment
|
|
|
|
.to_stream_time(state.out_segment.position())
|
|
|
|
};
|
|
|
|
|
|
|
|
let Some(stream_time) = stream_time else { return false };
|
|
|
|
q.set(stream_time);
|
|
|
|
|
|
|
|
true
|
|
|
|
} else {
|
|
|
|
false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
_ => gst::Pad::query_default(pad, Some(pad), query),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[glib::object_subclass]
|
2023-03-15 13:59:50 +00:00
|
|
|
impl ObjectSubclass for TranslateSrcPad {
|
|
|
|
const NAME: &'static str = "GstTranslateSrcPad";
|
|
|
|
type Type = super::TranslateSrcPad;
|
2023-03-10 13:47:38 +00:00
|
|
|
type ParentType = gst::Pad;
|
|
|
|
|
|
|
|
fn new() -> Self {
|
|
|
|
Default::default()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-15 13:59:50 +00:00
|
|
|
impl ObjectImpl for TranslateSrcPad {
|
2023-03-10 13:47:38 +00:00
|
|
|
fn properties() -> &'static [glib::ParamSpec] {
|
|
|
|
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
|
2023-03-11 15:27:51 +00:00
|
|
|
vec![
|
|
|
|
glib::ParamSpecString::builder(OUTPUT_LANG_CODE_PROPERTY)
|
|
|
|
.nick("Language Code")
|
|
|
|
.blurb("The Language the Stream must be translated to")
|
|
|
|
.default_value(DEFAULT_OUTPUT_LANG_CODE)
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
|
|
|
glib::ParamSpecEnum::builder(TRANSLATION_TOKENIZATION_PROPERTY)
|
|
|
|
.nick("Translations tokenization method")
|
|
|
|
.blurb("The tokenization method to apply to translations")
|
|
|
|
.default_value(TranslationTokenizationMethod::default())
|
|
|
|
.mutable_ready()
|
|
|
|
.build(),
|
|
|
|
]
|
2023-03-10 13:47:38 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
PROPERTIES.as_ref()
|
|
|
|
}
|
|
|
|
|
|
|
|
fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) {
|
|
|
|
match pspec.name() {
|
|
|
|
OUTPUT_LANG_CODE_PROPERTY => {
|
|
|
|
self.settings.lock().unwrap().language_code = value.get().unwrap()
|
|
|
|
}
|
2023-03-11 15:27:51 +00:00
|
|
|
TRANSLATION_TOKENIZATION_PROPERTY => {
|
|
|
|
self.settings.lock().unwrap().tokenization_method = value.get().unwrap()
|
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
_ => unimplemented!(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value {
|
|
|
|
match pspec.name() {
|
|
|
|
OUTPUT_LANG_CODE_PROPERTY => self.settings.lock().unwrap().language_code.to_value(),
|
2023-03-11 15:27:51 +00:00
|
|
|
TRANSLATION_TOKENIZATION_PROPERTY => {
|
|
|
|
self.settings.lock().unwrap().tokenization_method.to_value()
|
|
|
|
}
|
2023-03-10 13:47:38 +00:00
|
|
|
_ => unimplemented!(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-15 13:59:50 +00:00
|
|
|
impl GstObjectImpl for TranslateSrcPad {}
|
2023-03-10 13:47:38 +00:00
|
|
|
|
2023-03-15 13:59:50 +00:00
|
|
|
impl PadImpl for TranslateSrcPad {}
|