From 398656f9f870d0d5dca3ec48d487e3d77d5618ee Mon Sep 17 00:00:00 2001 From: Rafael Caricio Date: Thu, 31 Mar 2022 14:12:55 +0200 Subject: [PATCH] Working for local server --- Cargo.lock | 300 ++++++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 7 +- src/main.rs | 136 ++++++++++++++++++------ 3 files changed, 403 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d56ef2..ae9214a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,18 +34,39 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + [[package]] name = "byteorder" version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + [[package]] name = "cc" version = "1.0.73" @@ -67,6 +88,25 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cpufeatures" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "ctrlc" version = "3.2.1" @@ -77,6 +117,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "env_logger" version = "0.7.1" @@ -100,6 +150,22 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + [[package]] name = "futures-channel" version = "0.3.21" @@ -151,6 +217,27 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "glib" version = "0.15.9" @@ -347,10 +434,21 @@ dependencies = [ ] [[package]] -name = "hound" -version = "3.4.0" +name = "http" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a164bb2ceaeff4f42542bdb847c41517c78a60f5649671b2a07312b6e117549" +checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" [[package]] name = "humantime" @@ -361,6 +459,17 @@ dependencies = [ "quick-error", ] +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indenter" version = "0.3.3" @@ -368,10 +477,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] -name = "libc" -version = "0.2.120" +name = "itoa" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad5c14e80759d0939d013e6ca49930e59fc53dd8e5009132f76240c179380c09" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "libc" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" [[package]] name = "log" @@ -382,6 +497,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "memchr" version = "2.4.1" @@ -467,6 +588,12 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + [[package]] name = "pin-project-lite" version = "0.2.8" @@ -485,6 +612,12 @@ version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + [[package]] name = "pretty-hex" version = "0.2.1" @@ -559,6 +692,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + [[package]] name = "regex" version = "1.5.5" @@ -576,12 +739,51 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "ryu" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + [[package]] name = "serde" version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +[[package]] +name = "serde_derive" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha-1" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "signal-hook" version = "0.3.13" @@ -666,6 +868,21 @@ dependencies = [ "syn", ] +[[package]] +name = "tinyvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + [[package]] name = "toml" version = "0.5.8" @@ -675,12 +892,70 @@ dependencies = [ "serde", ] +[[package]] +name = "tungstenite" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96a2dea40e7570482f28eb57afbe42d97551905da6a9400acc5c328d24004f5" +dependencies = [ + "base64", + "byteorder", + "bytes", + "http", + "httparse", + "log", + "rand", + "sha-1", + "thiserror", + "url", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-xid" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "version-compare" version = "0.1.0" @@ -693,11 +968,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "webvttconverter" version = "0.1.0" dependencies = [ - "byteorder", "ctrlc", "eyre", "glib", @@ -705,10 +985,14 @@ dependencies = [ "gstreamer-app", "gstreamer-base", "gstreamer-video", - "hound", "log", "pretty_env_logger", + "serde", + "serde_derive", + "serde_json", "signal-hook", + "tungstenite", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index aadfd58..bdcb2c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,5 +14,8 @@ ctrlc = "3.2.1" signal-hook = "0.3.13" log = "0.4.14" pretty_env_logger = "0.4.0" -hound = "3.4" -byteorder = "1" +tungstenite = "0.17" +serde = "1" +serde_derive = "1" +serde_json = "1" +url = "2" diff --git a/src/main.rs b/src/main.rs index 984104d..f05a97f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,21 +1,63 @@ -use byteorder::{LittleEndian, ReadBytesExt}; use gst::prelude::*; use gstreamer_app as gst_app; -use hound::{WavSpec, WavWriter}; use log::{debug, error, info, trace, warn}; +use serde_derive::{Deserialize, Serialize}; use std::fs::File; use std::io::{Cursor, Write}; use std::process; use std::sync::{Arc, Mutex}; +use tungstenite::{connect, Message}; +use url::Url; + +#[derive(Deserialize, Serialize, Debug)] +pub struct Configuration { + config: ConfigInner, +} + +#[derive(Deserialize, Serialize, Debug)] +struct ConfigInner { + /// Sample rate the audio will be provided at. + sample_rate: i32, + + /// Show time ranges of each word in the transcription. + words: bool, +} + +impl Configuration { + pub fn new(sample_rate: i32) -> Self { + Self { + config: ConfigInner { + sample_rate, + // We always want to receive the words with their time ranges. + words: true, + }, + } + } +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct Transcript { + pub result: Vec, + pub text: String, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct WordInfo { + #[serde(rename = "conf")] + pub confidence: f64, + pub word: String, + pub start: f64, + pub end: f64, +} fn main() -> eyre::Result<()> { - pretty_env_logger::init(); + pretty_env_logger::init_timed(); gst::init()?; let pipeline = gst::parse_launch( r#" uridecodebin uri=file:///Users/rafael.caricio/video.mkv name=dec dec.src_1 ! audio/x-raw ! - audioconvert ! audiorate ! audioresample ! audio/x-raw,format=S16LE,rate=48000,channels=1 ! appsink name=sink + audioconvert ! audiorate ! audioresample ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! appsink name=sink "#, )? .downcast::() @@ -31,7 +73,17 @@ fn main() -> eyre::Result<()> { debug!("Pad {} added with caps {}", name, caps_type); }); - let raw_audio_content = Arc::new(Mutex::new(Vec::new())); + let (mut socket, response) = connect(Url::parse("ws://localhost:2700").unwrap())?; + + let config = Configuration::new(16_000); + info!( + "config payload: {}", + serde_json::to_string_pretty(&config).unwrap() + ); + let packet = serde_json::to_string(&config).unwrap(); + socket.write_message(Message::Text(packet)).unwrap(); + + let shared_socket = Arc::new(Mutex::new(socket)); let app_sink = pipeline .by_name("sink") @@ -42,15 +94,39 @@ fn main() -> eyre::Result<()> { app_sink.set_callbacks( gst_app::AppSinkCallbacks::builder() .new_sample({ - let raw_audio_content = raw_audio_content.clone(); + let shared_socket = shared_socket.clone(); move |app| { let sample = app.pull_sample().unwrap(); let buffer = sample.buffer().unwrap(); - let data = buffer.map_readable().unwrap(); - let mut raw_audio_content = raw_audio_content.lock().unwrap(); - raw_audio_content.extend(data.to_vec()); + let mut socket = shared_socket.lock().unwrap(); + + for chunk in data.chunks(8_000) { + socket + .write_message(Message::Binary(chunk.to_vec())) + .unwrap(); + + let msg = socket.read_message().unwrap(); + match msg { + Message::Text(payload) => { + match serde_json::from_str::(&payload) { + Ok(transcript) => { + info!( + "result: {}", + serde_json::to_string_pretty(&transcript.result) + .unwrap() + ); + } + Err(_) => { + // The payload is still not a final transcript, so we just ignore it + // info!("No results..."); + } + } + } + _ => {} + }; + } Ok(gst::FlowSuccess::Ok) } @@ -102,27 +178,27 @@ fn main() -> eyre::Result<()> { pipeline.set_state(gst::State::Null)?; - let mut out_wav_buffer = Cursor::new(Vec::new()); - let mut writer = WavWriter::new( - &mut out_wav_buffer, - WavSpec { - channels: 1, - sample_rate: 48000, - bits_per_sample: 16, - sample_format: hound::SampleFormat::Int, - }, - ) - .unwrap(); - - let mut raw_audio_content = Cursor::new(raw_audio_content.lock().unwrap().to_vec()); - - while let Ok(sample) = raw_audio_content.read_i16::() { - writer.write_sample(sample).unwrap(); - } - - drop(writer); - let mut file = File::create("out.wav")?; - file.write_all(&out_wav_buffer.into_inner())?; + // let mut out_wav_buffer = Cursor::new(Vec::new()); + // let mut writer = WavWriter::new( + // &mut out_wav_buffer, + // WavSpec { + // channels: 1, + // sample_rate: 48000, + // bits_per_sample: 16, + // sample_format: hound::SampleFormat::Int, + // }, + // ) + // .unwrap(); + // + // let mut raw_audio_content = Cursor::new(raw_audio_content.lock().unwrap().to_vec()); + // + // while let Ok(sample) = raw_audio_content.read_i16::() { + // writer.write_sample(sample).unwrap(); + // } + // + // drop(writer); + // let mut file = File::create("out.raw")?; + // file.write_all(&raw_audio_content.into_inner())?; Ok(()) }