Working for local server

Cargo.lock

@@ -34,18 +34,39 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "base64"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "block-buffer"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "byteorder"
 version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
+[[package]]
+name = "bytes"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
+
 [[package]]
 name = "cc"
 version = "1.0.73"
@@ -67,6 +88,25 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "cpufeatures"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
 [[package]]
 name = "ctrlc"
 version = "3.2.1"
@@ -77,6 +117,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "digest"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
 [[package]]
 name = "env_logger"
 version = "0.7.1"
@@ -100,6 +150,22 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "form_urlencoded"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191"
+dependencies = [
+ "matches",
+ "percent-encoding",
+]
+
 [[package]]
 name = "futures-channel"
 version = "0.3.21"
@@ -151,6 +217,27 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "generic-array"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
 [[package]]
 name = "glib"
 version = "0.15.9"
@@ -347,10 +434,21 @@ dependencies = [
 ]
 
 [[package]]
-name = "hound"
+name = "http"
-version = "3.4.0"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a164bb2ceaeff4f42542bdb847c41517c78a60f5649671b2a07312b6e117549"
+checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
+[[package]]
+name = "httparse"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4"
 
 [[package]]
 name = "humantime"
@@ -361,6 +459,17 @@ dependencies = [
  "quick-error",
 ]
 
+[[package]]
+name = "idna"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8"
+dependencies = [
+ "matches",
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
 [[package]]
 name = "indenter"
 version = "0.3.3"
@@ -368,10 +477,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
 
 [[package]]
-name = "libc"
+name = "itoa"
-version = "0.2.120"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad5c14e80759d0939d013e6ca49930e59fc53dd8e5009132f76240c179380c09"
+checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
+
+[[package]]
+name = "libc"
+version = "0.2.121"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f"
 
 [[package]]
 name = "log"
@@ -382,6 +497,12 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "matches"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
+
 [[package]]
 name = "memchr"
 version = "2.4.1"
@@ -467,6 +588,12 @@ version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5"
 
+[[package]]
+name = "percent-encoding"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.8"
@@ -485,6 +612,12 @@ version = "0.3.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
 
+[[package]]
+name = "ppv-lite86"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
+
 [[package]]
 name = "pretty-hex"
 version = "0.2.1"
@@ -559,6 +692,36 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
+dependencies = [
+ "getrandom",
+]
+
 [[package]]
 name = "regex"
 version = "1.5.5"
@@ -576,12 +739,51 @@ version = "0.6.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
 
+[[package]]
+name = "ryu"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
+
 [[package]]
 name = "serde"
 version = "1.0.136"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
 
+[[package]]
+name = "serde_derive"
+version = "1.0.136"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "sha-1"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
 [[package]]
 name = "signal-hook"
 version = "0.3.13"
@@ -666,6 +868,21 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "tinyvec"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
+
 [[package]]
 name = "toml"
 version = "0.5.8"
@@ -675,12 +892,70 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "tungstenite"
+version = "0.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d96a2dea40e7570482f28eb57afbe42d97551905da6a9400acc5c328d24004f5"
+dependencies = [
+ "base64",
+ "byteorder",
+ "bytes",
+ "http",
+ "httparse",
+ "log",
+ "rand",
+ "sha-1",
+ "thiserror",
+ "url",
+ "utf-8",
+]
+
+[[package]]
+name = "typenum"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+
+[[package]]
+name = "unicode-bidi"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f"
+
+[[package]]
+name = "unicode-normalization"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9"
+dependencies = [
+ "tinyvec",
+]
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
 
+[[package]]
+name = "url"
+version = "2.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
+dependencies = [
+ "form_urlencoded",
+ "idna",
+ "matches",
+ "percent-encoding",
+]
+
+[[package]]
+name = "utf-8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+
 [[package]]
 name = "version-compare"
 version = "0.1.0"
@@ -693,11 +968,16 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "wasi"
+version = "0.10.2+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+
 [[package]]
 name = "webvttconverter"
 version = "0.1.0"
 dependencies = [
- "byteorder",
  "ctrlc",
  "eyre",
  "glib",
@@ -705,10 +985,14 @@ dependencies = [
  "gstreamer-app",
  "gstreamer-base",
  "gstreamer-video",
- "hound",
  "log",
  "pretty_env_logger",
+ "serde",
+ "serde_derive",
+ "serde_json",
  "signal-hook",
+ "tungstenite",
+ "url",
 ]
 
 [[package]]

Cargo.toml

@@ -14,5 +14,8 @@ ctrlc = "3.2.1"
 signal-hook = "0.3.13"
 log = "0.4.14"
 pretty_env_logger = "0.4.0"
-hound = "3.4"
+tungstenite = "0.17"
-byteorder = "1"
+serde = "1"
+serde_derive = "1"
+serde_json = "1"
+url = "2"

src/main.rs

@@ -1,21 +1,63 @@
-use byteorder::{LittleEndian, ReadBytesExt};
 use gst::prelude::*;
 use gstreamer_app as gst_app;
-use hound::{WavSpec, WavWriter};
 use log::{debug, error, info, trace, warn};
+use serde_derive::{Deserialize, Serialize};
 use std::fs::File;
 use std::io::{Cursor, Write};
 use std::process;
 use std::sync::{Arc, Mutex};
+use tungstenite::{connect, Message};
+use url::Url;
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Configuration {
+    config: ConfigInner,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+struct ConfigInner {
+    /// Sample rate the audio will be provided at.
+    sample_rate: i32,
+
+    /// Show time ranges of each word in the transcription.
+    words: bool,
+}
+
+impl Configuration {
+    pub fn new(sample_rate: i32) -> Self {
+        Self {
+            config: ConfigInner {
+                sample_rate,
+                // We always want to receive the words with their time ranges.
+                words: true,
+            },
+        }
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Transcript {
+    pub result: Vec<WordInfo>,
+    pub text: String,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct WordInfo {
+    #[serde(rename = "conf")]
+    pub confidence: f64,
+    pub word: String,
+    pub start: f64,
+    pub end: f64,
+}
 
 fn main() -> eyre::Result<()> {
-    pretty_env_logger::init();
+    pretty_env_logger::init_timed();
     gst::init()?;
 
     let pipeline = gst::parse_launch(
         r#"
         uridecodebin uri=file:///Users/rafael.caricio/video.mkv name=dec dec.src_1 ! audio/x-raw !
-        audioconvert ! audiorate ! audioresample ! audio/x-raw,format=S16LE,rate=48000,channels=1 ! appsink name=sink
+        audioconvert ! audiorate ! audioresample ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! appsink name=sink
     "#,
     )?
         .downcast::<gst::Pipeline>()
@@ -31,7 +73,17 @@ fn main() -> eyre::Result<()> {
         debug!("Pad {} added with caps {}", name, caps_type);
     });
 
-    let raw_audio_content = Arc::new(Mutex::new(Vec::new()));
+    let (mut socket, response) = connect(Url::parse("ws://localhost:2700").unwrap())?;
+
+    let config = Configuration::new(16_000);
+    info!(
+        "config payload: {}",
+        serde_json::to_string_pretty(&config).unwrap()
+    );
+    let packet = serde_json::to_string(&config).unwrap();
+    socket.write_message(Message::Text(packet)).unwrap();
+
+    let shared_socket = Arc::new(Mutex::new(socket));
 
     let app_sink = pipeline
         .by_name("sink")
@@ -42,15 +94,39 @@ fn main() -> eyre::Result<()> {
     app_sink.set_callbacks(
         gst_app::AppSinkCallbacks::builder()
             .new_sample({
-                let raw_audio_content = raw_audio_content.clone();
+                let shared_socket = shared_socket.clone();
                 move |app| {
                     let sample = app.pull_sample().unwrap();
                     let buffer = sample.buffer().unwrap();
-
                     let data = buffer.map_readable().unwrap();
 
-                    let mut raw_audio_content = raw_audio_content.lock().unwrap();
+                    let mut socket = shared_socket.lock().unwrap();
-                    raw_audio_content.extend(data.to_vec());
+
+                    for chunk in data.chunks(8_000) {
+                        socket
+                            .write_message(Message::Binary(chunk.to_vec()))
+                            .unwrap();
+
+                        let msg = socket.read_message().unwrap();
+                        match msg {
+                            Message::Text(payload) => {
+                                match serde_json::from_str::<Transcript>(&payload) {
+                                    Ok(transcript) => {
+                                        info!(
+                                            "result: {}",
+                                            serde_json::to_string_pretty(&transcript.result)
+                                                .unwrap()
+                                        );
+                                    }
+                                    Err(_) => {
+                                        // The payload is still not a final transcript, so we just ignore it
+                                        // info!("No results...");
+                                    }
+                                }
+                            }
+                            _ => {}
+                        };
+                    }
 
                     Ok(gst::FlowSuccess::Ok)
                 }
@@ -102,27 +178,27 @@ fn main() -> eyre::Result<()> {
 
     pipeline.set_state(gst::State::Null)?;
 
-    let mut out_wav_buffer = Cursor::new(Vec::new());
+    // let mut out_wav_buffer = Cursor::new(Vec::new());
-    let mut writer = WavWriter::new(
+    // let mut writer = WavWriter::new(
-        &mut out_wav_buffer,
+    //     &mut out_wav_buffer,
-        WavSpec {
+    //     WavSpec {
-            channels: 1,
+    //         channels: 1,
-            sample_rate: 48000,
+    //         sample_rate: 48000,
-            bits_per_sample: 16,
+    //         bits_per_sample: 16,
-            sample_format: hound::SampleFormat::Int,
+    //         sample_format: hound::SampleFormat::Int,
-        },
+    //     },
-    )
+    // )
-    .unwrap();
+    // .unwrap();
-
+    //
-    let mut raw_audio_content = Cursor::new(raw_audio_content.lock().unwrap().to_vec());
+    // let mut raw_audio_content = Cursor::new(raw_audio_content.lock().unwrap().to_vec());
-
+    //
-    while let Ok(sample) = raw_audio_content.read_i16::<LittleEndian>() {
+    // while let Ok(sample) = raw_audio_content.read_i16::<LittleEndian>() {
-        writer.write_sample(sample).unwrap();
+    //     writer.write_sample(sample).unwrap();
-    }
+    // }
-
+    //
-    drop(writer);
+    // drop(writer);
-    let mut file = File::create("out.wav")?;
+    // let mut file = File::create("out.raw")?;
-    file.write_all(&out_wav_buffer.into_inner())?;
+    // file.write_all(&raw_audio_content.into_inner())?;
 
     Ok(())
 }