Working for local server

This commit is contained in:
Rafael Caricio 2022-03-31 14:12:55 +02:00
parent 152d02330b
commit 398656f9f8
Signed by: rafaelcaricio
GPG key ID: 3C86DBCE8E93C947
3 changed files with 403 additions and 40 deletions

300
Cargo.lock generated
View file

@ -34,18 +34,39 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "base64"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.3.2" version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "block-buffer"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
dependencies = [
"generic-array",
]
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.4.3" version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.0.73" version = "1.0.73"
@ -67,6 +88,25 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cpufeatures"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
dependencies = [
"libc",
]
[[package]]
name = "crypto-common"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8"
dependencies = [
"generic-array",
"typenum",
]
[[package]] [[package]]
name = "ctrlc" name = "ctrlc"
version = "3.2.1" version = "3.2.1"
@ -77,6 +117,16 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "digest"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]] [[package]]
name = "env_logger" name = "env_logger"
version = "0.7.1" version = "0.7.1"
@ -100,6 +150,22 @@ dependencies = [
"once_cell", "once_cell",
] ]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "form_urlencoded"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191"
dependencies = [
"matches",
"percent-encoding",
]
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.21" version = "0.3.21"
@ -151,6 +217,27 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "generic-array"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]] [[package]]
name = "glib" name = "glib"
version = "0.15.9" version = "0.15.9"
@ -347,10 +434,21 @@ dependencies = [
] ]
[[package]] [[package]]
name = "hound" name = "http"
version = "3.4.0" version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a164bb2ceaeff4f42542bdb847c41517c78a60f5649671b2a07312b6e117549" checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]]
name = "httparse"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4"
[[package]] [[package]]
name = "humantime" name = "humantime"
@ -361,6 +459,17 @@ dependencies = [
"quick-error", "quick-error",
] ]
[[package]]
name = "idna"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8"
dependencies = [
"matches",
"unicode-bidi",
"unicode-normalization",
]
[[package]] [[package]]
name = "indenter" name = "indenter"
version = "0.3.3" version = "0.3.3"
@ -368,10 +477,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
[[package]] [[package]]
name = "libc" name = "itoa"
version = "0.2.120" version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad5c14e80759d0939d013e6ca49930e59fc53dd8e5009132f76240c179380c09" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
[[package]]
name = "libc"
version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f"
[[package]] [[package]]
name = "log" name = "log"
@ -382,6 +497,12 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "matches"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.4.1" version = "2.4.1"
@ -467,6 +588,12 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5"
[[package]]
name = "percent-encoding"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.8" version = "0.2.8"
@ -485,6 +612,12 @@ version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
[[package]]
name = "ppv-lite86"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
[[package]] [[package]]
name = "pretty-hex" name = "pretty-hex"
version = "0.2.1" version = "0.2.1"
@ -559,6 +692,36 @@ dependencies = [
"proc-macro2", "proc-macro2",
] ]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
dependencies = [
"getrandom",
]
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.5.5" version = "1.5.5"
@ -576,12 +739,51 @@ version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "ryu"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.136" version = "1.0.136"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
[[package]]
name = "serde_derive"
version = "1.0.136"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "sha-1"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]] [[package]]
name = "signal-hook" name = "signal-hook"
version = "0.3.13" version = "0.3.13"
@ -666,6 +868,21 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "tinyvec"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]] [[package]]
name = "toml" name = "toml"
version = "0.5.8" version = "0.5.8"
@ -675,12 +892,70 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "tungstenite"
version = "0.17.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d96a2dea40e7570482f28eb57afbe42d97551905da6a9400acc5c328d24004f5"
dependencies = [
"base64",
"byteorder",
"bytes",
"http",
"httparse",
"log",
"rand",
"sha-1",
"thiserror",
"url",
"utf-8",
]
[[package]]
name = "typenum"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
[[package]]
name = "unicode-bidi"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f"
[[package]]
name = "unicode-normalization"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9"
dependencies = [
"tinyvec",
]
[[package]] [[package]]
name = "unicode-xid" name = "unicode-xid"
version = "0.2.2" version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "url"
version = "2.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
dependencies = [
"form_urlencoded",
"idna",
"matches",
"percent-encoding",
]
[[package]]
name = "utf-8"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]] [[package]]
name = "version-compare" name = "version-compare"
version = "0.1.0" version = "0.1.0"
@ -693,11 +968,16 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]] [[package]]
name = "webvttconverter" name = "webvttconverter"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"byteorder",
"ctrlc", "ctrlc",
"eyre", "eyre",
"glib", "glib",
@ -705,10 +985,14 @@ dependencies = [
"gstreamer-app", "gstreamer-app",
"gstreamer-base", "gstreamer-base",
"gstreamer-video", "gstreamer-video",
"hound",
"log", "log",
"pretty_env_logger", "pretty_env_logger",
"serde",
"serde_derive",
"serde_json",
"signal-hook", "signal-hook",
"tungstenite",
"url",
] ]
[[package]] [[package]]

View file

@ -14,5 +14,8 @@ ctrlc = "3.2.1"
signal-hook = "0.3.13" signal-hook = "0.3.13"
log = "0.4.14" log = "0.4.14"
pretty_env_logger = "0.4.0" pretty_env_logger = "0.4.0"
hound = "3.4" tungstenite = "0.17"
byteorder = "1" serde = "1"
serde_derive = "1"
serde_json = "1"
url = "2"

View file

@ -1,21 +1,63 @@
use byteorder::{LittleEndian, ReadBytesExt};
use gst::prelude::*; use gst::prelude::*;
use gstreamer_app as gst_app; use gstreamer_app as gst_app;
use hound::{WavSpec, WavWriter};
use log::{debug, error, info, trace, warn}; use log::{debug, error, info, trace, warn};
use serde_derive::{Deserialize, Serialize};
use std::fs::File; use std::fs::File;
use std::io::{Cursor, Write}; use std::io::{Cursor, Write};
use std::process; use std::process;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use tungstenite::{connect, Message};
use url::Url;
#[derive(Deserialize, Serialize, Debug)]
pub struct Configuration {
config: ConfigInner,
}
#[derive(Deserialize, Serialize, Debug)]
struct ConfigInner {
/// Sample rate the audio will be provided at.
sample_rate: i32,
/// Show time ranges of each word in the transcription.
words: bool,
}
impl Configuration {
pub fn new(sample_rate: i32) -> Self {
Self {
config: ConfigInner {
sample_rate,
// We always want to receive the words with their time ranges.
words: true,
},
}
}
}
#[derive(Deserialize, Serialize, Debug)]
pub struct Transcript {
pub result: Vec<WordInfo>,
pub text: String,
}
#[derive(Deserialize, Serialize, Debug)]
pub struct WordInfo {
#[serde(rename = "conf")]
pub confidence: f64,
pub word: String,
pub start: f64,
pub end: f64,
}
fn main() -> eyre::Result<()> { fn main() -> eyre::Result<()> {
pretty_env_logger::init(); pretty_env_logger::init_timed();
gst::init()?; gst::init()?;
let pipeline = gst::parse_launch( let pipeline = gst::parse_launch(
r#" r#"
uridecodebin uri=file:///Users/rafael.caricio/video.mkv name=dec dec.src_1 ! audio/x-raw ! uridecodebin uri=file:///Users/rafael.caricio/video.mkv name=dec dec.src_1 ! audio/x-raw !
audioconvert ! audiorate ! audioresample ! audio/x-raw,format=S16LE,rate=48000,channels=1 ! appsink name=sink audioconvert ! audiorate ! audioresample ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! appsink name=sink
"#, "#,
)? )?
.downcast::<gst::Pipeline>() .downcast::<gst::Pipeline>()
@ -31,7 +73,17 @@ fn main() -> eyre::Result<()> {
debug!("Pad {} added with caps {}", name, caps_type); debug!("Pad {} added with caps {}", name, caps_type);
}); });
let raw_audio_content = Arc::new(Mutex::new(Vec::new())); let (mut socket, response) = connect(Url::parse("ws://localhost:2700").unwrap())?;
let config = Configuration::new(16_000);
info!(
"config payload: {}",
serde_json::to_string_pretty(&config).unwrap()
);
let packet = serde_json::to_string(&config).unwrap();
socket.write_message(Message::Text(packet)).unwrap();
let shared_socket = Arc::new(Mutex::new(socket));
let app_sink = pipeline let app_sink = pipeline
.by_name("sink") .by_name("sink")
@ -42,15 +94,39 @@ fn main() -> eyre::Result<()> {
app_sink.set_callbacks( app_sink.set_callbacks(
gst_app::AppSinkCallbacks::builder() gst_app::AppSinkCallbacks::builder()
.new_sample({ .new_sample({
let raw_audio_content = raw_audio_content.clone(); let shared_socket = shared_socket.clone();
move |app| { move |app| {
let sample = app.pull_sample().unwrap(); let sample = app.pull_sample().unwrap();
let buffer = sample.buffer().unwrap(); let buffer = sample.buffer().unwrap();
let data = buffer.map_readable().unwrap(); let data = buffer.map_readable().unwrap();
let mut raw_audio_content = raw_audio_content.lock().unwrap(); let mut socket = shared_socket.lock().unwrap();
raw_audio_content.extend(data.to_vec());
for chunk in data.chunks(8_000) {
socket
.write_message(Message::Binary(chunk.to_vec()))
.unwrap();
let msg = socket.read_message().unwrap();
match msg {
Message::Text(payload) => {
match serde_json::from_str::<Transcript>(&payload) {
Ok(transcript) => {
info!(
"result: {}",
serde_json::to_string_pretty(&transcript.result)
.unwrap()
);
}
Err(_) => {
// The payload is still not a final transcript, so we just ignore it
// info!("No results...");
}
}
}
_ => {}
};
}
Ok(gst::FlowSuccess::Ok) Ok(gst::FlowSuccess::Ok)
} }
@ -102,27 +178,27 @@ fn main() -> eyre::Result<()> {
pipeline.set_state(gst::State::Null)?; pipeline.set_state(gst::State::Null)?;
let mut out_wav_buffer = Cursor::new(Vec::new()); // let mut out_wav_buffer = Cursor::new(Vec::new());
let mut writer = WavWriter::new( // let mut writer = WavWriter::new(
&mut out_wav_buffer, // &mut out_wav_buffer,
WavSpec { // WavSpec {
channels: 1, // channels: 1,
sample_rate: 48000, // sample_rate: 48000,
bits_per_sample: 16, // bits_per_sample: 16,
sample_format: hound::SampleFormat::Int, // sample_format: hound::SampleFormat::Int,
}, // },
) // )
.unwrap(); // .unwrap();
//
let mut raw_audio_content = Cursor::new(raw_audio_content.lock().unwrap().to_vec()); // let mut raw_audio_content = Cursor::new(raw_audio_content.lock().unwrap().to_vec());
//
while let Ok(sample) = raw_audio_content.read_i16::<LittleEndian>() { // while let Ok(sample) = raw_audio_content.read_i16::<LittleEndian>() {
writer.write_sample(sample).unwrap(); // writer.write_sample(sample).unwrap();
} // }
//
drop(writer); // drop(writer);
let mut file = File::create("out.wav")?; // let mut file = File::create("out.raw")?;
file.write_all(&out_wav_buffer.into_inner())?; // file.write_all(&raw_audio_content.into_inner())?;
Ok(()) Ok(())
} }