aws: s3sink: Fix handling of special characters in key

Properly URL-encode the string if needed, and add some tests for a
couple of cases.

Fixes: https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/431
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1333>
This commit is contained in:
Arun Raghavan 2023-09-20 18:17:41 -04:00
parent 829469d0fe
commit 51129febeb
2 changed files with 80 additions and 61 deletions

View file

@ -116,12 +116,13 @@ struct Settings {
impl Settings { impl Settings {
fn to_uri(&self) -> String { fn to_uri(&self) -> String {
format!( GstS3Url {
"s3://{}/{}/{}", region: self.region.clone(),
self.region, bucket: self.bucket.clone().unwrap(),
self.bucket.as_ref().unwrap(), object: self.key.clone().unwrap(),
self.key.as_ref().unwrap() version: None,
) }
.to_string()
} }
fn to_metadata(&self, imp: &S3Sink) -> Option<HashMap<String, String>> { fn to_metadata(&self, imp: &S3Sink) -> Option<HashMap<String, String>> {

View file

@ -8,6 +8,12 @@
// SPDX-License-Identifier: MPL-2.0 // SPDX-License-Identifier: MPL-2.0
// //
// The test times out on Windows for some reason, skip until we figure out why
#[cfg(not(target_os = "windows"))]
#[test_with::env(AWS_ACCESS_KEY_ID)]
#[test_with::env(AWS_SECRET_ACCESS_KEY)]
#[cfg(test)]
mod tests {
use gst::prelude::*; use gst::prelude::*;
const DEFAULT_S3_REGION: &str = "us-west-2"; const DEFAULT_S3_REGION: &str = "us-west-2";
@ -19,23 +25,19 @@ fn init() {
INIT.call_once(|| { INIT.call_once(|| {
gst::init().unwrap(); gst::init().unwrap();
gstaws::plugin_register_static().unwrap(); gstaws::plugin_register_static().unwrap();
// Makes it easier to get AWS SDK logs if needed
env_logger::init();
}); });
} }
// The test times out on Windows for some reason, skip until we figure out why // Common helper
#[cfg(not(target_os = "windows"))] async fn do_s3_test(key_prefix: &str) {
#[test_with::env(AWS_ACCESS_KEY_ID)]
#[test_with::env(AWS_SECRET_ACCESS_KEY)]
#[tokio::test]
async fn test_s3() {
init(); init();
// Makes it easier to get AWS SDK logs if needed
env_logger::init();
let region = std::env::var("AWS_REGION").unwrap_or_else(|_| DEFAULT_S3_REGION.to_string()); let region = std::env::var("AWS_REGION").unwrap_or_else(|_| DEFAULT_S3_REGION.to_string());
let bucket = let bucket =
std::env::var("AWS_S3_BUCKET").unwrap_or_else(|_| "gst-plugins-rs-tests".to_string()); std::env::var("AWS_S3_BUCKET").unwrap_or_else(|_| "gst-plugins-rs-tests".to_string());
let key = format!("s3-test-{:?}.txt", chrono::Utc::now()); let key = format!("{key_prefix}-{:?}.txt", chrono::Utc::now());
let uri = format!("s3://{region}/{bucket}/{key}"); let uri = format!("s3://{region}/{bucket}/{key}");
let content = "Hello, world!\n".as_bytes(); let content = "Hello, world!\n".as_bytes();
@ -43,7 +45,7 @@ async fn test_s3() {
let mut h1 = gst_check::Harness::new_empty(); let mut h1 = gst_check::Harness::new_empty();
// Need to add_parse() because the Harness API / Rust bindings aren't conducive to creating and // Need to add_parse() because the Harness API / Rust bindings aren't conducive to creating and
// adding an element manually // adding an element manually
h1.add_parse(format!("awss3sink uri={uri}").as_str()); h1.add_parse(format!("awss3sink uri=\"{uri}\"").as_str());
h1.set_src_caps(gst::Caps::builder("text/plain").build()); h1.set_src_caps(gst::Caps::builder("text/plain").build());
h1.play(); h1.play();
@ -77,3 +79,19 @@ async fn test_s3() {
.await .await
.unwrap(); .unwrap();
} }
#[tokio::test]
async fn test_s3_simple() {
do_s3_test("s3-test").await;
}
#[tokio::test]
async fn test_s3_whitespace() {
do_s3_test("s3 test").await;
}
#[tokio::test]
async fn test_s3_unicode() {
do_s3_test("s3 🧪 😱").await;
}
}