mirrors/gst-plugins-rs
mirrors/gst-plugins-rs
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git synced 2024-11-26 05:21:00 +00:00
Code Issues Projects Releases Wiki Activity

aws: Introduce a property to use path-style addressing

Browse source 
AWS SDK switched to virtual addressing as default instead of path
style earlier. While MinIO supports virtual host style requests,
path style requests are the default.

Introduce a property to allow the use of path style addressing if
required.

For more information, see
https://github.com/minio/minio/blob/master/docs/config/README.md#domain
https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1527>
This commit is contained in:
Sanchayan Maity 2024-04-05 13:43:10 +05:30 committed by Arun Raghavan
parent 2ad452ee89
commit a3e30b499f
5 changed files with 100 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
docs/plugins/gst_plugins_cache.json
View file

@ -79,6 +79,18 @@
"type": "gchararray", "type": "gchararray",
"writable": true "writable": true
}, },
"force-path-style": {
"blurb": "Force client to use path-style addressing for buckets",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "false",
"mutable": "null",
"readable": true,
"type": "gboolean",
"writable": true
},
"hlssink": { "hlssink": {
"blurb": "The underlying HLS sink being used", "blurb": "The underlying HLS sink being used",
"conditionally-available": false, "conditionally-available": false,
@ -315,6 +327,18 @@
"type": "gboolean", "type": "gboolean",
"writable": true "writable": true
}, },
"force-path-style": {
"blurb": "Force client to use path-style addressing for buckets",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "false",
"mutable": "null",
"readable": true,
"type": "gboolean",
"writable": true
},
"key": { "key": {
"blurb": "The key of the file to write", "blurb": "The key of the file to write",
"conditionally-available": false, "conditionally-available": false,
@ -529,6 +553,18 @@
"type": "gchararray", "type": "gchararray",
"writable": true "writable": true
}, },
"force-path-style": {
"blurb": "Force client to use path-style addressing for buckets",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "false",
"mutable": "null",
"readable": true,
"type": "gboolean",
"writable": true
},
"key": { "key": {
"blurb": "The key of the file to write", "blurb": "The key of the file to write",
"conditionally-available": false, "conditionally-available": false,
@ -747,6 +783,18 @@
"type": "gchararray", "type": "gchararray",
"writable": true "writable": true
}, },
"force-path-style": {
"blurb": "Force client to use path-style addressing for buckets",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "false",
"mutable": "null",
"readable": true,
"type": "gboolean",
"writable": true
},
"request-timeout": { "request-timeout": {
"blurb": "Timeout for each S3 request (in ms, set to -1 for infinity)", "blurb": "Timeout for each S3 request (in ms, set to -1 for infinity)",
"conditionally-available": false, "conditionally-available": false,

13
net/aws/src/s3hlssink/imp.rs
View file

@ -39,6 +39,7 @@ const S3_CHANNEL_SIZE: usize = 32;
const S3_ACL_DEFAULT: ObjectCannedAcl = ObjectCannedAcl::Private; const S3_ACL_DEFAULT: ObjectCannedAcl = ObjectCannedAcl::Private;
const DEFAULT_RETRY_ATTEMPTS: u32 = 5; const DEFAULT_RETRY_ATTEMPTS: u32 = 5;
const DEFAULT_TIMEOUT_IN_MSECS: u64 = 15000; const DEFAULT_TIMEOUT_IN_MSECS: u64 = 15000;
const DEFAULT_FORCE_PATH_STYLE: bool = false;
struct Settings { struct Settings {
access_key: Option<String>, access_key: Option<String>,
@ -57,6 +58,7 @@ struct Settings {
video_sink: bool, video_sink: bool,
config: Option<SdkConfig>, config: Option<SdkConfig>,
endpoint_uri: Option<String>, endpoint_uri: Option<String>,
force_path_style: bool,
} }
impl Default for Settings { impl Default for Settings {
@ -79,6 +81,7 @@ impl Default for Settings {
video_sink: false, video_sink: false,
config: None, config: None,
endpoint_uri: None, endpoint_uri: None,
force_path_style: DEFAULT_FORCE_PATH_STYLE,
} }
} }
} }
@ -376,6 +379,7 @@ impl S3HlsSink {
let sdk_config = settings.config.as_ref().expect("SDK config must be set"); let sdk_config = settings.config.as_ref().expect("SDK config must be set");
let config_builder = config::Builder::from(sdk_config) let config_builder = config::Builder::from(sdk_config)
.force_path_style(settings.force_path_style)
.region(settings.s3_region.clone()) .region(settings.s3_region.clone())
.retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts)); .retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts));
@ -529,6 +533,11 @@ impl ObjectImpl for S3HlsSink {
.blurb("The S3 endpoint URI to use") .blurb("The S3 endpoint URI to use")
.mutable_ready() .mutable_ready()
.build(), .build(),
glib::ParamSpecBoolean::builder("force-path-style")
.nick("Force path style")
.blurb("Force client to use path-style addressing for buckets")
.default_value(DEFAULT_FORCE_PATH_STYLE)
.build(),
] ]
}); });
@ -586,6 +595,9 @@ impl ObjectImpl for S3HlsSink {
.get::<Option<String>>() .get::<Option<String>>()
.expect("type checked upstream"); .expect("type checked upstream");
} }
"force-path-style" => {
settings.force_path_style = value.get::<bool>().expect("type checked upstream");
}
_ => unimplemented!(), _ => unimplemented!(),
} }
} }
@ -606,6 +618,7 @@ impl ObjectImpl for S3HlsSink {
"request-timeout" => (settings.request_timeout.as_millis() as u64).to_value(), "request-timeout" => (settings.request_timeout.as_millis() as u64).to_value(),
"stats" => self.create_stats().to_value(), "stats" => self.create_stats().to_value(),
"endpoint-uri" => settings.endpoint_uri.to_value(), "endpoint-uri" => settings.endpoint_uri.to_value(),
"force-path-style" => settings.force_path_style.to_value(),
_ => unimplemented!(), _ => unimplemented!(),
} }
} }

13
net/aws/src/s3sink/multipartsink.rs
View file

@ -38,6 +38,7 @@ use crate::s3utils::{self, duration_from_millis, duration_to_millis, WaitError};
use super::OnError; use super::OnError;
const DEFAULT_FORCE_PATH_STYLE: bool = false;
const DEFAULT_RETRY_ATTEMPTS: u32 = 5; const DEFAULT_RETRY_ATTEMPTS: u32 = 5;
const DEFAULT_BUFFER_SIZE: u64 = 5 * 1024 * 1024; const DEFAULT_BUFFER_SIZE: u64 = 5 * 1024 * 1024;
const DEFAULT_MULTIPART_UPLOAD_ON_ERROR: OnError = OnError::DoNothing; const DEFAULT_MULTIPART_UPLOAD_ON_ERROR: OnError = OnError::DoNothing;
@ -114,6 +115,7 @@ struct Settings {
multipart_upload_on_error: OnError, multipart_upload_on_error: OnError,
request_timeout: Duration, request_timeout: Duration,
endpoint_uri: Option<String>, endpoint_uri: Option<String>,
force_path_style: bool,
} }
impl Settings { impl Settings {
@ -168,6 +170,7 @@ impl Default for Settings {
multipart_upload_on_error: DEFAULT_MULTIPART_UPLOAD_ON_ERROR, multipart_upload_on_error: DEFAULT_MULTIPART_UPLOAD_ON_ERROR,
request_timeout: Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MSEC), request_timeout: Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MSEC),
endpoint_uri: None, endpoint_uri: None,
force_path_style: DEFAULT_FORCE_PATH_STYLE,
} }
} }
} }
@ -524,6 +527,7 @@ impl S3Sink {
})?; })?;
let config_builder = config::Builder::from(&sdk_config) let config_builder = config::Builder::from(&sdk_config)
.force_path_style(settings.force_path_style)
.retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts)); .retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts));
let config = if let Some(ref uri) = settings.endpoint_uri { let config = if let Some(ref uri) = settings.endpoint_uri {
@ -775,6 +779,11 @@ impl ObjectImpl for S3Sink {
.nick("content-disposition") .nick("content-disposition")
.blurb("Content-Disposition header to set for uploaded object") .blurb("Content-Disposition header to set for uploaded object")
.build(), .build(),
glib::ParamSpecBoolean::builder("force-path-style")
.nick("Force path style")
.blurb("Force client to use path-style addressing for buckets")
.default_value(DEFAULT_FORCE_PATH_STYLE)
.build(),
] ]
}); });
@ -888,6 +897,9 @@ impl ObjectImpl for S3Sink {
.get::<Option<String>>() .get::<Option<String>>()
.expect("type checked upstream"); .expect("type checked upstream");
} }
"force-path-style" => {
settings.force_path_style = value.get::<bool>().expect("type checked upstream");
}
_ => unimplemented!(), _ => unimplemented!(),
} }
} }
@ -929,6 +941,7 @@ impl ObjectImpl for S3Sink {
"endpoint-uri" => settings.endpoint_uri.to_value(), "endpoint-uri" => settings.endpoint_uri.to_value(),
"content-type" => settings.content_type.to_value(), "content-type" => settings.content_type.to_value(),
"content-disposition" => settings.content_disposition.to_value(), "content-disposition" => settings.content_disposition.to_value(),
"force-path-style" => settings.force_path_style.to_value(),
_ => unimplemented!(), _ => unimplemented!(),
} }
} }

13
net/aws/src/s3sink/putobjectsink.rs
View file

@ -36,6 +36,7 @@ const DEFAULT_FLUSH_INTERVAL_BUFFERS: u64 = 1;
const DEFAULT_FLUSH_INTERVAL_BYTES: u64 = 0; const DEFAULT_FLUSH_INTERVAL_BYTES: u64 = 0;
const DEFAULT_FLUSH_INTERVAL_TIME: gst::ClockTime = gst::ClockTime::from_nseconds(0); const DEFAULT_FLUSH_INTERVAL_TIME: gst::ClockTime = gst::ClockTime::from_nseconds(0);
const DEFAULT_FLUSH_ON_ERROR: bool = false; const DEFAULT_FLUSH_ON_ERROR: bool = false;
const DEFAULT_FORCE_PATH_STYLE: bool = false;
// General setting for create / abort requests // General setting for create / abort requests
const DEFAULT_REQUEST_TIMEOUT_MSEC: u64 = 15_000; const DEFAULT_REQUEST_TIMEOUT_MSEC: u64 = 15_000;
@ -80,6 +81,7 @@ struct Settings {
retry_attempts: u32, retry_attempts: u32,
request_timeout: Duration, request_timeout: Duration,
endpoint_uri: Option<String>, endpoint_uri: Option<String>,
force_path_style: bool,
flush_interval_buffers: u64, flush_interval_buffers: u64,
flush_interval_bytes: u64, flush_interval_bytes: u64,
flush_interval_time: Option<gst::ClockTime>, flush_interval_time: Option<gst::ClockTime>,
@ -136,6 +138,7 @@ impl Default for Settings {
retry_attempts: DEFAULT_RETRY_ATTEMPTS, retry_attempts: DEFAULT_RETRY_ATTEMPTS,
request_timeout: Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MSEC), request_timeout: Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MSEC),
endpoint_uri: None, endpoint_uri: None,
force_path_style: DEFAULT_FORCE_PATH_STYLE,
flush_interval_buffers: DEFAULT_FLUSH_INTERVAL_BUFFERS, flush_interval_buffers: DEFAULT_FLUSH_INTERVAL_BUFFERS,
flush_interval_bytes: DEFAULT_FLUSH_INTERVAL_BYTES, flush_interval_bytes: DEFAULT_FLUSH_INTERVAL_BYTES,
flush_interval_time: Some(DEFAULT_FLUSH_INTERVAL_TIME), flush_interval_time: Some(DEFAULT_FLUSH_INTERVAL_TIME),
@ -293,6 +296,7 @@ impl S3PutObjectSink {
})?; })?;
let config_builder = config::Builder::from(&sdk_config) let config_builder = config::Builder::from(&sdk_config)
.force_path_style(settings.force_path_style)
.retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts)); .retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts));
let config = if let Some(ref uri) = settings.endpoint_uri { let config = if let Some(ref uri) = settings.endpoint_uri {
@ -446,6 +450,11 @@ impl ObjectImpl for S3PutObjectSink {
.blurb("Whether to write out the data on error (like stopping without an EOS)") .blurb("Whether to write out the data on error (like stopping without an EOS)")
.default_value(DEFAULT_FLUSH_ON_ERROR) .default_value(DEFAULT_FLUSH_ON_ERROR)
.build(), .build(),
glib::ParamSpecBoolean::builder("force-path-style")
.nick("Force path style")
.blurb("Force client to use path-style addressing for buckets")
.default_value(DEFAULT_FORCE_PATH_STYLE)
.build(),
] ]
}); });
@ -542,6 +551,9 @@ impl ObjectImpl for S3PutObjectSink {
"flush-on-error" => { "flush-on-error" => {
settings.flush_on_error = value.get::<bool>().expect("type checked upstream"); settings.flush_on_error = value.get::<bool>().expect("type checked upstream");
} }
"force-path-style" => {
settings.force_path_style = value.get::<bool>().expect("type checked upstream");
}
_ => unimplemented!(), _ => unimplemented!(),
} }
} }
@ -575,6 +587,7 @@ impl ObjectImpl for S3PutObjectSink {
"flush-interval-bytes" => settings.flush_interval_bytes.to_value(), "flush-interval-bytes" => settings.flush_interval_bytes.to_value(),
"flush-interval-time" => settings.flush_interval_time.to_value(), "flush-interval-time" => settings.flush_interval_time.to_value(),
"flush-on-error" => settings.flush_on_error.to_value(), "flush-on-error" => settings.flush_on_error.to_value(),
"force-path-style" => settings.force_path_style.to_value(),
_ => unimplemented!(), _ => unimplemented!(),
} }
} }

13
net/aws/src/s3src/imp.rs
View file

@ -29,6 +29,7 @@ use gst_base::subclass::prelude::*;
use crate::s3url::*; use crate::s3url::*;
use crate::s3utils::{self, duration_from_millis, duration_to_millis, WaitError}; use crate::s3utils::{self, duration_from_millis, duration_to_millis, WaitError};
const DEFAULT_FORCE_PATH_STYLE: bool = false;
const DEFAULT_RETRY_ATTEMPTS: u32 = 5; const DEFAULT_RETRY_ATTEMPTS: u32 = 5;
const DEFAULT_REQUEST_TIMEOUT_MSEC: u64 = 15000; const DEFAULT_REQUEST_TIMEOUT_MSEC: u64 = 15000;
const DEFAULT_RETRY_DURATION_MSEC: u64 = 60_000; const DEFAULT_RETRY_DURATION_MSEC: u64 = 60_000;
@ -53,6 +54,7 @@ struct Settings {
retry_attempts: u32, retry_attempts: u32,
request_timeout: Duration, request_timeout: Duration,
endpoint_uri: Option<String>, endpoint_uri: Option<String>,
force_path_style: bool,
} }
impl Default for Settings { impl Default for Settings {
@ -66,6 +68,7 @@ impl Default for Settings {
retry_attempts: DEFAULT_RETRY_ATTEMPTS, retry_attempts: DEFAULT_RETRY_ATTEMPTS,
request_timeout: duration, request_timeout: duration,
endpoint_uri: None, endpoint_uri: None,
force_path_style: DEFAULT_FORCE_PATH_STYLE,
} }
} }
} }
@ -128,6 +131,7 @@ impl S3Src {
})?; })?;
let config_builder = config::Builder::from(&sdk_config) let config_builder = config::Builder::from(&sdk_config)
.force_path_style(settings.force_path_style)
.retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts)); .retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts));
let config = if let Some(ref uri) = settings.endpoint_uri { let config = if let Some(ref uri) = settings.endpoint_uri {
@ -316,6 +320,11 @@ impl ObjectImpl for S3Src {
.nick("S3 endpoint URI") .nick("S3 endpoint URI")
.blurb("The S3 endpoint URI to use") .blurb("The S3 endpoint URI to use")
.build(), .build(),
glib::ParamSpecBoolean::builder("force-path-style")
.nick("Force path style")
.blurb("Force client to use path-style addressing for buckets")
.default_value(DEFAULT_FORCE_PATH_STYLE)
.build(),
] ]
}); });
@ -365,6 +374,9 @@ impl ObjectImpl for S3Src {
.get::<Option<String>>() .get::<Option<String>>()
.expect("type checked upstream"); .expect("type checked upstream");
} }
"force-path-style" => {
settings.force_path_style = value.get::<bool>().expect("type checked upstream");
}
_ => unimplemented!(), _ => unimplemented!(),
} }
} }
@ -391,6 +403,7 @@ impl ObjectImpl for S3Src {
} }
"retry-attempts" => settings.retry_attempts.to_value(), "retry-attempts" => settings.retry_attempts.to_value(),
"endpoint-uri" => settings.endpoint_uri.to_value(), "endpoint-uri" => settings.endpoint_uri.to_value(),
"force-path-style" => settings.force_path_style.to_value(),
_ => unimplemented!(), _ => unimplemented!(),
} }
} }