aws: Introduce a property to use path-style addressing

AWS SDK switched to virtual addressing as default instead of path
style earlier. While MinIO supports virtual host style requests,
path style requests are the default.

Introduce a property to allow the use of path style addressing if
required.

For more information, see
https://github.com/minio/minio/blob/master/docs/config/README.md#domain
https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1527>
This commit is contained in:
Sanchayan Maity 2024-04-05 13:43:10 +05:30 committed by Arun Raghavan
parent 2ad452ee89
commit a3e30b499f
5 changed files with 100 additions and 0 deletions

View file

@ -79,6 +79,18 @@
"type": "gchararray", "type": "gchararray",
"writable": true "writable": true
}, },
"force-path-style": {
"blurb": "Force client to use path-style addressing for buckets",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "false",
"mutable": "null",
"readable": true,
"type": "gboolean",
"writable": true
},
"hlssink": { "hlssink": {
"blurb": "The underlying HLS sink being used", "blurb": "The underlying HLS sink being used",
"conditionally-available": false, "conditionally-available": false,
@ -315,6 +327,18 @@
"type": "gboolean", "type": "gboolean",
"writable": true "writable": true
}, },
"force-path-style": {
"blurb": "Force client to use path-style addressing for buckets",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "false",
"mutable": "null",
"readable": true,
"type": "gboolean",
"writable": true
},
"key": { "key": {
"blurb": "The key of the file to write", "blurb": "The key of the file to write",
"conditionally-available": false, "conditionally-available": false,
@ -529,6 +553,18 @@
"type": "gchararray", "type": "gchararray",
"writable": true "writable": true
}, },
"force-path-style": {
"blurb": "Force client to use path-style addressing for buckets",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "false",
"mutable": "null",
"readable": true,
"type": "gboolean",
"writable": true
},
"key": { "key": {
"blurb": "The key of the file to write", "blurb": "The key of the file to write",
"conditionally-available": false, "conditionally-available": false,
@ -747,6 +783,18 @@
"type": "gchararray", "type": "gchararray",
"writable": true "writable": true
}, },
"force-path-style": {
"blurb": "Force client to use path-style addressing for buckets",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "false",
"mutable": "null",
"readable": true,
"type": "gboolean",
"writable": true
},
"request-timeout": { "request-timeout": {
"blurb": "Timeout for each S3 request (in ms, set to -1 for infinity)", "blurb": "Timeout for each S3 request (in ms, set to -1 for infinity)",
"conditionally-available": false, "conditionally-available": false,

View file

@ -39,6 +39,7 @@ const S3_CHANNEL_SIZE: usize = 32;
const S3_ACL_DEFAULT: ObjectCannedAcl = ObjectCannedAcl::Private; const S3_ACL_DEFAULT: ObjectCannedAcl = ObjectCannedAcl::Private;
const DEFAULT_RETRY_ATTEMPTS: u32 = 5; const DEFAULT_RETRY_ATTEMPTS: u32 = 5;
const DEFAULT_TIMEOUT_IN_MSECS: u64 = 15000; const DEFAULT_TIMEOUT_IN_MSECS: u64 = 15000;
const DEFAULT_FORCE_PATH_STYLE: bool = false;
struct Settings { struct Settings {
access_key: Option<String>, access_key: Option<String>,
@ -57,6 +58,7 @@ struct Settings {
video_sink: bool, video_sink: bool,
config: Option<SdkConfig>, config: Option<SdkConfig>,
endpoint_uri: Option<String>, endpoint_uri: Option<String>,
force_path_style: bool,
} }
impl Default for Settings { impl Default for Settings {
@ -79,6 +81,7 @@ impl Default for Settings {
video_sink: false, video_sink: false,
config: None, config: None,
endpoint_uri: None, endpoint_uri: None,
force_path_style: DEFAULT_FORCE_PATH_STYLE,
} }
} }
} }
@ -376,6 +379,7 @@ impl S3HlsSink {
let sdk_config = settings.config.as_ref().expect("SDK config must be set"); let sdk_config = settings.config.as_ref().expect("SDK config must be set");
let config_builder = config::Builder::from(sdk_config) let config_builder = config::Builder::from(sdk_config)
.force_path_style(settings.force_path_style)
.region(settings.s3_region.clone()) .region(settings.s3_region.clone())
.retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts)); .retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts));
@ -529,6 +533,11 @@ impl ObjectImpl for S3HlsSink {
.blurb("The S3 endpoint URI to use") .blurb("The S3 endpoint URI to use")
.mutable_ready() .mutable_ready()
.build(), .build(),
glib::ParamSpecBoolean::builder("force-path-style")
.nick("Force path style")
.blurb("Force client to use path-style addressing for buckets")
.default_value(DEFAULT_FORCE_PATH_STYLE)
.build(),
] ]
}); });
@ -586,6 +595,9 @@ impl ObjectImpl for S3HlsSink {
.get::<Option<String>>() .get::<Option<String>>()
.expect("type checked upstream"); .expect("type checked upstream");
} }
"force-path-style" => {
settings.force_path_style = value.get::<bool>().expect("type checked upstream");
}
_ => unimplemented!(), _ => unimplemented!(),
} }
} }
@ -606,6 +618,7 @@ impl ObjectImpl for S3HlsSink {
"request-timeout" => (settings.request_timeout.as_millis() as u64).to_value(), "request-timeout" => (settings.request_timeout.as_millis() as u64).to_value(),
"stats" => self.create_stats().to_value(), "stats" => self.create_stats().to_value(),
"endpoint-uri" => settings.endpoint_uri.to_value(), "endpoint-uri" => settings.endpoint_uri.to_value(),
"force-path-style" => settings.force_path_style.to_value(),
_ => unimplemented!(), _ => unimplemented!(),
} }
} }

View file

@ -38,6 +38,7 @@ use crate::s3utils::{self, duration_from_millis, duration_to_millis, WaitError};
use super::OnError; use super::OnError;
const DEFAULT_FORCE_PATH_STYLE: bool = false;
const DEFAULT_RETRY_ATTEMPTS: u32 = 5; const DEFAULT_RETRY_ATTEMPTS: u32 = 5;
const DEFAULT_BUFFER_SIZE: u64 = 5 * 1024 * 1024; const DEFAULT_BUFFER_SIZE: u64 = 5 * 1024 * 1024;
const DEFAULT_MULTIPART_UPLOAD_ON_ERROR: OnError = OnError::DoNothing; const DEFAULT_MULTIPART_UPLOAD_ON_ERROR: OnError = OnError::DoNothing;
@ -114,6 +115,7 @@ struct Settings {
multipart_upload_on_error: OnError, multipart_upload_on_error: OnError,
request_timeout: Duration, request_timeout: Duration,
endpoint_uri: Option<String>, endpoint_uri: Option<String>,
force_path_style: bool,
} }
impl Settings { impl Settings {
@ -168,6 +170,7 @@ impl Default for Settings {
multipart_upload_on_error: DEFAULT_MULTIPART_UPLOAD_ON_ERROR, multipart_upload_on_error: DEFAULT_MULTIPART_UPLOAD_ON_ERROR,
request_timeout: Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MSEC), request_timeout: Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MSEC),
endpoint_uri: None, endpoint_uri: None,
force_path_style: DEFAULT_FORCE_PATH_STYLE,
} }
} }
} }
@ -524,6 +527,7 @@ impl S3Sink {
})?; })?;
let config_builder = config::Builder::from(&sdk_config) let config_builder = config::Builder::from(&sdk_config)
.force_path_style(settings.force_path_style)
.retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts)); .retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts));
let config = if let Some(ref uri) = settings.endpoint_uri { let config = if let Some(ref uri) = settings.endpoint_uri {
@ -775,6 +779,11 @@ impl ObjectImpl for S3Sink {
.nick("content-disposition") .nick("content-disposition")
.blurb("Content-Disposition header to set for uploaded object") .blurb("Content-Disposition header to set for uploaded object")
.build(), .build(),
glib::ParamSpecBoolean::builder("force-path-style")
.nick("Force path style")
.blurb("Force client to use path-style addressing for buckets")
.default_value(DEFAULT_FORCE_PATH_STYLE)
.build(),
] ]
}); });
@ -888,6 +897,9 @@ impl ObjectImpl for S3Sink {
.get::<Option<String>>() .get::<Option<String>>()
.expect("type checked upstream"); .expect("type checked upstream");
} }
"force-path-style" => {
settings.force_path_style = value.get::<bool>().expect("type checked upstream");
}
_ => unimplemented!(), _ => unimplemented!(),
} }
} }
@ -929,6 +941,7 @@ impl ObjectImpl for S3Sink {
"endpoint-uri" => settings.endpoint_uri.to_value(), "endpoint-uri" => settings.endpoint_uri.to_value(),
"content-type" => settings.content_type.to_value(), "content-type" => settings.content_type.to_value(),
"content-disposition" => settings.content_disposition.to_value(), "content-disposition" => settings.content_disposition.to_value(),
"force-path-style" => settings.force_path_style.to_value(),
_ => unimplemented!(), _ => unimplemented!(),
} }
} }

View file

@ -36,6 +36,7 @@ const DEFAULT_FLUSH_INTERVAL_BUFFERS: u64 = 1;
const DEFAULT_FLUSH_INTERVAL_BYTES: u64 = 0; const DEFAULT_FLUSH_INTERVAL_BYTES: u64 = 0;
const DEFAULT_FLUSH_INTERVAL_TIME: gst::ClockTime = gst::ClockTime::from_nseconds(0); const DEFAULT_FLUSH_INTERVAL_TIME: gst::ClockTime = gst::ClockTime::from_nseconds(0);
const DEFAULT_FLUSH_ON_ERROR: bool = false; const DEFAULT_FLUSH_ON_ERROR: bool = false;
const DEFAULT_FORCE_PATH_STYLE: bool = false;
// General setting for create / abort requests // General setting for create / abort requests
const DEFAULT_REQUEST_TIMEOUT_MSEC: u64 = 15_000; const DEFAULT_REQUEST_TIMEOUT_MSEC: u64 = 15_000;
@ -80,6 +81,7 @@ struct Settings {
retry_attempts: u32, retry_attempts: u32,
request_timeout: Duration, request_timeout: Duration,
endpoint_uri: Option<String>, endpoint_uri: Option<String>,
force_path_style: bool,
flush_interval_buffers: u64, flush_interval_buffers: u64,
flush_interval_bytes: u64, flush_interval_bytes: u64,
flush_interval_time: Option<gst::ClockTime>, flush_interval_time: Option<gst::ClockTime>,
@ -136,6 +138,7 @@ impl Default for Settings {
retry_attempts: DEFAULT_RETRY_ATTEMPTS, retry_attempts: DEFAULT_RETRY_ATTEMPTS,
request_timeout: Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MSEC), request_timeout: Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MSEC),
endpoint_uri: None, endpoint_uri: None,
force_path_style: DEFAULT_FORCE_PATH_STYLE,
flush_interval_buffers: DEFAULT_FLUSH_INTERVAL_BUFFERS, flush_interval_buffers: DEFAULT_FLUSH_INTERVAL_BUFFERS,
flush_interval_bytes: DEFAULT_FLUSH_INTERVAL_BYTES, flush_interval_bytes: DEFAULT_FLUSH_INTERVAL_BYTES,
flush_interval_time: Some(DEFAULT_FLUSH_INTERVAL_TIME), flush_interval_time: Some(DEFAULT_FLUSH_INTERVAL_TIME),
@ -293,6 +296,7 @@ impl S3PutObjectSink {
})?; })?;
let config_builder = config::Builder::from(&sdk_config) let config_builder = config::Builder::from(&sdk_config)
.force_path_style(settings.force_path_style)
.retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts)); .retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts));
let config = if let Some(ref uri) = settings.endpoint_uri { let config = if let Some(ref uri) = settings.endpoint_uri {
@ -446,6 +450,11 @@ impl ObjectImpl for S3PutObjectSink {
.blurb("Whether to write out the data on error (like stopping without an EOS)") .blurb("Whether to write out the data on error (like stopping without an EOS)")
.default_value(DEFAULT_FLUSH_ON_ERROR) .default_value(DEFAULT_FLUSH_ON_ERROR)
.build(), .build(),
glib::ParamSpecBoolean::builder("force-path-style")
.nick("Force path style")
.blurb("Force client to use path-style addressing for buckets")
.default_value(DEFAULT_FORCE_PATH_STYLE)
.build(),
] ]
}); });
@ -542,6 +551,9 @@ impl ObjectImpl for S3PutObjectSink {
"flush-on-error" => { "flush-on-error" => {
settings.flush_on_error = value.get::<bool>().expect("type checked upstream"); settings.flush_on_error = value.get::<bool>().expect("type checked upstream");
} }
"force-path-style" => {
settings.force_path_style = value.get::<bool>().expect("type checked upstream");
}
_ => unimplemented!(), _ => unimplemented!(),
} }
} }
@ -575,6 +587,7 @@ impl ObjectImpl for S3PutObjectSink {
"flush-interval-bytes" => settings.flush_interval_bytes.to_value(), "flush-interval-bytes" => settings.flush_interval_bytes.to_value(),
"flush-interval-time" => settings.flush_interval_time.to_value(), "flush-interval-time" => settings.flush_interval_time.to_value(),
"flush-on-error" => settings.flush_on_error.to_value(), "flush-on-error" => settings.flush_on_error.to_value(),
"force-path-style" => settings.force_path_style.to_value(),
_ => unimplemented!(), _ => unimplemented!(),
} }
} }

View file

@ -29,6 +29,7 @@ use gst_base::subclass::prelude::*;
use crate::s3url::*; use crate::s3url::*;
use crate::s3utils::{self, duration_from_millis, duration_to_millis, WaitError}; use crate::s3utils::{self, duration_from_millis, duration_to_millis, WaitError};
const DEFAULT_FORCE_PATH_STYLE: bool = false;
const DEFAULT_RETRY_ATTEMPTS: u32 = 5; const DEFAULT_RETRY_ATTEMPTS: u32 = 5;
const DEFAULT_REQUEST_TIMEOUT_MSEC: u64 = 15000; const DEFAULT_REQUEST_TIMEOUT_MSEC: u64 = 15000;
const DEFAULT_RETRY_DURATION_MSEC: u64 = 60_000; const DEFAULT_RETRY_DURATION_MSEC: u64 = 60_000;
@ -53,6 +54,7 @@ struct Settings {
retry_attempts: u32, retry_attempts: u32,
request_timeout: Duration, request_timeout: Duration,
endpoint_uri: Option<String>, endpoint_uri: Option<String>,
force_path_style: bool,
} }
impl Default for Settings { impl Default for Settings {
@ -66,6 +68,7 @@ impl Default for Settings {
retry_attempts: DEFAULT_RETRY_ATTEMPTS, retry_attempts: DEFAULT_RETRY_ATTEMPTS,
request_timeout: duration, request_timeout: duration,
endpoint_uri: None, endpoint_uri: None,
force_path_style: DEFAULT_FORCE_PATH_STYLE,
} }
} }
} }
@ -128,6 +131,7 @@ impl S3Src {
})?; })?;
let config_builder = config::Builder::from(&sdk_config) let config_builder = config::Builder::from(&sdk_config)
.force_path_style(settings.force_path_style)
.retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts)); .retry_config(RetryConfig::standard().with_max_attempts(settings.retry_attempts));
let config = if let Some(ref uri) = settings.endpoint_uri { let config = if let Some(ref uri) = settings.endpoint_uri {
@ -316,6 +320,11 @@ impl ObjectImpl for S3Src {
.nick("S3 endpoint URI") .nick("S3 endpoint URI")
.blurb("The S3 endpoint URI to use") .blurb("The S3 endpoint URI to use")
.build(), .build(),
glib::ParamSpecBoolean::builder("force-path-style")
.nick("Force path style")
.blurb("Force client to use path-style addressing for buckets")
.default_value(DEFAULT_FORCE_PATH_STYLE)
.build(),
] ]
}); });
@ -365,6 +374,9 @@ impl ObjectImpl for S3Src {
.get::<Option<String>>() .get::<Option<String>>()
.expect("type checked upstream"); .expect("type checked upstream");
} }
"force-path-style" => {
settings.force_path_style = value.get::<bool>().expect("type checked upstream");
}
_ => unimplemented!(), _ => unimplemented!(),
} }
} }
@ -391,6 +403,7 @@ impl ObjectImpl for S3Src {
} }
"retry-attempts" => settings.retry_attempts.to_value(), "retry-attempts" => settings.retry_attempts.to_value(),
"endpoint-uri" => settings.endpoint_uri.to_value(), "endpoint-uri" => settings.endpoint_uri.to_value(),
"force-path-style" => settings.force_path_style.to_value(),
_ => unimplemented!(), _ => unimplemented!(),
} }
} }