Merge branch 'main' into internals-rework

This commit is contained in:
Alex Auvolat 2023-01-02 15:07:44 +01:00
commit 939a6d67e8
No known key found for this signature in database
GPG key ID: 0E496D15096376BE
18 changed files with 332 additions and 268 deletions

18
Cargo.lock generated
View file

@ -1048,7 +1048,7 @@ dependencies = [
[[package]]
name = "garage"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"assert-json-diff",
"async-trait",
@ -1096,7 +1096,7 @@ dependencies = [
[[package]]
name = "garage_api"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"async-trait",
"base64",
@ -1141,7 +1141,7 @@ dependencies = [
[[package]]
name = "garage_block"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"arc-swap",
"async-compression",
@ -1167,7 +1167,7 @@ dependencies = [
[[package]]
name = "garage_db"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"clap 3.1.18",
"err-derive",
@ -1182,7 +1182,7 @@ dependencies = [
[[package]]
name = "garage_model"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"arc-swap",
"async-trait",
@ -1210,7 +1210,7 @@ dependencies = [
[[package]]
name = "garage_rpc"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"arc-swap",
"async-trait",
@ -1241,7 +1241,7 @@ dependencies = [
[[package]]
name = "garage_table"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"arc-swap",
"async-trait",
@ -1264,7 +1264,7 @@ dependencies = [
[[package]]
name = "garage_util"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"arc-swap",
"async-trait",
@ -1295,7 +1295,7 @@ dependencies = [
[[package]]
name = "garage_web"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"err-derive",
"futures",

120
Cargo.nix
View file

@ -32,7 +32,7 @@ args@{
ignoreLockHash,
}:
let
nixifiedLockHash = "a1d84930f23d3d8abc8abbed59b8ce3c9adf9f25d06bc1f39cbdf5bd90aceead";
nixifiedLockHash = "4639f63ff4c54c01f66ec3d0d362f6905456dd768d6e94df1a7367c763721fd7";
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
lockHashIgnored = if ignoreLockHash
@ -56,15 +56,15 @@ in
{
cargo2nixVersion = "0.11.0";
workspace = {
garage_db = rustPackages.unknown.garage_db."0.8.0";
garage_util = rustPackages.unknown.garage_util."0.8.0";
garage_rpc = rustPackages.unknown.garage_rpc."0.8.0";
garage_table = rustPackages.unknown.garage_table."0.8.0";
garage_block = rustPackages.unknown.garage_block."0.8.0";
garage_model = rustPackages.unknown.garage_model."0.8.0";
garage_api = rustPackages.unknown.garage_api."0.8.0";
garage_web = rustPackages.unknown.garage_web."0.8.0";
garage = rustPackages.unknown.garage."0.8.0";
garage_db = rustPackages.unknown.garage_db."0.8.1";
garage_util = rustPackages.unknown.garage_util."0.8.1";
garage_rpc = rustPackages.unknown.garage_rpc."0.8.1";
garage_table = rustPackages.unknown.garage_table."0.8.1";
garage_block = rustPackages.unknown.garage_block."0.8.1";
garage_model = rustPackages.unknown.garage_model."0.8.1";
garage_api = rustPackages.unknown.garage_api."0.8.1";
garage_web = rustPackages.unknown.garage_web."0.8.1";
garage = rustPackages.unknown.garage."0.8.1";
k2v-client = rustPackages.unknown.k2v-client."0.0.1";
};
"registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec {
@ -1494,9 +1494,9 @@ in
};
});
"unknown".garage."0.8.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage";
version = "0.8.0";
version = "0.8.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/garage");
features = builtins.concatLists [
@ -1522,14 +1522,14 @@ in
bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
garage_web = (rustPackages."unknown".garage_web."0.8.0" { inherit profileName; }).out;
garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
garage_web = (rustPackages."unknown".garage_web."0.8.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
@ -1563,9 +1563,9 @@ in
};
});
"unknown".garage_api."0.8.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_api."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_api";
version = "0.8.0";
version = "0.8.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/api");
features = builtins.concatLists [
@ -1584,11 +1584,11 @@ in
form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
@ -1617,9 +1617,9 @@ in
};
});
"unknown".garage_block."0.8.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_block."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_block";
version = "0.8.0";
version = "0.8.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/block");
features = builtins.concatLists [
@ -1632,10 +1632,10 @@ in
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
@ -1649,9 +1649,9 @@ in
};
});
"unknown".garage_db."0.8.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_db."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_db";
version = "0.8.0";
version = "0.8.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/db");
features = builtins.concatLists [
@ -1681,9 +1681,9 @@ in
};
});
"unknown".garage_model."0.8.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_model."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_model";
version = "0.8.0";
version = "0.8.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/model");
features = builtins.concatLists [
@ -1701,11 +1701,11 @@ in
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@ -1719,9 +1719,9 @@ in
};
});
"unknown".garage_rpc."0.8.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_rpc."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_rpc";
version = "0.8.0";
version = "0.8.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/rpc");
features = builtins.concatLists [
@ -1741,7 +1741,7 @@ in
${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out;
@ -1763,9 +1763,9 @@ in
};
});
"unknown".garage_table."0.8.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_table."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_table";
version = "0.8.0";
version = "0.8.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/table");
dependencies = {
@ -1774,9 +1774,9 @@ in
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@ -1789,9 +1789,9 @@ in
};
});
"unknown".garage_util."0.8.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_util."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_util";
version = "0.8.0";
version = "0.8.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/util");
features = builtins.concatLists [
@ -1806,7 +1806,7 @@ in
digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.3" { inherit profileName; }).out;
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
@ -1826,18 +1826,18 @@ in
};
});
"unknown".garage_web."0.8.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_web."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_web";
version = "0.8.0";
version = "0.8.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/web");
dependencies = {
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@ -2451,7 +2451,7 @@ in
dependencies = {
base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; }).out;
${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }).out;
${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
rusoto_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }).out;

View file

@ -5,13 +5,56 @@ weight = 20
## S3
### Using Minio SDK
First install the SDK:
```bash
pip3 install minio
```
Then instantiate a client object using garage root domain, api key and secret:
```python
import minio
client = minio.Minio(
"your.domain.tld",
"GKyourapikey",
"abcd[...]1234",
# Force the region, this is specific to garage
region="region",
)
```
Then use all the standard S3 endpoints as implemented by the Minio SDK:
```
# List buckets
print(client.list_buckets())
# Put an object containing 'content' to /path in bucket named 'bucket':
content = b"content"
client.put_object(
"bucket",
"path",
io.BytesIO(content),
len(content),
)
# Read the object back and check contents
data = client.get_object("bucket", "path").read()
assert data == content
```
For further documentation, see the Minio SDK
[Reference](https://docs.min.io/docs/python-client-api-reference.html)
### Using Amazon boto3
*Coming soon*
Some refs:
- Minio SDK
- [Reference](https://docs.min.io/docs/python-client-api-reference.html)
- Amazon boto3
See the official documentation:
- [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
- [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
- [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)

View file

@ -8,7 +8,7 @@ In this section, we cover the following web applications:
| Name | Status | Note |
|------|--------|------|
| [Nextcloud](#nextcloud) | ✅ | Both Primary Storage and External Storage are supported |
| [Peertube](#peertube) | ✅ | Must be configured with the website endpoint |
| [Peertube](#peertube) | ✅ | Supported with the website endpoint, proxifying private videos unsupported |
| [Mastodon](#mastodon) | ✅ | Natively supported |
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
| [Pixelfed](#pixelfed) | ❓ | Not yet tested |
@ -128,6 +128,10 @@ In other words, Peertube is only responsible of the "control plane" and offload
In return, this system is a bit harder to configure.
We show how it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
Starting from version 5.0, Peertube also supports improving the security for private videos by not exposing them directly
but relying on a single control point in the Peertube instance. This is based on S3 per-object and prefix ACL, which are not currently supported
in Garage, so this feature is unsupported. While this technically impedes security for private videos, it is not a blocking issue and could be
a reasonable trade-off for some instances.
### Create resources in Garage
@ -195,6 +199,11 @@ object_storage:
max_upload_part: 2GB
proxy:
# You may enable this feature, yet it will not provide any security benefit, so
# you should rather benefit from Garage public endpoint for all videos
proxify_private_files: false
streaming_playlists:
bucket_name: 'peertube-playlist'

View file

@ -1,6 +1,6 @@
[package]
name = "garage_api"
version = "0.8.0"
version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -14,11 +14,11 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_model = { version = "0.8.0", path = "../model" }
garage_table = { version = "0.8.0", path = "../table" }
garage_block = { version = "0.8.0", path = "../block" }
garage_util = { version = "0.8.0", path = "../util" }
garage_rpc = { version = "0.8.0", path = "../rpc" }
garage_model = { version = "0.8.1", path = "../model" }
garage_table = { version = "0.8.1", path = "../table" }
garage_block = { version = "0.8.1", path = "../block" }
garage_util = { version = "0.8.1", path = "../util" }
garage_rpc = { version = "0.8.1", path = "../rpc" }
async-trait = "0.1.7"
base64 = "0.13"

View file

@ -143,10 +143,13 @@ impl Endpoint {
}
generateQueryParameters! {
keywords: [],
fields: [
"format" => format,
"id" => id,
"search" => search,
"globalAlias" => global_alias,
"alias" => alias,
"accessKeyId" => access_key_id
]
}

View file

@ -96,7 +96,7 @@ impl Endpoint {
fn from_get(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [
EMPTY if causality_token => PollItem (query::sort_key, query::causality_token, opt_parse::timeout),
EMPTY => ReadItem (query::sort_key),
@ -111,7 +111,7 @@ impl Endpoint {
fn from_search(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [
],
no_key: [
@ -125,7 +125,7 @@ impl Endpoint {
fn from_head(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [
EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
],
@ -140,7 +140,7 @@ impl Endpoint {
fn from_post(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [
],
no_key: [
@ -155,7 +155,7 @@ impl Endpoint {
fn from_put(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [
EMPTY => InsertItem (query::sort_key),
@ -169,7 +169,7 @@ impl Endpoint {
fn from_delete(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
(query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [
EMPTY => DeleteItem (query::sort_key),
],
@ -232,6 +232,11 @@ impl Endpoint {
// parameter name => struct field
generateQueryParameters! {
keywords: [
"delete" => DELETE,
"search" => SEARCH
],
fields: [
"prefix" => prefix,
"start" => start,
"causality_token" => causality_token,
@ -240,13 +245,5 @@ generateQueryParameters! {
"reverse" => reverse,
"sort_key" => sort_key,
"timeout" => timeout
}
mod keywords {
//! This module contain all query parameters with no associated value
//! used to differentiate endpoints.
pub const EMPTY: &str = "";
pub const DELETE: &str = "delete";
pub const SEARCH: &str = "search";
]
}

View file

@ -4,10 +4,9 @@ macro_rules! router_match {
(@match $enum:expr , [ $($endpoint:ident,)* ]) => {{
// usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] }
// returns true if the variant was one of the listed variants, false otherwise.
use Endpoint::*;
match $enum {
$(
$endpoint { .. } => true,
Endpoint::$endpoint { .. } => true,
)*
_ => false
}
@ -15,10 +14,9 @@ macro_rules! router_match {
(@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{
// usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] }
// returns Some(field_value), or None if the variant was not one of the listed variants.
use Endpoint::*;
match $enum {
$(
$endpoint {$param, ..} => Some($param),
Endpoint::$endpoint {$param, ..} => Some($param),
)*
_ => None
}
@ -28,10 +26,9 @@ macro_rules! router_match {
$($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
]) => {{
{
use Endpoint::*;
match ($method, $reqpath) {
$(
(&Method::$meth, $path) if true $(&& $query.$required.is_some())? => $api {
(&Method::$meth, $path) if true $(&& $query.$required.is_some())? => Endpoint::$api {
$($(
$param: router_match!(@@parse_param $query, $conv, $param),
)*)?
@ -60,11 +57,9 @@ macro_rules! router_match {
// ]
// }
// See in from_{method} for more detailed usage.
use Endpoint::*;
use keywords::*;
match ($keyword, !$key.is_empty()){
$(
($kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok($api_k {
(Keyword::$kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok(Endpoint::$api_k {
$key,
$($(
$param_k: router_match!(@@parse_param $query, $conv_k, $param_k),
@ -72,7 +67,7 @@ macro_rules! router_match {
}),
)*
$(
($kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok($api_nk {
(Keyword::$kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok(Endpoint::$api_nk {
$($(
$param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk),
)*)?
@ -144,14 +139,39 @@ macro_rules! router_match {
/// This macro is used to generate part of the code in this module. It must be called only one, and
/// is useless outside of this module.
macro_rules! generateQueryParameters {
( $($rest:expr => $name:ident),* ) => {
(
keywords: [ $($kw_param:expr => $kw_name: ident),* ],
fields: [ $($f_param:expr => $f_name:ident),* ]
) => {
#[derive(Debug)]
#[allow(non_camel_case_types)]
enum Keyword {
EMPTY,
$( $kw_name, )*
}
impl std::fmt::Display for Keyword {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
Keyword::EMPTY => write!(f, "``"),
$( Keyword::$kw_name => write!(f, "`{}`", $kw_param), )*
}
}
}
impl Default for Keyword {
fn default() -> Self {
Keyword::EMPTY
}
}
/// Struct containing all query parameters used in endpoints. Think of it as an HashMap,
/// but with keys statically known.
#[derive(Debug, Default)]
struct QueryParameters<'a> {
keyword: Option<Cow<'a, str>>,
keyword: Option<Keyword>,
$(
$name: Option<Cow<'a, str>>,
$f_name: Option<Cow<'a, str>>,
)*
}
@ -160,34 +180,29 @@ macro_rules! generateQueryParameters {
fn from_query(query: &'a str) -> Result<Self, Error> {
let mut res: Self = Default::default();
for (k, v) in url::form_urlencoded::parse(query.as_bytes()) {
let repeated = match k.as_ref() {
match k.as_ref() {
$(
$rest => if !v.is_empty() {
res.$name.replace(v).is_some()
} else {
false
$kw_param => if let Some(prev_kw) = res.keyword.replace(Keyword::$kw_name) {
return Err(Error::bad_request(format!(
"Multiple keywords: '{}' and '{}'", prev_kw, $kw_param
)));
},
)*
$(
$f_param => if !v.is_empty() {
if res.$f_name.replace(v).is_some() {
return Err(Error::bad_request(format!(
"Query parameter repeated: '{}'", k
)));
}
},
)*
_ => {
if k.starts_with("response-") || k.starts_with("X-Amz-") {
false
} else if v.as_ref().is_empty() {
if res.keyword.replace(k).is_some() {
return Err(Error::bad_request("Multiple keywords"));
}
continue;
} else {
if !(k.starts_with("response-") || k.starts_with("X-Amz-")) {
debug!("Received an unknown query parameter: '{}'", k);
false
}
}
};
if repeated {
return Err(Error::bad_request(format!(
"Query parameter repeated: '{}'",
k
)));
}
}
Ok(res)
}
@ -198,8 +213,8 @@ macro_rules! generateQueryParameters {
if self.keyword.is_some() {
Some("Keyword not used")
} $(
else if self.$name.is_some() {
Some(concat!("'", $rest, "'"))
else if self.$f_name.is_some() {
Some(concat!("'", $f_param, "'"))
}
)* else {
None

View file

@ -355,7 +355,7 @@ impl Endpoint {
fn from_get(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
(query.keyword.take().unwrap_or_default(), key, query, None),
key: [
EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker),
EMPTY => GetObject (query_opt::version_id, opt_parse::part_number),
@ -412,7 +412,7 @@ impl Endpoint {
fn from_head(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
(query.keyword.take().unwrap_or_default(), key, query, None),
key: [
EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
],
@ -426,7 +426,7 @@ impl Endpoint {
fn from_post(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
(query.keyword.take().unwrap_or_default(), key, query, None),
key: [
EMPTY if upload_id => CompleteMultipartUpload (query::upload_id),
RESTORE => RestoreObject (query_opt::version_id),
@ -448,7 +448,7 @@ impl Endpoint {
) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, query, headers),
(query.keyword.take().unwrap_or_default(), key, query, headers),
key: [
EMPTY if part_number header "x-amz-copy-source" => UploadPartCopy (parse::part_number, query::upload_id),
EMPTY header "x-amz-copy-source" => CopyObject,
@ -490,7 +490,7 @@ impl Endpoint {
fn from_delete(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! {
@gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
(query.keyword.take().unwrap_or_default(), key, query, None),
key: [
EMPTY if upload_id => AbortMultipartUpload (query::upload_id),
EMPTY => DeleteObject (query_opt::version_id),
@ -624,6 +624,39 @@ impl Endpoint {
// parameter name => struct field
generateQueryParameters! {
keywords: [
"accelerate" => ACCELERATE,
"acl" => ACL,
"analytics" => ANALYTICS,
"cors" => CORS,
"delete" => DELETE,
"encryption" => ENCRYPTION,
"intelligent-tiering" => INTELLIGENT_TIERING,
"inventory" => INVENTORY,
"legal-hold" => LEGAL_HOLD,
"lifecycle" => LIFECYCLE,
"location" => LOCATION,
"logging" => LOGGING,
"metrics" => METRICS,
"notification" => NOTIFICATION,
"object-lock" => OBJECT_LOCK,
"ownershipControls" => OWNERSHIP_CONTROLS,
"policy" => POLICY,
"policyStatus" => POLICY_STATUS,
"publicAccessBlock" => PUBLIC_ACCESS_BLOCK,
"replication" => REPLICATION,
"requestPayment" => REQUEST_PAYMENT,
"restore" => RESTORE,
"retention" => RETENTION,
"select" => SELECT,
"tagging" => TAGGING,
"torrent" => TORRENT,
"uploads" => UPLOADS,
"versioning" => VERSIONING,
"versions" => VERSIONS,
"website" => WEBSITE
],
fields: [
"continuation-token" => continuation_token,
"delimiter" => delimiter,
"encoding-type" => encoding_type,
@ -644,43 +677,7 @@ generateQueryParameters! {
"upload-id-marker" => upload_id_marker,
"versionId" => version_id,
"version-id-marker" => version_id_marker
}
mod keywords {
//! This module contain all query parameters with no associated value S3 uses to differentiate
//! endpoints.
pub const EMPTY: &str = "";
pub const ACCELERATE: &str = "accelerate";
pub const ACL: &str = "acl";
pub const ANALYTICS: &str = "analytics";
pub const CORS: &str = "cors";
pub const DELETE: &str = "delete";
pub const ENCRYPTION: &str = "encryption";
pub const INTELLIGENT_TIERING: &str = "intelligent-tiering";
pub const INVENTORY: &str = "inventory";
pub const LEGAL_HOLD: &str = "legal-hold";
pub const LIFECYCLE: &str = "lifecycle";
pub const LOCATION: &str = "location";
pub const LOGGING: &str = "logging";
pub const METRICS: &str = "metrics";
pub const NOTIFICATION: &str = "notification";
pub const OBJECT_LOCK: &str = "object-lock";
pub const OWNERSHIP_CONTROLS: &str = "ownershipControls";
pub const POLICY: &str = "policy";
pub const POLICY_STATUS: &str = "policyStatus";
pub const PUBLIC_ACCESS_BLOCK: &str = "publicAccessBlock";
pub const REPLICATION: &str = "replication";
pub const REQUEST_PAYMENT: &str = "requestPayment";
pub const RESTORE: &str = "restore";
pub const RETENTION: &str = "retention";
pub const SELECT: &str = "select";
pub const TAGGING: &str = "tagging";
pub const TORRENT: &str = "torrent";
pub const UPLOADS: &str = "uploads";
pub const VERSIONING: &str = "versioning";
pub const VERSIONS: &str = "versions";
pub const WEBSITE: &str = "website";
]
}
#[cfg(test)]

View file

@ -1,6 +1,6 @@
[package]
name = "garage_block"
version = "0.8.0"
version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -14,10 +14,10 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_db = { version = "0.8.0", path = "../db" }
garage_rpc = { version = "0.8.0", path = "../rpc" }
garage_util = { version = "0.8.0", path = "../util" }
garage_table = { version = "0.8.0", path = "../table" }
garage_db = { version = "0.8.1", path = "../db" }
garage_rpc = { version = "0.8.1", path = "../rpc" }
garage_util = { version = "0.8.1", path = "../util" }
garage_table = { version = "0.8.1", path = "../table" }
opentelemetry = "0.17"

View file

@ -1,6 +1,6 @@
[package]
name = "garage_db"
version = "0.8.0"
version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"

View file

@ -1,6 +1,6 @@
[package]
name = "garage"
version = "0.8.0"
version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -21,14 +21,14 @@ path = "tests/lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_db = { version = "0.8.0", path = "../db" }
garage_api = { version = "0.8.0", path = "../api" }
garage_block = { version = "0.8.0", path = "../block" }
garage_model = { version = "0.8.0", path = "../model" }
garage_rpc = { version = "0.8.0", path = "../rpc" }
garage_table = { version = "0.8.0", path = "../table" }
garage_util = { version = "0.8.0", path = "../util" }
garage_web = { version = "0.8.0", path = "../web" }
garage_db = { version = "0.8.1", path = "../db" }
garage_api = { version = "0.8.1", path = "../api" }
garage_block = { version = "0.8.1", path = "../block" }
garage_model = { version = "0.8.1", path = "../model" }
garage_rpc = { version = "0.8.1", path = "../rpc" }
garage_table = { version = "0.8.1", path = "../table" }
garage_util = { version = "0.8.1", path = "../util" }
garage_web = { version = "0.8.1", path = "../web" }
backtrace = "0.3"
bytes = "1.0"

View file

@ -22,7 +22,7 @@ tokio = "1.17.0"
# cli deps
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
garage_util = { version = "0.8.0", path = "../util", optional = true }
garage_util = { version = "0.8.1", path = "../util", optional = true }
[features]

View file

@ -1,6 +1,6 @@
[package]
name = "garage_model"
version = "0.8.0"
version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -14,11 +14,11 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_db = { version = "0.8.0", default-features = false, path = "../db" }
garage_rpc = { version = "0.8.0", path = "../rpc" }
garage_table = { version = "0.8.0", path = "../table" }
garage_block = { version = "0.8.0", path = "../block" }
garage_util = { version = "0.8.0", path = "../util" }
garage_db = { version = "0.8.1", default-features = false, path = "../db" }
garage_rpc = { version = "0.8.1", path = "../rpc" }
garage_table = { version = "0.8.1", path = "../table" }
garage_block = { version = "0.8.1", path = "../block" }
garage_util = { version = "0.8.1", path = "../util" }
async-trait = "0.1.7"
arc-swap = "1.0"

View file

@ -1,6 +1,6 @@
[package]
name = "garage_rpc"
version = "0.8.0"
version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -14,7 +14,7 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_util = { version = "0.8.0", path = "../util" }
garage_util = { version = "0.8.1", path = "../util" }
arc-swap = "1.0"
bytes = "1.0"

View file

@ -1,6 +1,6 @@
[package]
name = "garage_table"
version = "0.8.0"
version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -14,9 +14,9 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_db = { version = "0.8.0", path = "../db" }
garage_rpc = { version = "0.8.0", path = "../rpc" }
garage_util = { version = "0.8.0", path = "../util" }
garage_db = { version = "0.8.1", path = "../db" }
garage_rpc = { version = "0.8.1", path = "../rpc" }
garage_util = { version = "0.8.1", path = "../util" }
opentelemetry = "0.17"

View file

@ -1,6 +1,6 @@
[package]
name = "garage_util"
version = "0.8.0"
version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -14,7 +14,7 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_db = { version = "0.8.0", path = "../db" }
garage_db = { version = "0.8.1", path = "../db" }
arc-swap = "1.0"
async-trait = "0.1"

View file

@ -1,6 +1,6 @@
[package]
name = "garage_web"
version = "0.8.0"
version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018"
license = "AGPL-3.0"
@ -14,10 +14,10 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_api = { version = "0.8.0", path = "../api" }
garage_model = { version = "0.8.0", path = "../model" }
garage_util = { version = "0.8.0", path = "../util" }
garage_table = { version = "0.8.0", path = "../table" }
garage_api = { version = "0.8.1", path = "../api" }
garage_model = { version = "0.8.1", path = "../model" }
garage_util = { version = "0.8.1", path = "../util" }
garage_table = { version = "0.8.1", path = "../table" }
err-derive = "0.3"
tracing = "0.1.30"