Add minio & rclone to our functional tests

It is now possible to configure which clients
you do not want to test with the env variable SKIP_XXX=1,
XXX being the client name. eg. SKIP_S3CMD=1 ./script/test-smoke.sh

.drone.yml

@@ -37,7 +37,7 @@ steps:
       - nonexistent_skip_this_step
 
   - name: build
-    image: lxpz/garage_builder_amd64:1
+    image: superboum/garage_builder_amd64:3
     volumes:
     - name: cargo_home
       path: /drone/cargo
@@ -49,7 +49,7 @@ steps:
       - cargo build
 
   - name: cargo-test
-    image: lxpz/garage_builder_amd64:1
+    image: superboum/garage_builder_amd64:3
     volumes:
     - name: cargo_home
       path: /drone/cargo
@@ -85,7 +85,7 @@ steps:
       - nonexistent_skip_this_step
 
   - name: smoke-test
-    image: lxpz/garage_builder_amd64:1
+    image: superboum/garage_builder_amd64:3
     volumes:
     - name: cargo_home
       path: /drone/cargo
@@ -129,6 +129,6 @@ steps:
 
 ---
 kind: signature
-hmac: de82026387bd09e547dbc9cc5d232fd865204b4f393d32508c50b58f8e60611d
+hmac: d584c2a15ede6d5702fbe27ae5ae2b2bf7a04461ae7aed2d53cbda83b7fd503e
 
 ...

script/builder_image/Dockerfile

@@ -1,6 +1,7 @@
 FROM rust:buster
 RUN apt-get update && \
-	apt-get install --yes libsodium-dev awscli python-pip && \
+	apt-get install --yes libsodium-dev awscli python-pip wget rclone openssl socat && \
 	    rm -rf /var/lib/apt/lists/*
+RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc
 RUN rustup component add rustfmt
 RUN pip install s3cmd

script/dev-clean.sh

@@ -3,5 +3,6 @@
 set -ex
 
 killall -9 garage || echo "garage is not running"
+killall -9 socat || echo "socat is not running"
 rm -rf /tmp/garage*
 rm -rf /tmp/config.*.toml

script/dev-cluster.sh

@@ -47,6 +47,21 @@ EOF
 
 echo -en "$LABEL configuration written to $CONF_PATH\n"
 
+if [ -z "$SKIP_HTTPS" ]; then
+  echo -en "$LABEL Starting dummy HTTPS reverse proxy\n"
+  mkdir -p /tmp/garagessl
+  openssl req \
+    -new \
+    -x509 \
+    -keyout /tmp/garagessl/test.key \
+    -out /tmp/garagessl/test.crt \
+    -nodes  \
+    -subj "/C=XX/ST=XX/L=XX/O=XX/OU=XX/CN=localhost/emailAddress=X@X.XX" \
+    -addext "subjectAltName = DNS:localhost, IP:127.0.0.1"
+  cat /tmp/garagessl/test.key /tmp/garagessl/test.crt > /tmp/garagessl/test.pem
+  socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 &
+fi
+
 (garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
 done
 

script/dev-env-aws.sh

@@ -1,14 +1,9 @@
 #!/bin/bash
 
-SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
-REPO_FOLDER="${SCRIPT_FOLDER}/../"
-GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
-GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
-PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
-
 export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
 export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
 export AWS_DEFAULT_REGION='garage'
 
-alias awsgrg="aws s3 \
+function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; }
-  --endpoint-url http://127.0.0.1:3911"
+
+aws --version

script/dev-env-mc.sh

@@ -0,0 +1,22 @@
+ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
+SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
+
+mkdir -p /tmp/garage.mc/certs/CAs
+
+cat > /tmp/garage.mc/config.json <<EOF
+{
+  "version": "10",
+  "aliases": {
+    "garage": {
+      "url": "https://localhost:4443",
+      "accessKey": "$ACCESS_KEY",
+      "secretKey": "$SECRET_KEY",
+      "api": "S3v4",
+      "path": "auto"
+    }
+  }
+}
+EOF
+
+function mc { command mc --insecure --config-dir /tmp/garage.mc $@ ; }
+mc --version

script/dev-env-rclone.sh

@@ -0,0 +1,21 @@
+ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
+SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
+
+cat > /tmp/garage.rclone.conf <<EOF
+[garage]
+type = s3
+provider = Other
+env_auth = false
+access_key_id = $ACCESS_KEY
+secret_access_key = $SECRET_KEY
+endpoint = http://127.0.0.1:3911
+bucket_acl = private
+force_path_style = true
+region = garage
+no_check_bucket = true
+EOF
+# It seems that region is mandatory as rclone does not support redirection
+
+
+function rclone { command rclone --config /tmp/garage.rclone.conf $@ ; }
+rclone --version

script/dev-env-s3cmd.sh

@@ -1,19 +1,15 @@
-#!/bin/bash
-
-SCRIPT_FOLDER="`dirname \"${BASH_SOURCE[0]}\"`"
-REPO_FOLDER="${SCRIPT_FOLDER}/../"
-GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
-GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
-PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
-
 ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f1`
 SECRET_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
 
-alias s3grg="s3cmd \
+cat > /tmp/garage.s3cmd.cfg <<EOF
-  --host 127.0.0.1:3911 \
+[default]
-  --host-bucket 127.0.0.1:3911 \
+access_key = $ACCESS_KEY
-  --access_key=$ACCESS_KEY \
+secret_key = $SECRET_KEY
-  --secret_key=$SECRET_KEY \
+host_base = 127.0.0.1:3911
-  --region=garage \
+host_bucket = 127.0.0.1:3911
-  --no-ssl"
+use_https = False
+EOF
 
+function s3cmd { command s3cmd --config=/tmp/garage.s3cmd.cfg $@ ; }
+s3cmd --version
+python3 --version

script/test-smoke.sh

@@ -1,22 +1,23 @@
 #!/bin/bash
 
 set -ex
-shopt -s expand_aliases
 
 export LC_ALL=C.UTF-8
 export LANG=C.UTF-8
 SCRIPT_FOLDER="`dirname \"$0\"`"
 REPO_FOLDER="${SCRIPT_FOLDER}/../"
+GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
+GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
+PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
 
-echo "setup"
+echo "⏳ Setup"
 cargo build
 ${SCRIPT_FOLDER}/dev-clean.sh
 ${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
 ${SCRIPT_FOLDER}/dev-configure.sh
 ${SCRIPT_FOLDER}/dev-bucket.sh
-source ${SCRIPT_FOLDER}/dev-env-aws.sh
-source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
 
+which garage
 garage status
 garage key list
 garage bucket list
@@ -25,58 +26,86 @@ dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline sto
 dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked
 dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB
 
-echo "s3 api testing..."
+echo "🧪 S3 API testing..."
-awsgrg --version
-s3cmd --version
-python3 --version
 
-for idx in $(seq 1 3); do
+# AWS
-  # AWS sends
+if [ -z "$SKIP_AWS" ]; then
-  awsgrg cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
+  echo "🛠️ Testing with awscli"
+  source ${SCRIPT_FOLDER}/dev-env-aws.sh
+  for idx in $(seq 1 3); do
+    aws s3 cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
+    aws s3 ls s3://eprouvette
+    aws s3 cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
+    diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rm /tmp/garage.$idx.dl
+    aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
+  done
+fi
 
-  awsgrg ls s3://eprouvette
+# S3CMD
+if [ -z "$SKIP_S3CMD" ]; then
+  echo "🛠️ Testing with s3cmd"
+  source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
+  for idx in $(seq 1 3); do
+    s3cmd put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
+    s3cmd ls s3://eprouvette
+    s3cmd get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
+    diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rm /tmp/garage.$idx.dl
+    s3cmd rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
+  done
+fi
 
-  awsgrg cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
+# Minio Client
-  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+if [ -z "$SKIP_MC" ]; then
-  rm /tmp/garage.$idx.dl
+  echo "🛠️ Testing with mc (minio client)"
+  source ${SCRIPT_FOLDER}/dev-env-mc.sh
+  for idx in $(seq 1 3); do
+    mc cp "/tmp/garage.$idx.rnd" "garage/eprouvette/&+-é\"/garage.$idx.mc"
+    mc ls garage/eprouvette
+    mc cp "garage/eprouvette/&+-é\"/garage.$idx.mc" "/tmp/garage.$idx.dl"
+    diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rm /tmp/garage.$idx.dl
+    mc rm "garage/eprouvette/&+-é\"/garage.$idx.mc"
+  done
+fi
 
-  s3grg get "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
+# RClone
-  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+if [ -z "$SKIP_RCLONE" ]; then
-  rm /tmp/garage.$idx.dl
+  echo "🛠️ Testing with rclone"
+  source ${SCRIPT_FOLDER}/dev-env-rclone.sh
+  for idx in $(seq 1 3); do
+    cp /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/"
+    rm /tmp/garage.$idx.dl
+    rclone ls garage:eprouvette
+    rclone copy "garage:eprouvette/&+-é\"/garage.$idx.dl" "/tmp/"
+    diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
+    rm /tmp/garage.$idx.dl
+    rclone delete "garage:eprouvette/&+-é\"/garage.$idx.dl"
+  done
+fi
 
-  awsgrg rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
-
-  # S3CMD sends
-  s3grg put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
-
-  s3grg ls s3://eprouvette
-	
-  s3grg get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
-  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
-  rm /tmp/garage.$idx.dl
-
-  awsgrg cp "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
-  diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
-  rm /tmp/garage.$idx.dl
-	
-  s3grg rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
-done
 rm /tmp/garage.{1,2,3}.rnd
 
-echo "website testing"
+if [ -z "$SKIP_AWS" ]; then
-echo "<h1>hello world</h1>" > /tmp/garage-index.html
+  echo "🧪 Website Testing"
-awsgrg cp /tmp/garage-index.html s3://eprouvette/index.html
+  echo "<h1>hello world</h1>" > /tmp/garage-index.html
-[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 404 ]
+  aws s3 cp /tmp/garage-index.html s3://eprouvette/index.html
-garage bucket website --allow eprouvette
+  [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 404 ]
-[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 200 ]
+  garage bucket website --allow eprouvette
-garage bucket website --deny eprouvette
+  [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 200 ]
-[ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 404 ]
+  garage bucket website --deny eprouvette
-awsgrg rm s3://eprouvette/index.html
+  [ `curl -s -o /dev/null -w "%{http_code}" --header "Host: eprouvette.garage.tld"  http://127.0.0.1:3923/ ` == 404 ]
-rm /tmp/garage-index.html
+  aws s3 rm s3://eprouvette/index.html
+  rm /tmp/garage-index.html
+fi
 
-echo "teardown"
+echo "🏁 Teardown"
+AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
+AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
 garage bucket deny --read --write eprouvette --key $AWS_ACCESS_KEY_ID
 garage bucket delete --yes eprouvette
 garage key delete --yes $AWS_ACCESS_KEY_ID
 
-echo "success"
+echo "✅ Success"