mirror of
https://git.deuxfleurs.fr/Deuxfleurs/garage.git
synced 2024-11-23 08:31:02 +00:00
[nix-improvements] get rid of Drone
This commit is contained in:
Alex Auvolat
parent
1be75fbf4e
commit
561fad0b44
7 changed files with 22 additions and 356 deletions
300
.drone.yml
300
.drone.yml
|
|
@ -1,300 +0,0 @@
|
|||
---
|
||||
kind: pipeline
|
||||
name: default
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: check formatting
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-shell --attr devShell --run "cargo fmt -- --check"
|
||||
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
|
||||
- name: unit + func tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage
|
||||
GARAGE_TEST_INTEGRATION_PATH: tmp-garage-integration
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-build --no-build-output --attr test.amd64
|
||||
- ./result/bin/garage_db-*
|
||||
- ./result/bin/garage_api-*
|
||||
- ./result/bin/garage_model-*
|
||||
- ./result/bin/garage_rpc-*
|
||||
- ./result/bin/garage_table-*
|
||||
- ./result/bin/garage_util-*
|
||||
- ./result/bin/garage_web-*
|
||||
- ./result/bin/garage-*
|
||||
- ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
|
||||
- rm result
|
||||
- rm -rv tmp-garage-integration
|
||||
|
||||
- name: integration tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- custom
|
||||
- push
|
||||
- pull_request
|
||||
- tag
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: release-linux-amd64
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
|
||||
|
||||
- name: integration tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
- name: upgrade tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
TARGET: "x86_64-unknown-linux-musl"
|
||||
commands:
|
||||
- nix-shell --attr ci --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
DOCKER_PLATFORM: "linux/amd64"
|
||||
CONTAINER_NAME: "dxflrs/amd64_garage"
|
||||
HOME: "/kaniko"
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr ci --run "to_docker"
|
||||
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- promote
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: release-linux-i386
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
|
||||
|
||||
- name: integration tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
- name: upgrade tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false)
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
TARGET: "i686-unknown-linux-musl"
|
||||
commands:
|
||||
- nix-shell --attr ci --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
DOCKER_PLATFORM: "linux/386"
|
||||
CONTAINER_NAME: "dxflrs/386_garage"
|
||||
HOME: "/kaniko"
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr ci --run "to_docker"
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- promote
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: release-linux-arm64
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
TARGET: "aarch64-unknown-linux-musl"
|
||||
commands:
|
||||
- nix-shell --attr ci --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
DOCKER_PLATFORM: "linux/arm64"
|
||||
CONTAINER_NAME: "dxflrs/arm64_garage"
|
||||
HOME: "/kaniko"
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr ci --run "to_docker"
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- promote
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: release-linux-arm
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
TARGET: "armv6l-unknown-linux-musleabihf"
|
||||
commands:
|
||||
- nix-shell --attr ci --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
DOCKER_PLATFORM: "linux/arm"
|
||||
CONTAINER_NAME: "dxflrs/arm_garage"
|
||||
HOME: "/kaniko"
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr ci --run "to_docker"
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- promote
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: refresh-release-page
|
||||
|
||||
node:
|
||||
nix-daemon: 1
|
||||
|
||||
steps:
|
||||
- name: multiarch-docker
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
DOCKER_AUTH:
|
||||
from_secret: docker_auth
|
||||
HOME: "/root"
|
||||
commands:
|
||||
- mkdir -p /root/.docker
|
||||
- echo $DOCKER_AUTH > /root/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --attr ci --run "multiarch_docker"
|
||||
- name: refresh-index
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
|
||||
- nix-shell --attr ci --run "refresh_index"
|
||||
|
||||
depends_on:
|
||||
- release-linux-amd64
|
||||
- release-linux-i386
|
||||
- release-linux-arm64
|
||||
- release-linux-arm
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- promote
|
||||
- cron
|
||||
|
||||
---
|
||||
kind: signature
|
||||
hmac: 9ff9f2ad9387b12ee49a327991a5373f01e40d28e0aeef3a13d3b741d8eb3c53
|
||||
|
||||
...
|
||||
|
|
@ -54,7 +54,7 @@ steps:
|
|||
- source: garagehq_aws_secret_access_key
|
||||
target: AWS_SECRET_ACCESS_KEY
|
||||
commands:
|
||||
- nix-shell --attr ci --run "to_s3_woodpecker"
|
||||
- nix-shell --attr ci --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
|
|
|
|||
|
|
@ -48,7 +48,5 @@ locations. They use Garage themselves for the following tasks:
|
|||
|
||||
- As a backup target using `rclone` and `restic`
|
||||
|
||||
- In the Drone continuous integration platform to store task logs
|
||||
|
||||
The Deuxfleurs Garage cluster is a multi-site cluster currently composed of
|
||||
9 nodes in 3 physical locations.
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ nix-build \
|
|||
--git_version $(git rev-parse HEAD)
|
||||
```
|
||||
|
||||
*The result is located in `result/bin`. You can pass arguments to cross compile: check `.drone.yml` for examples.*
|
||||
*The result is located in `result/bin`. You can pass arguments to cross compile: check `.woodpecker/release.yml` for examples.*
|
||||
|
||||
If you modify a `Cargo.toml` or regenerate any `Cargo.lock`, you must run `cargo2nix`:
|
||||
|
||||
|
|
|
|||
|
|
@ -81,12 +81,9 @@ Our cache will be checked.
|
|||
- http://www.lpenz.org/articles/nixchannel/index.html
|
||||
|
||||
|
||||
## Drone
|
||||
## Woodpecker
|
||||
|
||||
Do not try to set a build as trusted from the interface or the CLI tool,
|
||||
your request would be ignored. Instead, directly edit the database (table `repos`, column `repo_trusted`).
|
||||
|
||||
Drone can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner.
|
||||
Woodpecker can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner.
|
||||
|
||||
## Building Docker containers
|
||||
|
||||
|
|
@ -99,3 +96,4 @@ We were:
|
|||
- Unable to use the kaniko container provided by Google as we can't run arbitrary logic: we need to put our secret in .docker/config.json.
|
||||
|
||||
Finally we chose to build kaniko through nix and use it in a `nix-shell`.
|
||||
We then switched to using kaniko from nixpkgs when it was packaged.
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ and the docker containers on Docker Hub.
|
|||
|
||||
## Automation
|
||||
|
||||
We automated our release process with Nix and Drone to make it more reliable.
|
||||
We automated our release process with Nix and Woodpecker to make it more reliable.
|
||||
Here we describe how we have done in case you want to debug or improve it.
|
||||
|
||||
### Caching build steps
|
||||
|
|
@ -62,52 +62,31 @@ Sending to the cache is done through `nix copy`, for example:
|
|||
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' result
|
||||
```
|
||||
|
||||
*Note that you need the signing key. In our case, it is stored as a secret in Drone.*
|
||||
*The signing key possessed by the Garage maintainers is required to update the Nix cache.*
|
||||
|
||||
The previous command will only send the built packet and not its dependencies.
|
||||
To send its dependency, a tool named `nix-copy-closure` has been created but it is not compatible with the S3 protocol.
|
||||
|
||||
Instead, you can use the following commands to list all the runtime dependencies:
|
||||
The previous command will only send the built package and not its dependencies.
|
||||
In the case of our CI pipeline, we want to cache all intermediate build steps
|
||||
as well. This can be done using this quite involved command (here as an example
|
||||
for the `pkgs.amd64.relase` package):
|
||||
|
||||
```bash
|
||||
nix copy \
|
||||
--to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||
$(nix-store -qR result/)
|
||||
nix copy -j8 \
|
||||
--to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/nix-signing-key.sec' \
|
||||
$(nix path-info pkgs.amd64.release --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/')
|
||||
```
|
||||
|
||||
*We could also write this expression with xargs but this tool is not available in our container.*
|
||||
This command will simultaneously build all of the required Nix paths (using at
|
||||
most 8 parallel Nix builder jobs) and send the resulting objects to the cache.
|
||||
|
||||
But in certain cases, we want to cache compile time dependencies also.
|
||||
For example, the Nix project does not provide binaries for cross compiling to i686 and thus we need to compile gcc on our own.
|
||||
We do not want to compile gcc each time, so even if it is a compile time dependency, we want to cache it.
|
||||
|
||||
This time, the command is a bit more involved:
|
||||
|
||||
```bash
|
||||
nix copy --to \
|
||||
's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||
$(nix-store -qR --include-outputs \
|
||||
$(nix-instantiate))
|
||||
```
|
||||
|
||||
This is the command we use in our CI as we expect the final binary to change, so we mainly focus on
|
||||
caching our development dependencies.
|
||||
|
||||
*Currently there is no automatic garbage collection of the cache: we should monitor its growth.
|
||||
Hopefully, we can erase it totally without breaking any build, the next build will only be slower.*
|
||||
|
||||
In practise, we concluded that we do not want to cache all the compilation dependencies.
|
||||
Instead, we want to cache the toolchain we use to build Garage each time we change it.
|
||||
So we removed from Drone any automatic update of the cache and instead handle them manually with:
|
||||
This can be run for all the Garage packages we build using the following command:
|
||||
|
||||
```
|
||||
source ~/.awsrc
|
||||
nix-shell --run 'refresh_toolchain'
|
||||
nix-shell --attr cache --run 'refresh_cache'
|
||||
```
|
||||
|
||||
Internally, it will run `nix-build` on `nix/toolchain.nix` and send the output plus its depedencies to the cache.
|
||||
|
||||
To erase the cache:
|
||||
We don't automate this step at each CI build, as *there is currently no automatic garbage collection of the cache.*
|
||||
This means we should also monitor the cache's size; if it ever becomes too big we can erase it with:
|
||||
|
||||
```
|
||||
mc rm --recursive --force 'garage/nix/'
|
||||
|
|
@ -157,9 +136,9 @@ nix-shell --run refresh_index
|
|||
|
||||
If you want to compile for different architectures, you will need to repeat all these commands for each architecture.
|
||||
|
||||
**In practise, and except for debugging, you will never directly run these commands. Release is handled by drone**
|
||||
**In practice, and except for debugging, you will never directly run these commands. Release is handled by Woodpecker.**
|
||||
|
||||
### Drone
|
||||
### Drone (obsolete)
|
||||
|
||||
Our instance is available at [https://drone.deuxfleurs.fr](https://drone.deuxfleurs.fr).
|
||||
You need an account on [https://git.deuxfleurs.fr](https://git.deuxfleurs.fr) to use it.
|
||||
|
|
|
|||
|
|
@ -34,15 +34,6 @@ in
|
|||
];
|
||||
shellHook = ''
|
||||
function to_s3 {
|
||||
aws \
|
||||
--endpoint-url https://garage.deuxfleurs.fr \
|
||||
--region garage \
|
||||
s3 cp \
|
||||
./result-bin/bin/garage \
|
||||
s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage
|
||||
}
|
||||
|
||||
function to_s3_woodpecker {
|
||||
aws \
|
||||
--endpoint-url https://garage.deuxfleurs.fr \
|
||||
--region garage \
|
||||
|
|
|
|||
Loading…
Reference in a new issue