From 854be47328cc0283119dfb78f9ba0b8306c85f75 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Wed, 14 Jun 2023 13:32:20 +0200 Subject: [PATCH] [CI] Forgejo Actions based release process Refs: https://codeberg.org/forgejo/website/pulls/230 (cherry picked from commit 87d56bf6c73d726dae8aafbc7e147969f1899931) [CI] Forgejo Actions based release process (squash) base64 -w0 to avoid wrapping when the doer name is long as it creates a broken config.json (cherry picked from commit 9efdc27e49bdfb3e62401baf27b224385f9f3e5e) [CI] Forgejo Actions based release process (squash) generate .xz files and sources Generate .xz files Check .sha256 Generate the source tarbal (cherry picked from commit 7afec520c4b1032d7e67a05a41e4e2913bcd9312) [CI] Forgejo Actions based release process (squash) release notes (cherry picked from commit d8f4f4807b28297b318d2f555a76d0efef762cf7) [CI] Forgejo Actions based release process (squash) publish and sign release (cherry picked from commit a52778c74785fe57cdee3b64b4c6c8a326471532) (cherry picked from commit cf2ec6274094ac7aebda71d54c64581f528df00a) [CI] Forgejo Actions based release process (squash) version use Actions environment variables in Makefile (#25319) (#25318) uses Actions variable to determine the version. But Forgejo builds happen in a container where they are not available. Do not use them. Also verify the version of the binary is as expected for sanity check. (cherry picked from commit 6decf111a132a869f9e5c6f4d20e368b8f74309f) (cherry picked from commit 206d0b3886b2d56b585bf552e53d952b35f07284) [CI] read STORED_VERSION_FILE if available (cherry picked from commit af74085ebf51c91a51db865a66667fca511838d5) [CI] backward compatible executable compilation Add a new static-executable target to use in Dockerfiles and restore the $(EXECUTABLE) target to what it was before to for backward compatibility. The release process now builds static executables instead of dynamically linked ones which makes them more portable. It changes the requirements at compile time and is not backward compatible. In particular it may break packaging that rely on the target that currently creates a dynamically linked executable. (cherry picked from commit 84d02a174a4398b1f8ee62e08db5483a9859650f) --- .forgejo/actions/build-release/action.yml | 154 +++++++++++++++++ .forgejo/actions/publish-release/action.yml | 93 +++++++++++ .forgejo/testdata/build-release/Dockerfile | 3 + .forgejo/testdata/build-release/Makefile | 5 + .../workflows/build-release-integration.yml | 99 +++++++++++ .forgejo/workflows/build-release.yml | 155 ++++++++++++++++++ .forgejo/workflows/publish-release.yml | 25 +++ Dockerfile | 35 ++-- Dockerfile.rootless | 35 ++-- Makefile | 32 ++-- 10 files changed, 594 insertions(+), 42 deletions(-) create mode 100644 .forgejo/actions/build-release/action.yml create mode 100644 .forgejo/actions/publish-release/action.yml create mode 100644 .forgejo/testdata/build-release/Dockerfile create mode 100644 .forgejo/testdata/build-release/Makefile create mode 100644 .forgejo/workflows/build-release-integration.yml create mode 100644 .forgejo/workflows/build-release.yml create mode 100644 .forgejo/workflows/publish-release.yml diff --git a/.forgejo/actions/build-release/action.yml b/.forgejo/actions/build-release/action.yml new file mode 100644 index 0000000000..01fdfdedfc --- /dev/null +++ b/.forgejo/actions/build-release/action.yml @@ -0,0 +1,154 @@ +name: 'Build release' +author: 'Forgejo authors' +description: | + Build release + +inputs: + forgejo: + description: 'URL of the Forgejo instance where the release is uploaded' + required: true + owner: + description: 'User or organization where the release is uploaded, relative to the Forgejo instance' + required: true + repository: + description: 'Repository where the release is uploaded, relative to the owner' + required: true + doer: + description: 'Name of the user authoring the release' + required: true + tag-version: + description: 'Version of the release derived from the tag withint the leading v' + required: true + suffix: + description: 'Suffix to add to the image tag' + token: + description: 'token' + required: true + dockerfile: + description: 'path to the dockerfile' + default: 'Dockerfile' + platforms: + description: 'Coma separated list of platforms' + default: 'linux/amd64,linux/arm64' + release-notes: + description: 'Full text of the release notes' + default: 'Release notes placeholder' + binary-name: + description: 'Name of the binary' + binary-path: + description: 'Path of the binary within the container to extract into binary-name' + verbose: + description: 'Increase the verbosity level' + default: 'false' + +runs: + using: "composite" + steps: + - run: echo "${{ github.action_path }}" >> $GITHUB_PATH + shell: bash + + - name: Install dependencies + run: | + apt-get install -y -qq xz-utils + + - name: set -x if verbose is required + id: verbose + run: | + if ${{ inputs.verbose }} ; then + echo "shell=set -x" >> "$GITHUB_OUTPUT" + fi + + - name: Create the insecure and buildx-config variables for the container registry + id: registry + run: | + ${{ steps.verbose.outputs.shell }} + url="${{ inputs.forgejo }}" + hostport=${url##http*://} + hostport=${hostport%%/} + echo "host-port=${hostport}" >> "$GITHUB_OUTPUT" + if ! [[ $url =~ ^http:// ]] ; then + exit 0 + fi + cat >> "$GITHUB_OUTPUT" < /etc/docker/daemon.json < ~/.docker/config.json + env: + CI_REGISTRY: "${{ steps.registry.outputs.host-port }}" + + - name: Build the container image for each architecture + uses: https://github.com/docker/build-push-action@v4 + # workaround until https://github.com/docker/build-push-action/commit/d8823bfaed2a82c6f5d4799a2f8e86173c461aba is in @v4 or @v5 is released + env: + ACTIONS_RUNTIME_TOKEN: '' + with: + context: . + push: true + file: ${{ inputs.dockerfile }} + platforms: ${{ inputs.platforms }} + tags: ${{ steps.registry.outputs.host-port }}/${{ inputs.owner }}/${{ inputs.repository }}:${{ inputs.tag-version }}${{ inputs.suffix }} + + - name: Extract the binary from the container images into the release directory + if: inputs.binary-name != '' + run: | + ${{ steps.verbose.outputs.shell }} + mkdir -p release + cd release + for platform in $(echo ${{ inputs.platforms }} | tr ',' ' '); do + arch=$(echo $platform | sed -e 's|linux/||g' -e 's|arm/v6|arm-6|g') + docker create --platform $platform --name forgejo-$arch ${{ steps.registry.outputs.host-port }}/${{ inputs.owner }}/${{ inputs.repository }}:${{ inputs.tag-version }}${{ inputs.suffix }} + binary="${{ inputs.binary-name }}-${{ inputs.tag-version }}-linux" + docker cp forgejo-$arch:${{ inputs.binary-path }} $binary-$arch + chmod +x $binary-$arch + # the displayed version has a + instead of the first -, deal with it + pattern=$(echo "${{ inputs.tag-version }}" | tr - .) + if ! ./$binary-$arch --version | grep "$pattern" ; then + echo "ERROR: expected version pattern $pattern not found in the output of $binary-$arch --version" + ./$binary-$arch --version + exit 1 + fi + xz --keep -9 $binary-$arch + shasum -a 256 $binary-$arch > $binary-$arch.sha256 + shasum -a 256 $binary-$arch.xz > $binary-$arch.xz.sha256 + docker rm forgejo-$arch + done + + - name: publish release + if: inputs.binary-name != '' + uses: https://code.forgejo.org/actions/forgejo-release@v1 + with: + direction: upload + release-dir: release + release-notes: "${{ inputs.release-notes }}" + token: ${{ inputs.token }} + verbose: ${{ steps.verbose.outputs.value }} diff --git a/.forgejo/actions/publish-release/action.yml b/.forgejo/actions/publish-release/action.yml new file mode 100644 index 0000000000..bafa473000 --- /dev/null +++ b/.forgejo/actions/publish-release/action.yml @@ -0,0 +1,93 @@ +name: 'Publish release' +author: 'Forgejo authors' +description: | + Publish release + +inputs: + forgejo: + description: 'URL of the Forgejo instance where the release is uploaded' + required: true + from-owner: + description: 'the owner from which a release is to be copied' + required: true + to-owner: + description: 'the owner to which a release is to be copied' + required: true + repo: + description: 'the repository from which a release is to be copied relative to from-owner and to-owner' + default: 'forgejo' + ref-name: + description: 'ref_name of the tag of the release to be copied' + required: true + doer: + description: 'Name of the user authoring the release' + required: true + token: + description: 'application token on FORGEJO with permission to the repository and the packages' + required: true + gpg-private-key: + description: 'GPG Private Key to sign the release artifacts' + gpg-passphrase: + description: 'Passphrase of the GPG Private Key' + +runs: + using: "composite" + steps: + - id: hostport + run: | + url="${{ inputs.forgejo }}" + hostport=${url##http*://} + hostport=${hostport%%/} + echo "value=$hostport" >> "$GITHUB_OUTPUT" + + - id: tag-version + run: | + version="${{ inputs.ref-name }}" + version=${version##*v} + echo "value=$version" >> "$GITHUB_OUTPUT" + + - name: apt-get install docker.io + run: | + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qq -y docker.io + + - name: download release + uses: https://code.forgejo.org/actions/forgejo-release@v1 + with: + url: ${{ inputs.forgejo }} + repo: ${{ inputs.from-owner }}/${{ inputs.repo }} + direction: download + release-dir: release + download-retry: 60 + token: ${{ inputs.token }} + + - name: upload release + uses: https://code.forgejo.org/actions/forgejo-release@v1 + with: + url: ${{ inputs.forgejo }} + repo: ${{ inputs.to-owner }}/${{ inputs.repo }} + direction: upload + release-dir: release + release-notes: "See https://codeberg.org/forgejo/forgejo/src/branch/forgejo/RELEASE-NOTES.md#${{ steps.tag-version.outputs.value }}" + token: ${{ inputs.token }} + gpg-private-key: ${{ inputs.gpg-private-key }} + gpg-passphrase: ${{ inputs.gpg-passphrase }} + + - name: login to the registry + uses: https://github.com/docker/login-action@v2 + with: + registry: ${{ steps.hostport.outputs.value }} + username: ${{ inputs.doer }} + password: ${{ inputs.token }} + + - uses: https://code.forgejo.org/forgejo/forgejo-container-image@v1 + env: + VERIFY: 'false' + with: + url: https://${{ steps.hostport.outputs.value }} + destination-owner: ${{ inputs.to-owner }} + owner: ${{ inputs.from-owner }} + suffixes: '-rootless' + project: ${{ inputs.repo }} + tag: ${{ steps.tag-version.outputs.value }} + doer: ${{ inputs.doer }} + token: ${{ inputs.token }} diff --git a/.forgejo/testdata/build-release/Dockerfile b/.forgejo/testdata/build-release/Dockerfile new file mode 100644 index 0000000000..4b6933845c --- /dev/null +++ b/.forgejo/testdata/build-release/Dockerfile @@ -0,0 +1,3 @@ +FROM public.ecr.aws/docker/library/alpine:3.18 +RUN mkdir -p /app/gitea +RUN ( echo '#!/bin/sh' ; echo "echo forgejo v1.2.3" ) > /app/gitea/gitea ; chmod +x /app/gitea/gitea diff --git a/.forgejo/testdata/build-release/Makefile b/.forgejo/testdata/build-release/Makefile new file mode 100644 index 0000000000..406acd06d2 --- /dev/null +++ b/.forgejo/testdata/build-release/Makefile @@ -0,0 +1,5 @@ +VERSION ?= $(shell cat VERSION 2>/dev/null) +sources-tarbal: + mkdir -p dist/release + echo $(VERSION) > VERSION + sources=forgejo-src-$(VERSION).tar.gz ; tar --transform 's|^./|forgejo-src-$(VERSION)/|' -czf dist/release/forgejo-src-$(VERSION).tar.gz . ; cd dist/release ; shasum -a 256 $$sources > $$sources.sha256 diff --git a/.forgejo/workflows/build-release-integration.yml b/.forgejo/workflows/build-release-integration.yml new file mode 100644 index 0000000000..d893bd39af --- /dev/null +++ b/.forgejo/workflows/build-release-integration.yml @@ -0,0 +1,99 @@ +name: Integration tests for the release process + +on: + push: + paths: + - Makefile + - Dockerfile + - Dockerfile.rootless + - docker/** + - .forgejo/actions/build-release/action.yml + - .forgejo/workflows/build-release.yml + - .forgejo/workflows/build-release-integration.yml + +jobs: + release-simulation: + runs-on: self-hosted + if: secrets.ROLE != 'forgejo-integration' && secrets.ROLE != 'forgejo-experimental' && secrets.ROLE != 'forgejo-release' + steps: + - uses: actions/checkout@v3 + + - id: forgejo + uses: https://code.forgejo.org/actions/setup-forgejo@v1 + with: + user: root + password: admin1234 + image-version: 1.19 + lxc-ip-prefix: 10.0.9 + + - name: publish the forgejo release + run: | + set -x + + version=1.2.3 + cat > /etc/docker/daemon.json < $binary$suffix + if test "$suffix" = .xz ; then + unxz --keep $binary$suffix + fi + chmod +x $binary + ./$binary --version | grep $version + curl --fail -L -sS $url/root/forgejo/releases/download/v$version/$binary$suffix.sha256 > $binary$suffix.sha256 + shasum -a 256 --check $binary$suffix.sha256 + rm $binary$suffix + done + done + + sources=forgejo-src-$version.tar.gz + curl --fail -L -sS $url/root/forgejo/releases/download/v$version/$sources > $sources + curl --fail -L -sS $url/root/forgejo/releases/download/v$version/$sources.sha256 > $sources.sha256 + shasum -a 256 --check $sources.sha256 + + docker pull ${{ steps.forgejo.outputs.host-port }}/root/forgejo:$version + docker pull ${{ steps.forgejo.outputs.host-port }}/root/forgejo:$version-rootless diff --git a/.forgejo/workflows/build-release.yml b/.forgejo/workflows/build-release.yml new file mode 100644 index 0000000000..80be728a8f --- /dev/null +++ b/.forgejo/workflows/build-release.yml @@ -0,0 +1,155 @@ +name: Build release + +on: + push: + tags: 'v*' + +jobs: + release: + runs-on: self-hosted + # root is used for testing, allow it + if: secrets.ROLE == 'forgejo-integration' || github.repository_owner == 'root' + steps: + - uses: actions/checkout@v3 + + - name: Increase the verbosity when there are no secrets + id: verbose + run: | + if test -z "${{ secrets.TOKEN }}"; then + value=true + else + value=false + fi + echo "value=$value" >> "$GITHUB_OUTPUT" + + - name: Sanitize the name of the repository + id: repository + run: | + set -x # comment out + repository="${{ github.repository }}" + echo "value=${repository##*/}" >> "$GITHUB_OUTPUT" + + - name: When in a test environment, create a token + id: token + if: ${{ secrets.TOKEN == '' }} + run: | + apt-get -qq install -y jq + url="${{ env.GITHUB_SERVER_URL }}" + hostport=${url##http*://} + hostport=${hostport%%/} + doer=root + api=http://$doer:admin1234@$hostport/api/v1/users/$doer/tokens + curl -sS -X DELETE $api/release + token=$(curl -sS -X POST -H 'Content-Type: application/json' --data-raw '{"name": "release", "scopes": ["all"]}' $api | jq --raw-output .sha1) + echo "value=${token}" >> "$GITHUB_OUTPUT" + + - uses: https://code.forgejo.org/actions/setup-go@v4 + with: + go-version: ">=1.20" + check-latest: true + + - name: Create the version from ref_name + id: tag-version + run: | + version="${{ github.ref_name }}" + version=${version##*v} + echo "value=$version" >> "$GITHUB_OUTPUT" + + - name: Create the release notes + id: release-notes + run: | + cat >> "$GITHUB_OUTPUT" </dev/null) +ifneq ($(STORED_VERSION),) + GITEA_VERSION ?= $(STORED_VERSION) else - ifneq ($(GITHUB_REF_NAME),) - VERSION ?= $(subst release/v,,$(GITHUB_REF_NAME)) - else - VERSION ?= main - endif - - STORED_VERSION=$(shell cat $(STORED_VERSION_FILE) 2>/dev/null) - ifneq ($(STORED_VERSION),) - GITEA_VERSION ?= $(STORED_VERSION) - else - GITEA_VERSION ?= $(shell git describe --tags --always | sed 's/-/+/' | sed 's/^v//') - endif -endif - -# if version = "main" then update version to "nightly" -ifeq ($(VERSION),main) - VERSION := main-nightly + GITEA_VERSION ?= $(shell git describe --tags --always | sed 's/-/+/' | sed 's/^v//') endif +VERSION = ${GITEA_VERSION} LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(GITEA_VERSION)" -X "main.Tags=$(TAGS)" @@ -829,9 +812,14 @@ security-check: $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ) CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@ +static-executable: $(GO_SOURCES) $(TAGS_PREREQ) + CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -o $(EXECUTABLE) + .PHONY: release release: frontend generate release-linux release-copy release-compress vendor release-sources release-check +sources-tarbal: vendor release-sources release-check + $(DIST_DIRS): mkdir -p $(DIST_DIRS)