From c04a4afac1e513297c7b24ef62e92487d0035288 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 12 Feb 2022 11:43:08 +0100 Subject: [PATCH] Reduce CI go module downloads, add make targets (#18708, #18475, #18443) (#18741) Backport #18708 Backport #18475 Backport #18443 The CI currently downloads all go modules in each pipeline step because go modules reside outside the project directory. Fix this by introducing a volume for the `/go` directory [1] so modules are only downloaded once per pipeline using a new `deps-backend` make target. For completeness, I also included new `deps` and `deps-frontend` targets and the frontend one is also triggered explicitly on CI where needed. [1] https://docs.drone.io/pipeline/kubernetes/examples/language/golang/#dependencies * Also backports #18475 and #18443 so that is was able to merge cleanly. Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: techknowlogick --- .drone.yml | 259 ++++++++++++++++++++++++++++++++++++++------------ .golangci.yml | 4 + Makefile | 13 +++ 3 files changed, 215 insertions(+), 61 deletions(-) diff --git a/.drone.yml b/.drone.yml index d9de559039..bba8747482 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,12 +13,25 @@ trigger: - tag - pull_request +volumes: + - name: deps + temp: {} + steps: - name: deps-frontend - pull: always image: node:16 + pull: always commands: - - make node_modules + - make deps-frontend + + - name: deps-backend + image: golang:1.17 + pull: always + commands: + - make deps-backend + volumes: + - name: deps + path: /go - name: lint-frontend image: node:16 @@ -27,17 +40,17 @@ steps: depends_on: [deps-frontend] - name: lint-backend - pull: always image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env + pull: always commands: - make lint-backend environment: GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not GOSUMDB: sum.golang.org TAGS: bindata sqlite sqlite_unlock_notify + depends_on: [deps-backend] - name: lint-backend-windows - pull: always image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env commands: - make golangci-lint vet @@ -47,9 +60,9 @@ steps: TAGS: bindata sqlite sqlite_unlock_notify GOOS: windows GOARCH: amd64 + depends_on: [deps-backend] - name: lint-backend-gogit - pull: always image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env commands: - make lint-backend @@ -57,6 +70,7 @@ steps: GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not GOSUMDB: sum.golang.org TAGS: bindata gogit sqlite sqlite_unlock_notify + depends_on: [deps-backend] - name: checks-frontend image: node:16 @@ -65,11 +79,13 @@ steps: depends_on: [deps-frontend] - name: checks-backend - pull: always image: golang:1.17 commands: - make checks-backend - depends_on: [lint-backend] + depends_on: [deps-backend] + volumes: + - name: deps + path: /go - name: test-frontend image: node:16 @@ -84,14 +100,17 @@ steps: depends_on: [test-frontend] - name: build-backend-no-gcc - pull: always image: golang:1.16 # this step is kept as the lowest version of golang that we support + pull: always environment: GO111MODULE: on GOPROXY: https://goproxy.cn commands: - go build -o gitea_no_gcc # test if build succeeds without the sqlite tag - depends_on: [checks-backend] + depends_on: [deps-backend, checks-backend] + volumes: + - name: deps + path: /go - name: build-backend-arm64 image: golang:1.17 @@ -104,7 +123,10 @@ steps: commands: - make backend # test cross compile - rm ./gitea # clean - depends_on: [checks-backend] + depends_on: [deps-backend, checks-backend] + volumes: + - name: deps + path: /go - name: build-backend-windows image: golang:1.17 @@ -116,7 +138,10 @@ steps: TAGS: bindata gogit commands: - go build -o gitea_windows - depends_on: [checks-backend] + depends_on: [deps-backend, checks-backend] + volumes: + - name: deps + path: /go - name: build-backend-386 image: golang:1.17 @@ -127,7 +152,10 @@ steps: GOARCH: 386 commands: - go build -o gitea_linux_386 # test if compatible with 32 bit - depends_on: [checks-backend] + depends_on: [deps-backend, checks-backend] + volumes: + - name: deps + path: /go --- kind: pipeline @@ -147,21 +175,28 @@ trigger: - tag - pull_request +volumes: + - name: deps + temp: {} + services: - name: mysql image: mysql:5.7 + pull: always environment: MYSQL_ALLOW_EMPTY_PASSWORD: yes MYSQL_DATABASE: test - name: mysql8 image: mysql:8 + pull: always environment: MYSQL_ALLOW_EMPTY_PASSWORD: yes MYSQL_DATABASE: testgitea - name: mssql image: mcr.microsoft.com/mssql/server:latest + pull: always environment: ACCEPT_EULA: Y MSSQL_PID: Standard @@ -169,14 +204,17 @@ services: - name: ldap image: gitea/test-openldap:latest + pull: always - name: elasticsearch + image: elasticsearch:7.5.0 + pull: always environment: discovery.type: single-node - image: elasticsearch:7.5.0 - name: minio image: minio/minio:RELEASE.2021-03-12T00-00-47Z + pull: always commands: - minio server /data environment: @@ -186,6 +224,7 @@ services: steps: - name: fetch-tags image: docker:git + pull: always commands: - git fetch --tags --force when: @@ -193,19 +232,28 @@ steps: exclude: - pull_request - - name: tag-pre-condition + - name: deps-backend + image: golang:1.17 pull: always + commands: + - make deps-backend + volumes: + - name: deps + path: /go + + - name: tag-pre-condition image: drone/git + pull: always commands: - git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA} - name: prepare-test-env image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env + pull: always commands: - ./build/test-env-prepare.sh - name: build - pull: always image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env user: gitea commands: @@ -215,8 +263,10 @@ steps: GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not GOSUMDB: sum.golang.org TAGS: bindata sqlite sqlite_unlock_notify - depends_on: - - prepare-test-env + depends_on: [deps-backend, prepare-test-env] + volumes: + - name: deps + path: /go - name: unit-test image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env @@ -229,9 +279,12 @@ steps: RACE_ENABLED: true GITHUB_READ_TOKEN: from_secret: github_read_token + depends_on: [deps-backend, prepare-test-env] + volumes: + - name: deps + path: /go - name: unit-test-gogit - pull: always image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env user: gitea commands: @@ -242,6 +295,10 @@ steps: RACE_ENABLED: true GITHUB_READ_TOKEN: from_secret: github_read_token + depends_on: [deps-backend, prepare-test-env] + volumes: + - name: deps + path: /go - name: test-mysql image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env @@ -255,8 +312,10 @@ steps: TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" - depends_on: - - build + depends_on: [build] + volumes: + - name: deps + path: /go - name: test-mysql8 image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env @@ -269,8 +328,10 @@ steps: RACE_ENABLED: true TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 - depends_on: - - build + depends_on: [build] + volumes: + - name: deps + path: /go - name: test-mssql image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env @@ -283,8 +344,10 @@ steps: RACE_ENABLED: true TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 - depends_on: - - build + depends_on: [build] + volumes: + - name: deps + path: /go - name: generate-coverage image: golang:1.17 @@ -293,9 +356,7 @@ steps: environment: GOPROXY: https://goproxy.cn TAGS: bindata - depends_on: - - unit-test - - test-mysql + depends_on: [unit-test, test-mysql] when: branch: - main @@ -304,15 +365,14 @@ steps: - pull_request - name: coverage-codecov + image: woodpeckerci/plugin-codecov:next-alpine pull: always - image: plugins/codecov settings: files: - coverage.all token: from_secret: codecov_token - depends_on: - - generate-coverage + depends_on: [generate-coverage] when: branch: - main @@ -337,6 +397,10 @@ trigger: - tag - pull_request +volumes: + - name: deps + temp: {} + services: - name: pgsql pull: default @@ -352,6 +416,7 @@ services: steps: - name: fetch-tags image: docker:git + pull: always commands: - git fetch --tags --force when: @@ -359,13 +424,22 @@ steps: exclude: - pull_request + - name: deps-backend + image: golang:1.17 + pull: always + commands: + - make deps-backend + volumes: + - name: deps + path: /go + - name: prepare-test-env image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env + pull: always commands: - ./build/test-env-prepare.sh - name: build - pull: always image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env user: gitea commands: @@ -375,8 +449,10 @@ steps: GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not GOSUMDB: sum.golang.org TAGS: bindata gogit sqlite sqlite_unlock_notify - depends_on: - - prepare-test-env + depends_on: [deps-backend, prepare-test-env] + volumes: + - name: deps + path: /go - name: test-sqlite image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env @@ -389,8 +465,10 @@ steps: RACE_ENABLED: true TEST_TAGS: gogit sqlite sqlite_unlock_notify USE_REPO_TEST_DIR: 1 - depends_on: - - build + depends_on: [build] + volumes: + - name: deps + path: /go - name: test-pgsql image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env @@ -404,8 +482,10 @@ steps: TEST_TAGS: gogit TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 - depends_on: - - build + depends_on: [build] + volumes: + - name: deps + path: /go --- kind: pipeline @@ -425,8 +505,8 @@ trigger: steps: - name: download - pull: always image: jonasfranz/crowdin + pull: always settings: download: true export_dir: options/locale/ @@ -437,14 +517,14 @@ steps: from_secret: crowdin_key - name: update - pull: default image: alpine:3.13 + pull: always commands: - ./build/update-locales.sh - name: push - pull: always image: appleboy/drone-git-push + pull: always settings: author_email: "teabot@gitea.io" author_name: GiteaBot @@ -457,8 +537,8 @@ steps: from_secret: git_push_ssh_key - name: upload_translations - pull: always image: jonasfranz/crowdin + pull: always settings: files: locale_en-US.ini: options/locale/locale_en-US.ini @@ -488,12 +568,13 @@ trigger: steps: - name: download image: golang:1.17 + pull: always commands: - timeout -s ABRT 40m make generate-license generate-gitignore - name: push - pull: always image: appleboy/drone-git-push + pull: always settings: author_email: "teabot@gitea.io" author_name: GiteaBot @@ -529,15 +610,35 @@ depends_on: - testing-amd64 - testing-arm64 +volumes: + - name: deps + temp: {} + steps: - name: fetch-tags image: docker:git + pull: always commands: - git fetch --tags --force - - name: static + - name: deps-frontend + image: node:16 pull: always + commands: + - make deps-frontend + + - name: deps-backend + image: golang:1.17 + pull: always + commands: + - make deps-backend + volumes: + - name: deps + path: /go + + - name: static image: techknowlogick/xgo:go-1.17.x + pull: always commands: - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get install -y nodejs - export PATH=$PATH:$GOPATH/bin @@ -545,10 +646,13 @@ steps: environment: GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not TAGS: bindata sqlite sqlite_unlock_notify + volumes: + - name: deps + path: /go - name: gpg-sign - pull: always image: plugins/gpgsign:1 + pull: always settings: detach_sign: true excludes: @@ -562,8 +666,8 @@ steps: from_secret: gpgsign_passphrase - name: release-branch - pull: always image: woodpeckerci/plugin-s3:latest + pull: always settings: acl: public-read bucket: gitea-artifacts @@ -624,16 +728,35 @@ depends_on: - testing-arm64 - testing-amd64 +volumes: + - name: deps + temp: {} + steps: - name: fetch-tags - pull: default image: docker:git + pull: always commands: - git fetch --tags --force - - name: static + - name: deps-frontend + image: node:16 pull: always + commands: + - make deps-frontend + + - name: deps-backend + image: golang:1.17 + pull: always + commands: + - make deps-backend + volumes: + - name: deps + path: /go + + - name: static image: techknowlogick/xgo:go-1.17.x + pull: always commands: - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get install -y nodejs - export PATH=$PATH:$GOPATH/bin @@ -641,10 +764,14 @@ steps: environment: GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not TAGS: bindata sqlite sqlite_unlock_notify + depends_on: [fetch-tags] + volumes: + - name: deps + path: /go - name: gpg-sign - pull: always image: plugins/gpgsign:1 + pull: always settings: detach_sign: true excludes: @@ -656,10 +783,11 @@ steps: from_secret: gpgsign_key GPGSIGN_PASSPHRASE: from_secret: gpgsign_passphrase + depends_on: [static] - name: release-tag - pull: always image: woodpeckerci/plugin-s3:latest + pull: always settings: acl: public-read bucket: gitea-artifacts @@ -673,16 +801,18 @@ steps: from_secret: aws_access_key_id AWS_SECRET_ACCESS_KEY: from_secret: aws_secret_access_key + depends_on: [gpg-sign] - name: github - pull: always image: plugins/github-release:1 + pull: always settings: files: - "dist/release/*" environment: GITHUB_TOKEN: from_secret: github_token + depends_on: [gpg-sign] --- kind: pipeline @@ -704,16 +834,16 @@ trigger: steps: - name: build-docs - pull: always image: plugins/hugo:latest + pull: always commands: - apk add --no-cache make bash curl - cd docs - make trans-copy clean build - name: publish-docs - pull: always image: techknowlogick/drone-netlify:latest + pull: always settings: path: docs/public/ site_id: d2260bae-7861-4c02-8646-8f6440b12672 @@ -749,12 +879,13 @@ trigger: steps: - name: fetch-tags image: docker:git + pull: always commands: - git fetch --tags --force - name: publish - pull: always image: techknowlogick/drone-docker:latest + pull: always settings: auto_tag: true auto_tag_suffix: linux-amd64 @@ -811,12 +942,13 @@ trigger: steps: - name: fetch-tags image: docker:git + pull: always commands: - git fetch --tags --force - name: publish - pull: always image: techknowlogick/drone-docker:latest + pull: always settings: auto_tag: false tags: dev-linux-amd64 @@ -872,12 +1004,13 @@ trigger: steps: - name: fetch-tags image: docker:git + pull: always commands: - git fetch --tags --force - name: publish - pull: always image: techknowlogick/drone-docker:latest + pull: always settings: auto_tag: false tags: ${DRONE_BRANCH##release/v}-dev-linux-amd64 @@ -929,8 +1062,8 @@ trigger: steps: - name: dryrun - pull: always image: techknowlogick/drone-docker:latest + pull: always settings: dry_run: true repo: gitea/gitea @@ -967,12 +1100,13 @@ trigger: steps: - name: fetch-tags image: docker:git + pull: always commands: - git fetch --tags --force - name: publish - pull: always image: techknowlogick/drone-docker:latest + pull: always settings: auto_tag: true auto_tag_suffix: linux-arm64 @@ -1029,12 +1163,13 @@ trigger: steps: - name: fetch-tags image: docker:git + pull: always commands: - git fetch --tags --force - name: publish - pull: always image: techknowlogick/drone-docker:latest + pull: always settings: auto_tag: false tags: dev-linux-arm64 @@ -1090,12 +1225,13 @@ trigger: steps: - name: fetch-tags image: docker:git + pull: always commands: - git fetch --tags --force - name: publish - pull: always image: techknowlogick/drone-docker:latest + pull: always settings: auto_tag: false tags: ${DRONE_BRANCH##release/v}-dev-linux-arm64 @@ -1140,8 +1276,8 @@ platform: steps: - name: manifest-rootless - pull: always image: plugins/manifest + pull: always settings: auto_tag: true ignore_missing: true @@ -1186,6 +1322,7 @@ steps: - name: manifest-rootless pull: always image: plugins/manifest + pull: always settings: auto_tag: false ignore_missing: true @@ -1260,8 +1397,8 @@ depends_on: steps: - name: discord - pull: always image: appleboy/drone-discord:1.2.4 + pull: always settings: message: "{{#success build.status}} āœ… Build #{{build.number}} of `{{repo.name}}` succeeded.\n\nšŸ“ Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\nšŸŒ {{ build.link }} {{else}} āŒ Build #{{build.number}} of `{{repo.name}}` failed.\n\nšŸ“ Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\nšŸŒ {{ build.link }} {{/success}}\n" webhook_id: diff --git a/.golangci.yml b/.golangci.yml index 235bb76715..8fc377bbab 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,6 +23,10 @@ linters: run: timeout: 3m + skip-dirs: + - node_modules + - public + - web_src linters-settings: gocritic: diff --git a/Makefile b/Makefile index 342366492c..6e5fad719e 100644 --- a/Makefile +++ b/Makefile @@ -166,6 +166,9 @@ help: @echo " - watch-backend watch backend files and continuously rebuild" @echo " - clean delete backend and integration files" @echo " - clean-all delete backend, frontend and integration files" + @echo " - deps install dependencies" + @echo " - deps-frontend install frontend dependencies" + @echo " - deps-backend install backend dependencies" @echo " - lint lint everything" @echo " - lint-frontend lint frontend files" @echo " - lint-backend lint backend files" @@ -656,6 +659,16 @@ docs: fi cd docs; make trans-copy clean build-offline; +.PHONY: deps +deps: deps-frontend deps-backend + +.PHONY: deps-frontend +deps-frontend: node_modules + +.PHONY: deps-backend +deps-backend: + $(GO) mod download + node_modules: package-lock.json npm install --no-save @touch node_modules