Reduce CI go module downloads, add make targets (#18708, #18475, #18443) (#18741)

Backport #18708 
Backport #18475 
Backport #18443 

The CI currently downloads all go modules in each pipeline step because go modules reside outside the project directory. Fix this by introducing a volume for the `/go` directory [1] so modules are only downloaded once per pipeline using a new `deps-backend` make target.

For completeness, I also included new `deps` and `deps-frontend` targets and the frontend one is also triggered explicitly on CI where needed.

[1] https://docs.drone.io/pipeline/kubernetes/examples/language/golang/#dependencies

* Also backports #18475 and #18443 so that is was able to merge cleanly.
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

.drone.yml

@@ -13,12 +13,25 @@ trigger:
     - tag
     - pull_request
 
+volumes:
+  - name: deps
+    temp: {}
+
 steps:
   - name: deps-frontend
-    pull: always
     image: node:16
+    pull: always
     commands:
-      - make node_modules
+      - make deps-frontend
+
+  - name: deps-backend
+    image: golang:1.17
+    pull: always
+    commands:
+      - make deps-backend
+    volumes:
+      - name: deps
+        path: /go
 
   - name: lint-frontend
     image: node:16
@@ -27,17 +40,17 @@ steps:
     depends_on: [deps-frontend]
 
   - name: lint-backend
-    pull: always
     image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
+    pull: always
     commands:
       - make lint-backend
     environment:
       GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
       GOSUMDB: sum.golang.org
       TAGS: bindata sqlite sqlite_unlock_notify
+    depends_on: [deps-backend]
 
   - name: lint-backend-windows
-    pull: always
     image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
     commands:
       - make golangci-lint vet
@@ -47,9 +60,9 @@ steps:
       TAGS: bindata sqlite sqlite_unlock_notify
       GOOS: windows
       GOARCH: amd64
+    depends_on: [deps-backend]
 
   - name: lint-backend-gogit
-    pull: always
     image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
     commands:
       - make lint-backend
@@ -57,6 +70,7 @@ steps:
       GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
       GOSUMDB: sum.golang.org
       TAGS: bindata gogit sqlite sqlite_unlock_notify
+    depends_on: [deps-backend]
 
   - name: checks-frontend
     image: node:16
@@ -65,11 +79,13 @@ steps:
     depends_on: [deps-frontend]
 
   - name: checks-backend
-    pull: always
     image: golang:1.17
     commands:
       - make checks-backend
-    depends_on: [lint-backend]
+    depends_on: [deps-backend]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: test-frontend
     image: node:16
@@ -84,14 +100,17 @@ steps:
     depends_on: [test-frontend]
 
   - name: build-backend-no-gcc
-    pull: always
     image: golang:1.16 # this step is kept as the lowest version of golang that we support
+    pull: always
     environment:
       GO111MODULE: on
       GOPROXY: https://goproxy.cn
     commands:
       - go build -o gitea_no_gcc # test if build succeeds without the sqlite tag
-    depends_on: [checks-backend]
+    depends_on: [deps-backend, checks-backend]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: build-backend-arm64
     image: golang:1.17
@@ -104,7 +123,10 @@ steps:
     commands:
       - make backend # test cross compile
       - rm ./gitea # clean
-    depends_on: [checks-backend]
+    depends_on: [deps-backend, checks-backend]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: build-backend-windows
     image: golang:1.17
@@ -116,7 +138,10 @@ steps:
       TAGS: bindata gogit
     commands:
       - go build -o gitea_windows
-    depends_on: [checks-backend]
+    depends_on: [deps-backend, checks-backend]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: build-backend-386
     image: golang:1.17
@@ -127,7 +152,10 @@ steps:
       GOARCH: 386
     commands:
       - go build -o gitea_linux_386 # test if compatible with 32 bit
-    depends_on: [checks-backend]
+    depends_on: [deps-backend, checks-backend]
+    volumes:
+      - name: deps
+        path: /go
 
 ---
 kind: pipeline
@@ -147,21 +175,28 @@ trigger:
     - tag
     - pull_request
 
+volumes:
+  - name: deps
+    temp: {}
+
 services:
   - name: mysql
     image: mysql:5.7
+    pull: always
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: yes
       MYSQL_DATABASE: test
 
   - name: mysql8
     image: mysql:8
+    pull: always
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: yes
       MYSQL_DATABASE: testgitea
 
   - name: mssql
     image: mcr.microsoft.com/mssql/server:latest
+    pull: always
     environment:
       ACCEPT_EULA: Y
       MSSQL_PID: Standard
@@ -169,14 +204,17 @@ services:
 
   - name: ldap
     image: gitea/test-openldap:latest
+    pull: always
 
   - name: elasticsearch
+    image: elasticsearch:7.5.0
+    pull: always
     environment:
       discovery.type: single-node
-    image: elasticsearch:7.5.0
 
   - name: minio
     image: minio/minio:RELEASE.2021-03-12T00-00-47Z
+    pull: always
     commands:
     - minio server /data
     environment:
@@ -186,6 +224,7 @@ services:
 steps:
   - name: fetch-tags
     image: docker:git
+    pull: always
     commands:
       - git fetch --tags --force
     when:
@@ -193,19 +232,28 @@ steps:
         exclude:
           - pull_request
 
-  - name: tag-pre-condition
+  - name: deps-backend
+    image: golang:1.17
     pull: always
+    commands:
+      - make deps-backend
+    volumes:
+      - name: deps
+        path: /go
+
+  - name: tag-pre-condition
     image: drone/git
+    pull: always
     commands:
       - git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA}
 
   - name: prepare-test-env
     image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
+    pull: always
     commands:
       - ./build/test-env-prepare.sh
 
   - name: build
-    pull: always
     image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
     user: gitea
     commands:
@@ -215,8 +263,10 @@ steps:
       GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
       GOSUMDB: sum.golang.org
       TAGS: bindata sqlite sqlite_unlock_notify
-    depends_on:
-      - prepare-test-env
+    depends_on: [deps-backend, prepare-test-env]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: unit-test
     image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
@@ -229,9 +279,12 @@ steps:
       RACE_ENABLED: true
       GITHUB_READ_TOKEN:
         from_secret: github_read_token
+    depends_on: [deps-backend, prepare-test-env]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: unit-test-gogit
-    pull: always
     image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
     user: gitea
     commands:
@@ -242,6 +295,10 @@ steps:
       RACE_ENABLED: true
       GITHUB_READ_TOKEN:
         from_secret: github_read_token
+    depends_on: [deps-backend, prepare-test-env]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: test-mysql
     image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
@@ -255,8 +312,10 @@ steps:
       TEST_LDAP: 1
       USE_REPO_TEST_DIR: 1
       TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200"
-    depends_on:
-      - build
+    depends_on: [build]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: test-mysql8
     image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
@@ -269,8 +328,10 @@ steps:
       RACE_ENABLED: true
       TEST_LDAP: 1
       USE_REPO_TEST_DIR: 1
-    depends_on:
-      - build
+    depends_on: [build]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: test-mssql
     image: gitea/test_env:linux-amd64  # https://gitea.com/gitea/test-env
@@ -283,8 +344,10 @@ steps:
       RACE_ENABLED: true
       TEST_LDAP: 1
       USE_REPO_TEST_DIR: 1
-    depends_on:
-      - build
+    depends_on: [build]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: generate-coverage
     image: golang:1.17
@@ -293,9 +356,7 @@ steps:
     environment:
       GOPROXY: https://goproxy.cn
       TAGS: bindata
-    depends_on:
-      - unit-test
-      - test-mysql
+    depends_on: [unit-test, test-mysql]
     when:
       branch:
         - main
@@ -304,15 +365,14 @@ steps:
         - pull_request
 
   - name: coverage-codecov
+    image: woodpeckerci/plugin-codecov:next-alpine
     pull: always
-    image: plugins/codecov
     settings:
       files:
         - coverage.all
       token:
         from_secret: codecov_token
-    depends_on:
-      - generate-coverage
+    depends_on: [generate-coverage]
     when:
       branch:
         - main
@@ -337,6 +397,10 @@ trigger:
     - tag
     - pull_request
 
+volumes:
+  - name: deps
+    temp: {}
+
 services:
   - name: pgsql
     pull: default
@@ -352,6 +416,7 @@ services:
 steps:
   - name: fetch-tags
     image: docker:git
+    pull: always
     commands:
       - git fetch --tags --force
     when:
@@ -359,13 +424,22 @@ steps:
         exclude:
           - pull_request
 
+  - name: deps-backend
+    image: golang:1.17
+    pull: always
+    commands:
+      - make deps-backend
+    volumes:
+      - name: deps
+        path: /go
+
   - name: prepare-test-env
     image: gitea/test_env:linux-arm64  # https://gitea.com/gitea/test-env
+    pull: always
     commands:
       - ./build/test-env-prepare.sh
 
   - name: build
-    pull: always
     image: gitea/test_env:linux-arm64  # https://gitea.com/gitea/test-env
     user: gitea
     commands:
@@ -375,8 +449,10 @@ steps:
       GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
       GOSUMDB: sum.golang.org
       TAGS: bindata gogit sqlite sqlite_unlock_notify
-    depends_on:
-      - prepare-test-env
+    depends_on: [deps-backend, prepare-test-env]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: test-sqlite
     image: gitea/test_env:linux-arm64  # https://gitea.com/gitea/test-env
@@ -389,8 +465,10 @@ steps:
       RACE_ENABLED: true
       TEST_TAGS: gogit sqlite sqlite_unlock_notify
       USE_REPO_TEST_DIR: 1
-    depends_on:
-      - build
+    depends_on: [build]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: test-pgsql
     image: gitea/test_env:linux-arm64  # https://gitea.com/gitea/test-env
@@ -404,8 +482,10 @@ steps:
       TEST_TAGS: gogit
       TEST_LDAP: 1
       USE_REPO_TEST_DIR: 1
-    depends_on:
-      - build
+    depends_on: [build]
+    volumes:
+      - name: deps
+        path: /go
 
 ---
 kind: pipeline
@@ -425,8 +505,8 @@ trigger:
 
 steps:
   - name: download
-    pull: always
     image: jonasfranz/crowdin
+    pull: always
     settings:
       download: true
       export_dir: options/locale/
@@ -437,14 +517,14 @@ steps:
         from_secret: crowdin_key
 
   - name: update
-    pull: default
     image: alpine:3.13
+    pull: always
     commands:
       - ./build/update-locales.sh
 
   - name: push
-    pull: always
     image: appleboy/drone-git-push
+    pull: always
     settings:
       author_email: "teabot@gitea.io"
       author_name: GiteaBot
@@ -457,8 +537,8 @@ steps:
         from_secret: git_push_ssh_key
 
   - name: upload_translations
-    pull: always
     image: jonasfranz/crowdin
+    pull: always
     settings:
       files:
         locale_en-US.ini: options/locale/locale_en-US.ini
@@ -488,12 +568,13 @@ trigger:
 steps:
   - name: download
     image: golang:1.17
+    pull: always
     commands:
       - timeout -s ABRT 40m make generate-license generate-gitignore
 
   - name: push
-    pull: always
     image: appleboy/drone-git-push
+    pull: always
     settings:
       author_email: "teabot@gitea.io"
       author_name: GiteaBot
@@ -529,15 +610,35 @@ depends_on:
   - testing-amd64
   - testing-arm64
 
+volumes:
+  - name: deps
+    temp: {}
+
 steps:
   - name: fetch-tags
     image: docker:git
+    pull: always
     commands:
       - git fetch --tags --force
 
-  - name: static
+  - name: deps-frontend
+    image: node:16
     pull: always
+    commands:
+      - make deps-frontend
+
+  - name: deps-backend
+    image: golang:1.17
+    pull: always
+    commands:
+      - make deps-backend
+    volumes:
+      - name: deps
+        path: /go
+
+  - name: static
     image: techknowlogick/xgo:go-1.17.x
+    pull: always
     commands:
       - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get install -y nodejs
       - export PATH=$PATH:$GOPATH/bin
@@ -545,10 +646,13 @@ steps:
     environment:
       GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
       TAGS: bindata sqlite sqlite_unlock_notify
+    volumes:
+      - name: deps
+        path: /go
 
   - name: gpg-sign
-    pull: always
     image: plugins/gpgsign:1
+    pull: always
     settings:
       detach_sign: true
       excludes:
@@ -562,8 +666,8 @@ steps:
         from_secret: gpgsign_passphrase
 
   - name: release-branch
-    pull: always
     image: woodpeckerci/plugin-s3:latest
+    pull: always
     settings:
       acl: public-read
       bucket: gitea-artifacts
@@ -624,16 +728,35 @@ depends_on:
   - testing-arm64
   - testing-amd64
 
+volumes:
+  - name: deps
+    temp: {}
+
 steps:
   - name: fetch-tags
-    pull: default
     image: docker:git
+    pull: always
     commands:
       - git fetch --tags --force
 
-  - name: static
+  - name: deps-frontend
+    image: node:16
     pull: always
+    commands:
+      - make deps-frontend
+
+  - name: deps-backend
+    image: golang:1.17
+    pull: always
+    commands:
+      - make deps-backend
+    volumes:
+      - name: deps
+        path: /go
+
+  - name: static
     image: techknowlogick/xgo:go-1.17.x
+    pull: always
     commands:
       - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get install -y nodejs
       - export PATH=$PATH:$GOPATH/bin
@@ -641,10 +764,14 @@ steps:
     environment:
       GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
       TAGS: bindata sqlite sqlite_unlock_notify
+    depends_on: [fetch-tags]
+    volumes:
+      - name: deps
+        path: /go
 
   - name: gpg-sign
-    pull: always
     image: plugins/gpgsign:1
+    pull: always
     settings:
       detach_sign: true
       excludes:
@@ -656,10 +783,11 @@ steps:
         from_secret: gpgsign_key
       GPGSIGN_PASSPHRASE:
         from_secret: gpgsign_passphrase
+    depends_on: [static]
 
   - name: release-tag
-    pull: always
     image: woodpeckerci/plugin-s3:latest
+    pull: always
     settings:
       acl: public-read
       bucket: gitea-artifacts
@@ -673,16 +801,18 @@ steps:
         from_secret: aws_access_key_id
       AWS_SECRET_ACCESS_KEY:
         from_secret: aws_secret_access_key
+    depends_on: [gpg-sign]
 
   - name: github
-    pull: always
     image: plugins/github-release:1
+    pull: always
     settings:
       files:
         - "dist/release/*"
     environment:
       GITHUB_TOKEN:
         from_secret: github_token
+    depends_on: [gpg-sign]
 
 ---
 kind: pipeline
@@ -704,16 +834,16 @@ trigger:
 
 steps:
   - name: build-docs
-    pull: always
     image: plugins/hugo:latest
+    pull: always
     commands:
       - apk add --no-cache make bash curl
       - cd docs
       - make trans-copy clean build
 
   - name: publish-docs
-    pull: always
     image: techknowlogick/drone-netlify:latest
+    pull: always
     settings:
       path: docs/public/
       site_id: d2260bae-7861-4c02-8646-8f6440b12672
@@ -749,12 +879,13 @@ trigger:
 steps:
   - name: fetch-tags
     image: docker:git
+    pull: always
     commands:
       - git fetch --tags --force
 
   - name: publish
-    pull: always
     image: techknowlogick/drone-docker:latest
+    pull: always
     settings:
       auto_tag: true
       auto_tag_suffix: linux-amd64
@@ -811,12 +942,13 @@ trigger:
 steps:
   - name: fetch-tags
     image: docker:git
+    pull: always
     commands:
       - git fetch --tags --force
 
   - name: publish
-    pull: always
     image: techknowlogick/drone-docker:latest
+    pull: always
     settings:
       auto_tag: false
       tags: dev-linux-amd64
@@ -872,12 +1004,13 @@ trigger:
 steps:
   - name: fetch-tags
     image: docker:git
+    pull: always
     commands:
       - git fetch --tags --force
 
   - name: publish
-    pull: always
     image: techknowlogick/drone-docker:latest
+    pull: always
     settings:
       auto_tag: false
       tags: ${DRONE_BRANCH##release/v}-dev-linux-amd64
@@ -929,8 +1062,8 @@ trigger:
 
 steps:
   - name: dryrun
-    pull: always
     image: techknowlogick/drone-docker:latest
+    pull: always
     settings:
       dry_run: true
       repo: gitea/gitea
@@ -967,12 +1100,13 @@ trigger:
 steps:
   - name: fetch-tags
     image: docker:git
+    pull: always
     commands:
       - git fetch --tags --force
 
   - name: publish
-    pull: always
     image: techknowlogick/drone-docker:latest
+    pull: always
     settings:
       auto_tag: true
       auto_tag_suffix: linux-arm64
@@ -1029,12 +1163,13 @@ trigger:
 steps:
   - name: fetch-tags
     image: docker:git
+    pull: always
     commands:
       - git fetch --tags --force
 
   - name: publish
-    pull: always
     image: techknowlogick/drone-docker:latest
+    pull: always
     settings:
       auto_tag: false
       tags: dev-linux-arm64
@@ -1090,12 +1225,13 @@ trigger:
 steps:
   - name: fetch-tags
     image: docker:git
+    pull: always
     commands:
       - git fetch --tags --force
 
   - name: publish
-    pull: always
     image: techknowlogick/drone-docker:latest
+    pull: always
     settings:
       auto_tag: false
       tags: ${DRONE_BRANCH##release/v}-dev-linux-arm64
@@ -1140,8 +1276,8 @@ platform:
 
 steps:
   - name: manifest-rootless
-    pull: always
     image: plugins/manifest
+    pull: always
     settings:
       auto_tag: true
       ignore_missing: true
@@ -1186,6 +1322,7 @@ steps:
   - name: manifest-rootless
     pull: always
     image: plugins/manifest
+    pull: always
     settings:
       auto_tag: false
       ignore_missing: true
@@ -1260,8 +1397,8 @@ depends_on:
 
 steps:
   - name: discord
-    pull: always
     image: appleboy/drone-discord:1.2.4
+    pull: always
     settings:
       message: "{{#success build.status}} ✅  Build #{{build.number}} of `{{repo.name}}` succeeded.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{else}} ❌  Build #{{build.number}} of `{{repo.name}}` failed.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{/success}}\n"
       webhook_id:

.golangci.yml

@@ -23,6 +23,10 @@ linters:
 
 run:
   timeout: 3m
+  skip-dirs:
+    - node_modules
+    - public
+    - web_src
 
 linters-settings:
   gocritic:

Makefile

@@ -166,6 +166,9 @@ help:
 	@echo " - watch-backend                    watch backend files and continuously rebuild"
 	@echo " - clean                            delete backend and integration files"
 	@echo " - clean-all                        delete backend, frontend and integration files"
+	@echo " - deps                             install dependencies"
+	@echo " - deps-frontend                    install frontend dependencies"
+	@echo " - deps-backend                     install backend dependencies"
 	@echo " - lint                             lint everything"
 	@echo " - lint-frontend                    lint frontend files"
 	@echo " - lint-backend                     lint backend files"
@@ -656,6 +659,16 @@ docs:
 	fi
 	cd docs; make trans-copy clean build-offline;
 
+.PHONY: deps
+deps: deps-frontend deps-backend
+
+.PHONY: deps-frontend
+deps-frontend: node_modules
+
+.PHONY: deps-backend
+deps-backend:
+	$(GO) mod download
+
 node_modules: package-lock.json
 	npm install --no-save
 	@touch node_modules