Improve ci tests (#353)

* improve ci tests

* filter store

* improve ci configs

* fix create dir before accessing

.woodpecker/docs.yml

@@ -1,7 +1,3 @@
-clone:
-  git:
-    image: plugins/git:next
-
 pipeline:
   build:
     image: node:14-alpine
@@ -25,7 +21,7 @@ pipeline:
       - chmod 0600 $HOME/.ssh/id_rsa
       - git config --global user.email "woodpecker-bot@obermui.de"
       - git config --global user.name "woodpecker-bot"
-      - git clone -b master --single-branch git@github.com:woodpecker-ci/woodpecker-ci.github.io.git /repo
+      - git clone -b ${CI_REPO_DEFAULT_BRANCH} --single-branch git@github.com:woodpecker-ci/woodpecker-ci.github.io.git /repo
       # copy all docs files and delete all old ones, but leave CNAME and index.yaml untouched
       - rsync -r --exclude .git --exclude CNAME --exclude index.yaml --exclude README.md --delete docs/build/ /repo
       - cd /repo
@@ -36,6 +32,6 @@ pipeline:
       - git push
     when:
       event: push
-      branch: master
+      branch: ${CI_REPO_DEFAULT_BRANCH}
       path: "docs/**"
 

.woodpecker/helm.yml

@@ -1,7 +1,3 @@
-clone:
-  git:
-    image: plugins/git:next
-
 pipeline:
   set-version:
     image: alpine/helm:3.5.3

.woodpecker/main.yml

@@ -1,93 +1,10 @@
-clone:
-  git:
-    image: plugins/git:next
-
 pipeline:
-  web-deps:
+  build-web:
     image: node:16-alpine
     commands:
       - cd web/
       - yarn install --frozen-lockfile
-    when:
-      path: "web/**"
-
-#  TODO: enable if we have enouth mem (~2g) to lint, cause an oom atm.
-#        For reviewers, please run localy to verify it passes
-#  web-lint:
-#    TODO: disabled group for now to prevent oom
-#    group: web-test
-#    image: node:16-alpine
-#    commands:
-#      - cd web/
-#      - yarn lint
-#    when:
-#      path: "web/**"
-
-  web-formatcheck:
-    group: web-test
-    image: node:16-alpine
-    commands:
-      - cd web/
-      - yarn formatcheck
-    when:
-      path: "web/**"
-
-  web-typecheck:
-    group: web-test
-    image: node:16-alpine
-    commands:
-      - cd web/
-      - yarn typecheck
-    when:
-      path: "web/**"
-
-  web-test:
-    group: web-test
-    image: node:16-alpine
-    commands:
-      - cd web/
-      - yarn test
-    when:
-      path: "web/**"
-
-  web-build:
-    image: node:16-alpine
-    commands:
-      - cd web/
       - yarn build
-    when:
-      path: "web/**"
-
-  test:
-    image: golang:1.16
-    group: test
-    commands:
-      - make test
-      - make lint
-
-  test-postgres:
-    image: golang:1.16
-    group: db-test
-    environment:
-      - WOODPECKER_DATABASE_DRIVER=postgres
-      - WOODPECKER_DATABASE_CONFIG=host=postgres user=postgres dbname=postgres sslmode=disable
-    commands:
-      - go test -timeout 30s github.com/woodpecker-ci/woodpecker/server/store/datastore
-
-  test-mysql:
-    image: golang:1.16
-    group: db-test
-    environment:
-      - WOODPECKER_DATABASE_DRIVER=mysql
-      - WOODPECKER_DATABASE_CONFIG=root@tcp(mysql:3306)/test?parseTime=true
-    commands:
-      - go test -timeout 30s github.com/woodpecker-ci/woodpecker/server/store/datastore
-
-  build-frontend:
-    image: node:16-alpine
-    commands:
-      - apk add make
-      - make release-frontend
 
   build-server:
     group: build
@@ -114,7 +31,7 @@ pipeline:
       - make bundle
     when:
       event: [push, tag]
-      branch: master
+      branch: ${CI_REPO_DEFAULT_BRANCH}
 
   publish-server:
     group: bundle
@@ -125,7 +42,7 @@ pipeline:
     # TODO: only release as next after 0.15.0 got released
     tag: [next, latest]
     when:
-      branch: master
+      branch: ${CI_REPO_DEFAULT_BRANCH}
       event: push
 
   publish-server-alpine:
@@ -137,7 +54,7 @@ pipeline:
     # TODO: only release as next-alpine after 0.15.0 got released
     tag: [next-alpine, latest-alpine]
     when:
-      branch: master
+      branch: ${CI_REPO_DEFAULT_BRANCH}
       event: push
 
   publish-agent:
@@ -149,7 +66,7 @@ pipeline:
     # TODO: only release as next after 0.15.0 got released
     tag: [next, latest]
     when:
-      branch: master
+      branch: ${CI_REPO_DEFAULT_BRANCH}
       event: push
 
   publish-agent-alpine:
@@ -161,7 +78,7 @@ pipeline:
     # TODO: only release as next-alpine after 0.15.0 got released
     tag: [next-alpine, latest-alpine]
     when:
-      branch: master
+      branch: ${CI_REPO_DEFAULT_BRANCH}
       event: push
 
   publish-cli:
@@ -172,7 +89,7 @@ pipeline:
     secrets: [docker_username, docker_password]
     tag: [next]
     when:
-      branch: master
+      branch: ${CI_REPO_DEFAULT_BRANCH}
       event: push
 
   publish-cli-alpine:
@@ -183,7 +100,7 @@ pipeline:
     secrets: [ docker_username, docker_password ]
     tag: [next-alpine]
     when:
-      branch: master
+      branch: ${CI_REPO_DEFAULT_BRANCH}
       event: push
 
   release-server:
@@ -253,7 +170,7 @@ pipeline:
     when:
       event: tag
 
-  # TODO: upload build artifacts for pushes to master
+  # TODO: upload build artifacts for pushes to ${CI_REPO_DEFAULT_BRANCH}
 
   release:
     image: plugins/github-release
@@ -269,16 +186,6 @@ pipeline:
     when:
       event: tag
 
-services:
-  postgres:
-    image: postgres:9.6
-    ports: ["5432"]
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_HOST_AUTH_METHOD=trust
-  mysql:
-    image: mysql:5.6.27
-    ports: ["3306"]
-    environment:
-      - MYSQL_DATABASE=test
-      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
+depends_on:
+  - test
+  - web

.woodpecker/test.yml

@@ -0,0 +1,76 @@
+pipeline:
+  dummy-web:
+    image: golang:1.16
+    commands:
+      - mkdir -p web/dist/
+      - echo "test" > web/dist/index.html
+
+  lint:
+    image: golang:1.16
+    group: test
+    commands:
+      - make lint
+
+  agent:
+    image: golang:1.16
+    group: test
+    commands:
+      - make test-agent
+
+  server:
+    image: golang:1.16
+    group: test
+    commands:
+      - make test-server
+
+  cli:
+    image: golang:1.16
+    group: test
+    commands:
+      - make test-cli
+
+  lib:
+    image: golang:1.16
+    group: test
+    commands:
+      - make test-lib
+
+  sqlite:
+    image: golang:1.16
+    group: test
+    environment:
+      - DATABASE_DRIVER=sqlite3
+    commands:
+      - make test-server-datastore
+
+  postgres:
+    image: golang:1.16
+    group: test
+    environment:
+      - DATABASE_DRIVER=postgres
+      - DATABASE_CONFIG=host=postgres user=postgres dbname=postgres sslmode=disable
+    commands:
+      - make test-server-datastore
+
+  mysql:
+    image: golang:1.16
+    group: test
+    environment:
+      - DATABASE_DRIVER=mysql
+      - DATABASE_CONFIG=root@tcp(mysql:3306)/test?parseTime=true
+    commands:
+      - make test-server-datastore
+
+services:
+  postgres:
+    image: postgres:9.6
+    ports: ["5432"]
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_HOST_AUTH_METHOD=trust
+  mysql:
+    image: mysql:5.6.27
+    ports: ["3306"]
+    environment:
+      - MYSQL_DATABASE=test
+      - MYSQL_ALLOW_EMPTY_PASSWORD=yes

.woodpecker/web.yml

@@ -0,0 +1,45 @@
+pipeline:
+  deps:
+    image: node:16-alpine
+    commands:
+      - cd web/
+      - yarn install --frozen-lockfile
+    when:
+      path: "web/**"
+
+  lint:
+    group: test
+    image: node:16-alpine
+    commands:
+      - cd web/
+      - yarn lint
+    when:
+      path: "web/**"
+
+  formatcheck:
+    group: test
+    image: node:16-alpine
+    commands:
+      - cd web/
+      - yarn formatcheck
+    when:
+      path: "web/**"
+
+  typecheck:
+    group: test
+    image: node:16-alpine
+    commands:
+      - cd web/
+      - yarn typecheck
+    when:
+      path: "web/**"
+
+  test:
+    group: test
+    image: node:16-alpine
+    commands:
+      - cd web/
+      - yarn test
+    when:
+      path: "web/**"
+

Makefile

@@ -45,11 +45,22 @@ lint:
 frontend-dependencies:
 	(cd web/; yarn install --frozen-lockfile)
 
+lint-frontend:
+	(cd web/; yarn)
+	(cd web/; yarn lesshint)
+	(cd web/; yarn lint --quiet)
+
 test-agent:
-	$(DOCKER_RUN) go test -race -timeout 30s github.com/woodpecker-ci/woodpecker/cmd/agent $(GO_PACKAGES)
+	$(DOCKER_RUN) go test -race -timeout 30s github.com/woodpecker-ci/woodpecker/cmd/agent github.com/woodpecker-ci/woodpecker/agent/...
 
 test-server:
-	$(DOCKER_RUN) go test -race -timeout 30s github.com/woodpecker-ci/woodpecker/cmd/server
+	$(DOCKER_RUN) go test -race -timeout 30s github.com/woodpecker-ci/woodpecker/cmd/server $(shell go list github.com/woodpecker-ci/woodpecker/server/... | grep -v '/store')
+
+test-server-datastore:
+	$(DOCKER_RUN) go test -timeout 30s github.com/woodpecker-ci/woodpecker/server/store/...
+
+test-cli:
+	$(DOCKER_RUN) go test -race -timeout 30s github.com/woodpecker-ci/woodpecker/cmd/cli github.com/woodpecker-ci/woodpecker/cli/...
 
 test-frontend: frontend-dependencies
 	(cd web/; yarn run lint)
@@ -58,9 +69,9 @@ test-frontend: frontend-dependencies
 	(cd web/; yarn run test)
 
 test-lib:
-	$(DOCKER_RUN) go test -race -timeout 30s $(shell go list ./... | grep -v '/cmd/')
+	$(DOCKER_RUN) go test -race -timeout 30s $(shell go list ./... | grep -v '/cmd\|/agent\|/cli\|/server')
 
-test: test-lib test-agent test-server
+test: formatcheck vet test-agent test-server test-server-datastore test-cli test-frontend test-lib
 
 build-frontend:
 	(cd web/; yarn install --frozen-lockfile; yarn build)