From bed69f9f72c55acfb8626d8b5b78bc51cdf79d63 Mon Sep 17 00:00:00 2001 From: Anbraten Date: Tue, 21 Dec 2021 10:52:10 +0100 Subject: [PATCH] Improve dev docs (#636) --- .gitignore | 1 + .vscode/settings.json | 13 ++ docs/docs/92-development.md | 221 ------------------ .../docs/92-development/01-getting-started.md | 127 ++++++++++ docs/docs/92-development/03-ui.md | 31 +++ docs/docs/92-development/04-docs.md | 20 ++ docs/docs/92-development/05-architecture.md | 9 + docs/docs/92-development/06-guides.md | 36 +++ docs/docs/92-development/_category_.yml | 4 + docs/docs/92-development/ui-proxy.svg | 16 ++ web/vite.config.ts | 18 ++ 11 files changed, 275 insertions(+), 221 deletions(-) create mode 100644 .vscode/settings.json delete mode 100644 docs/docs/92-development.md create mode 100644 docs/docs/92-development/01-getting-started.md create mode 100644 docs/docs/92-development/03-ui.md create mode 100644 docs/docs/92-development/04-docs.md create mode 100644 docs/docs/92-development/05-architecture.md create mode 100644 docs/docs/92-development/06-guides.md create mode 100644 docs/docs/92-development/_category_.yml create mode 100644 docs/docs/92-development/ui-proxy.svg diff --git a/.gitignore b/.gitignore index 15838fece..a26144e2c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ ### IDEs ### .idea/ .vscode/* +!.vscode/settings.json !.vscode/launch.json !.vscode/extensions.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..c0622716b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,13 @@ +{ + "git.ignoreLimitWarning": true, + "search.exclude": { + "**/node_modules": true, + "**/bower_components": true, + "**/*.code-search": true, + "vendor/": true + }, + "go.lintTool":"golangci-lint", + "go.lintFlags": [ + "--fast" + ] +} diff --git a/docs/docs/92-development.md b/docs/docs/92-development.md deleted file mode 100644 index 4a2342876..000000000 --- a/docs/docs/92-development.md +++ /dev/null @@ -1,221 +0,0 @@ -# Development - -## Preparation - -### Install Tools - -#### Go - -Install Golang (>=1.16) as described by [this guide](https://go.dev/doc/install). - -#### make - -> GNU Make is a tool which controls the generation of executables and other non-source files of a program from the program's source files. (https://www.gnu.org/software/make/) - -Install make on: - - Ubuntu: `apt install make` - [Docs](https://wiki.ubuntuusers.de/Makefile/) - - [Windows](https://stackoverflow.com/a/32127632/8461267) - - Mac OS: `brew install make` - -#### Node.js & Yarn - -Install [Node.js (>=14)](https://nodejs.org/en/download/) if you want to build Woodpeckers UI or documentation. - -For dependencies installation (node_modules) for the UI and documentation of Woodpecker the package-manager Yarn is used. The installation of Yarn is described by [this guide](https://yarnpkg.com/getting-started/install). - - -### Create a `.env` file with your development configuration - -Similar to the environment variables you can set for your production setup of Woodpecker, you can create a `.env` in the root of the Woodpecker project and add any need config to it. - -A common config for debugging would look like this: - -```ini -WOODPECKER_OPEN=true -WOODPECKER_ADMIN=your-username - -# if you want to test webhooks with an online SCM like Github this address needs to be accessible from public server -WOODPECKER_HOST=http://your-dev-address.com/ - -# github (sample for a SCM config - see /docs/administration/vcs/overview for other SCMs) -WOODPECKER_GITHUB=true -WOODPECKER_GITHUB_CLIENT= -WOODPECKER_GITHUB_SECRET= - -# agent -WOODPECKER_SERVER=localhost:9000 -WOODPECKER_SECRET=a-long-and-secure-password-used-for-the-local-development-system -WOODPECKER_MAX_PROCS=1 - -# enable if you want to develop the UI -# WOODPECKER_DEV_WWW_PROXY=http://localhost:3000 - -# used so you can login without using a public address -WOODPECKER_DEV_OAUTH_HOST=http://localhost:8000 - -# disable health-checks while debugging (normally not needed while developing) -WOODPECKER_HEALTHCHECK=false - -# WOODPECKER_LOG_LEVEL=debug -# WOODPECKER_LOG_LEVEL=trace -``` - -### O-Auth - -Create an O-Auth app for your SCM as describe in the [SCM documentation](/docs/administration/vcs/overview). If you set `WOODPECKER_DEV_OAUTH_HOST=http://localhost:8000` you can use that address with the path as explained for the specific SCM to login without the need for a public address. For example for Github you would use `http://localhost:8000/authorize` as authorization callback URL. - -## Developing with VS-Code - -You can use different methods for debugging the Woodpecker applications. One of the currently recommend ways to debug and test the Woodpecker application is using [VS-Code](https://code.visualstudio.com/) or [VS-Codium](https://vscodium.com/) (Open-Source binaries of VS-Code) as most maintainers are using it and Woodpecker already includes the needed debug configurations for it. - -As a starting guide for programming Go with VS-Code you can use this video guide: -[![Getting started with Go in VS-Code](https://img.youtube.com/vi/1MXIGYrMk80/0.jpg)](https://www.youtube.com/watch?v=1MXIGYrMk80) - -### Debugging Woodpecker - -The Woodpecker source code already includes launch configurations for the Woodpecker server and agent. To start debugging you can click on the debug icon in the navigation bar of VS-Code (ctrl-shift-d). On that page you will see the existing launch jobs at the top. Simply select the agent or server and click on the play button. You can set breakpoints in the source files to stop at specific points. - -![Woodpecker debugging with VS-Code](92-development/vscode-debug.png) - -## UI development - -To develop the UI you need to install [Node.js and Yarn](#nodejs--yarn). In addition it is recommended to use VS-Code with the recommended plugin selection to get features like auto-formatting, linting and typechecking. The UI is written with [Vue 3](https://v3.vuejs.org/) as Single-Page-Application accessing the Woodpecker REST api. - -The UI code is placed in `web/`. Change to that folder in your terminal with `cd web/` and install all dependencies by running `yarn install`. Start the UI locally with [hot-reloading](https://stackoverflow.com/a/41429055/8461267) by running: `yarn start`. To access the UI you now have to start the Woodpecker server. -For this you have to add the line `WOODPECKER_DEV_WWW_PROXY=http://localhost:3000` to your `.env` config and start the server after that as explained in the [debugging](#debugging) section. - -The UI will now be served under [http://localhost:8000](http://localhost:8000) (don't access the UI from port 3000 as that only show the UI without access to the actual api). - -### Tools and frameworks - -The following list contains some tools and frameworks used by the Woodpecker UI. For some points we added some guidelines / hints to help you developing. - -- [Vue 3](https://v3.vuejs.org/) - - use `setup` and composition api - - place (re-usable) components in `web/src/components/` - - views should have a route in `web/src/router.ts` and are located in `web/src/views/` -- [Windicss](https://windicss.org/) (similar to Tailwind) - - use Windicss classes where possible - - if needed extend the Windicss config to use new classes -- [Vite](https://vitejs.dev/) (similar to Webpack) -- [Typescript](https://www.typescriptlang.org/) - - avoid using `any` and `unknown` (the linter will prevent you from doing so anyways :wink:) -- [eslint](https://eslint.org/) -- [Volar & vue-tsc](https://github.com/johnsoncodehk/volar/) for type-checking in .vue file - - use the take-over mode of Volar as described by [this guide](https://github.com/johnsoncodehk/volar/discussions/471) - -## Documentation development - -The documentation is using docusaurus as framework. You can learn more about it from its [official documentation](https://docusaurus.io/docs/). - -If you only want to change some text it probably is enough if you just search for the corresponding [Markdown](https://www.markdownguide.org/basic-syntax/) file inside the `docs/docs/` folder and adjust it. If you want to change larger parts and test the rendered documentation you can run docusaurus locally. Similarly to the UI you need to install [Node.js and Yarn](#nodejs--yarn). After that you can run and build docusaurus locally by using the following commands: - -```bash -cd docs/ - -yarn install - -# build plugins used by the docs -yarn build:woodpecker-plugins - -# start docs with hot-reloading, so you can change the docs and directly see the changes in the browser without reloading it manually -yarn start - -# or build the docs to deploy it to some static page hosting -yarn build -``` - -## Testing & linting code - -To test or lint parts of Woodpecker you can run one of the following commands: - -```bash -# test server code -make test-server - -# test agent code -make test-agent - -# test cli code -make test-cli - -# test datastore / database related code like migrations of the server -make test-server-datastore - -# lint go code -make lint - -# lint UI code -make lint-frontend - -# test UI code -make test-frontend -``` - -If you want to test a specific go file you can also use: - -```bash -go test -race -timeout 30s github.com/woodpecker-ci/woodpecker/ -``` - -Or you can open the test-file inside [VS-Code](#developing-with-vs-code) and run or debug the test by clicking on the inline commands: - -![Run test via VS-Code](92-development/vscode-run-test.png) - -## Package architecture of Woodpecker - -![Woodpecker architecture](92-development/woodpecker-architecture.png) - -## Run applications from terminal - -If you want to run a Woodpecker applications from your terminal you can use one of the following commands from the base of the Woodpecker project. They will execute Woodpecker in a similar way as described in [debugging Woodpecker](#debugging-woodpecker) without the ability to really debug it in your editor. - -```bash -# start server -$ go run ./cmd/server - -# start agent -$ go run ./cmd/agent - -# execute cli command -$ go run ./cmd/cli [command] -``` - -## Add new migration - -Woodpecker uses migrations to change the database schema if a database model has been changed. - -:::info -If you add a new property to a model, you dont have to do anything, -the column will be created based on the xorm tag automaticaly. -If you add a new model, you only have to add it to the `syncAll()` function at -`server/store/datastore/migration/migration.go` to get a table created. -::: - -If a developer for example removes a property `Counter` from the model `Repo` in `server/model/` they would need to add a new migration task like the following example to a file like `server/store/datastore/migration/004_repos_drop_repo_counter.go`: - -```go -package migration - -import ( - "xorm.io/xorm" -) - -var alterTableReposDropCounter = task{ - name: "alter-table-drop-counter", - fn: func(sess *xorm.Session) error { - return dropTableColumns(sess, "repos", "repo_counter") - }, -} -``` - -:::warning -Do not `sess.Begin()`, `sess.Commit()` or `sess.Close()` the session, it is managed from outside. -::: - -To automatically execute the migration after the start of the server, the new migration needs to be added to the end of `migrationTasks` in `server/store/datastore/migration/migration.go`. - -:::tip -Woodpecker uses [Xorm](https://xorm.io/) as ORM for the database connection. -If you can find its documentation at [gobook.io/read/gitea.com/xorm](https://gobook.io/read/gitea.com/xorm/manual-en-US). -::: diff --git a/docs/docs/92-development/01-getting-started.md b/docs/docs/92-development/01-getting-started.md new file mode 100644 index 000000000..d81dd9c4c --- /dev/null +++ b/docs/docs/92-development/01-getting-started.md @@ -0,0 +1,127 @@ +# Getting started + +## Preparation + +### Install Go + +Install Golang (>=1.16) as described by [this guide](https://go.dev/doc/install). + +### Install make + +> GNU Make is a tool which controls the generation of executables and other non-source files of a program from the program's source files. (https://www.gnu.org/software/make/) + +Install make on: + - Ubuntu: `apt install make` - [Docs](https://wiki.ubuntuusers.de/Makefile/) + - [Windows](https://stackoverflow.com/a/32127632/8461267) + - Mac OS: `brew install make` + +### Install Node.js & Yarn + +Install [Node.js (>=14)](https://nodejs.org/en/download/) if you want to build Woodpeckers UI or documentation. + +For dependencies installation (node_modules) for the UI and documentation of Woodpecker the package-manager Yarn is used. The installation of Yarn is described by [this guide](https://yarnpkg.com/getting-started/install). + +### Create a `.env` file with your development configuration + +Similar to the environment variables you can set for your production setup of Woodpecker, you can create a `.env` in the root of the Woodpecker project and add any need config to it. + +A common config for debugging would look like this: + +```ini +WOODPECKER_OPEN=true +WOODPECKER_ADMIN=your-username + +# if you want to test webhooks with an online SCM like Github this address needs to be accessible from public server +WOODPECKER_HOST=http://your-dev-address.com/ + +# github (sample for a SCM config - see /docs/administration/vcs/overview for other SCMs) +WOODPECKER_GITHUB=true +WOODPECKER_GITHUB_CLIENT= +WOODPECKER_GITHUB_SECRET= + +# agent +WOODPECKER_SERVER=localhost:9000 +WOODPECKER_SECRET=a-long-and-secure-password-used-for-the-local-development-system +WOODPECKER_MAX_PROCS=1 + +# enable if you want to develop the UI +# WOODPECKER_DEV_WWW_PROXY=http://localhost:3000 + +# used so you can login without using a public address +WOODPECKER_DEV_OAUTH_HOST=http://localhost:8000 + +# disable health-checks while debugging (normally not needed while developing) +WOODPECKER_HEALTHCHECK=false + +# WOODPECKER_LOG_LEVEL=debug +# WOODPECKER_LOG_LEVEL=trace +``` + +### Setup O-Auth + +Create an O-Auth app for your SCM as describe in the [SCM documentation](/docs/administration/vcs/overview). If you set `WOODPECKER_DEV_OAUTH_HOST=http://localhost:8000` you can use that address with the path as explained for the specific SCM to login without the need for a public address. For example for Github you would use `http://localhost:8000/authorize` as authorization callback URL. + +## Developing with VS-Code + +You can use different methods for debugging the Woodpecker applications. One of the currently recommend ways to debug and test the Woodpecker application is using [VS-Code](https://code.visualstudio.com/) or [VS-Codium](https://vscodium.com/) (Open-Source binaries of VS-Code) as most maintainers are using it and Woodpecker already includes the needed debug configurations for it. + +As a starting guide for programming Go with VS-Code you can use this video guide: +[![Getting started with Go in VS-Code](https://img.youtube.com/vi/1MXIGYrMk80/0.jpg)](https://www.youtube.com/watch?v=1MXIGYrMk80) + +### Debugging Woodpecker + +The Woodpecker source code already includes launch configurations for the Woodpecker server and agent. To start debugging you can click on the debug icon in the navigation bar of VS-Code (ctrl-shift-d). On that page you will see the existing launch jobs at the top. Simply select the agent or server and click on the play button. You can set breakpoints in the source files to stop at specific points. + +![Woodpecker debugging with VS-Code](./vscode-debug.png) + +## Testing & linting code + +To test or lint parts of Woodpecker you can run one of the following commands: + +```bash +# test server code +make test-server + +# test agent code +make test-agent + +# test cli code +make test-cli + +# test datastore / database related code like migrations of the server +make test-server-datastore + +# lint go code +make lint + +# lint UI code +make lint-frontend + +# test UI code +make test-frontend +``` + +If you want to test a specific go file you can also use: + +```bash +go test -race -timeout 30s github.com/woodpecker-ci/woodpecker/ +``` + +Or you can open the test-file inside [VS-Code](#developing-with-vs-code) and run or debug the test by clicking on the inline commands: + +![Run test via VS-Code](./vscode-run-test.png) + +## Run applications from terminal + +If you want to run a Woodpecker applications from your terminal you can use one of the following commands from the base of the Woodpecker project. They will execute Woodpecker in a similar way as described in [debugging Woodpecker](#debugging-woodpecker) without the ability to really debug it in your editor. + +```bash +# start server +$ go run ./cmd/server + +# start agent +$ go run ./cmd/agent + +# execute cli command +$ go run ./cmd/cli [command] +``` diff --git a/docs/docs/92-development/03-ui.md b/docs/docs/92-development/03-ui.md new file mode 100644 index 000000000..27eac4aa8 --- /dev/null +++ b/docs/docs/92-development/03-ui.md @@ -0,0 +1,31 @@ +# UI Development + +To develop the UI you need to install [Node.js and Yarn](/docs/development/getting-started#nodejs--yarn). In addition it is recommended to use VS-Code with the recommended plugin selection to get features like auto-formatting, linting and typechecking. The UI is written with [Vue 3](https://v3.vuejs.org/) as Single-Page-Application accessing the Woodpecker REST api. + +## Setup +The UI code is placed in `web/`. Change to that folder in your terminal with `cd web/` and install all dependencies by running `yarn install`. For production builds the generated UI code is integrated into the Woodpecker server by using [go-embed](https://pkg.go.dev/embed). + +Testing UI changes would require us to rebuild the UI after each adjustment to the code by running `yarn build` and restarting the Woodpecker server. To avoid this you can make use of the dev-proxy integrated into the Woodpecker server. This integrated dev-proxy will forward all none api request to a separate http-server which will only serve the UI files. + +![UI Proxy architecture](./ui-proxy.svg) + +Start the UI server locally with [hot-reloading](https://stackoverflow.com/a/41429055/8461267) by running: `yarn start`. To enable the forwarding of requests to the UI server you have to enable the dev-proxy inside the Woodpecker server by adding `WOODPECKER_DEV_WWW_PROXY=http://localhost:8010` to your `.env` file. +After starting the Woodpecker server as explained in the [debugging](/docs/development/getting-started#debugging) section, you should now be able to access the UI under [http://localhost:8000](http://localhost:8000). + +## Tools and frameworks + +The following list contains some tools and frameworks used by the Woodpecker UI. For some points we added some guidelines / hints to help you developing. + +- [Vue 3](https://v3.vuejs.org/) + - use `setup` and composition api + - place (re-usable) components in `web/src/components/` + - views should have a route in `web/src/router.ts` and are located in `web/src/views/` +- [Windicss](https://windicss.org/) (similar to Tailwind) + - use Windicss classes where possible + - if needed extend the Windicss config to use new classes +- [Vite](https://vitejs.dev/) (similar to Webpack) +- [Typescript](https://www.typescriptlang.org/) + - avoid using `any` and `unknown` (the linter will prevent you from doing so anyways :wink:) +- [eslint](https://eslint.org/) +- [Volar & vue-tsc](https://github.com/johnsoncodehk/volar/) for type-checking in .vue file + - use the take-over mode of Volar as described by [this guide](https://github.com/johnsoncodehk/volar/discussions/471) diff --git a/docs/docs/92-development/04-docs.md b/docs/docs/92-development/04-docs.md new file mode 100644 index 000000000..b8fba9b08 --- /dev/null +++ b/docs/docs/92-development/04-docs.md @@ -0,0 +1,20 @@ +# Documentation + +The documentation is using docusaurus as framework. You can learn more about it from its [official documentation](https://docusaurus.io/docs/). + +If you only want to change some text it probably is enough if you just search for the corresponding [Markdown](https://www.markdownguide.org/basic-syntax/) file inside the `docs/docs/` folder and adjust it. If you want to change larger parts and test the rendered documentation you can run docusaurus locally. Similarly to the UI you need to install [Node.js and Yarn](/docs/development/getting-started#nodejs--yarn). After that you can run and build docusaurus locally by using the following commands: + +```bash +cd docs/ + +yarn install + +# build plugins used by the docs +yarn build:woodpecker-plugins + +# start docs with hot-reloading, so you can change the docs and directly see the changes in the browser without reloading it manually +yarn start + +# or build the docs to deploy it to some static page hosting +yarn build +``` diff --git a/docs/docs/92-development/05-architecture.md b/docs/docs/92-development/05-architecture.md new file mode 100644 index 000000000..82fa15ede --- /dev/null +++ b/docs/docs/92-development/05-architecture.md @@ -0,0 +1,9 @@ +# Architecture + +## Package architecture + +![Woodpecker architecture](./woodpecker-architecture.png) + +## System architecture + +TODO diff --git a/docs/docs/92-development/06-guides.md b/docs/docs/92-development/06-guides.md new file mode 100644 index 000000000..7970ba344 --- /dev/null +++ b/docs/docs/92-development/06-guides.md @@ -0,0 +1,36 @@ +# Guides + +## ORM + +Woodpecker uses [Xorm](https://xorm.io/) as ORM for the database connection. +You can find its documentation at [gobook.io/read/gitea.com/xorm](https://gobook.io/read/gitea.com/xorm/manual-en-US/). + +## Add a new migration + +Woodpecker uses migrations to change the database schema if a database model has been changed. If for example a developer removes a property `Counter` from the model `Repo` in `server/model/` they would need to add a new migration task like the following example to a file like `server/store/datastore/migration/004_repos_drop_repo_counter.go`: + +```go +package migration + +import ( + "xorm.io/xorm" +) + +var alterTableReposDropCounter = task{ + name: "alter-table-drop-counter", + fn: func(sess *xorm.Session) error { + return dropTableColumns(sess, "repos", "repo_counter") + }, +} +``` + +:::info +Adding new properties to models will be handled automatically by the underlying [ORM](#orm) based on the [struct field tags](https://stackoverflow.com/questions/10858787/what-are-the-uses-for-tags-in-go) of the model. If you add a completely new model, you have to add it to the `syncAll()` function at `server/store/datastore/migration/migration.go` to get a new table created. +::: + +:::warning +You should not use `sess.Begin()`, `sess.Commit()` or `sess.Close()` inside a migration. Session / transaction handling will be done by the underlying migration manager. +::: + +To automatically execute the migration after the start of the server, the new migration needs to be added to the end of `migrationTasks` in `server/store/datastore/migration/migration.go`. After a successful execution of that transaction the server will automatically add the migration to a list, so it wont be executed again on the next start. + diff --git a/docs/docs/92-development/_category_.yml b/docs/docs/92-development/_category_.yml new file mode 100644 index 000000000..02d70e09b --- /dev/null +++ b/docs/docs/92-development/_category_.yml @@ -0,0 +1,4 @@ +label: 'Development' +# position: 3 +collapsible: true +collapsed: true diff --git a/docs/docs/92-development/ui-proxy.svg b/docs/docs/92-development/ui-proxy.svg new file mode 100644 index 000000000..6ccb74458 --- /dev/null +++ b/docs/docs/92-development/ui-proxy.svg @@ -0,0 +1,16 @@ + + + + + + + forward requestin dev modeUI Server Port 8010Woodpecker ServerBrowserPort 8000/login/logout/api/.../api/web-config.js/streamEvery other request(404 Handler) \ No newline at end of file diff --git a/web/vite.config.ts b/web/vite.config.ts index 9179bcdb3..1eeb11d03 100644 --- a/web/vite.config.ts +++ b/web/vite.config.ts @@ -8,6 +8,19 @@ import { defineConfig } from 'vite'; import WindiCSS from 'vite-plugin-windicss'; import svgLoader from 'vite-svg-loader'; +function woodpeckerInfoPlugin() { + return { + name: 'woodpecker-info', + configureServer() { + const info = + 'Please add `WOODPECKER_DEV_WWW_PROXY=http://localhost:8010` to your `.env` file.\n' + + 'After starting the woodpecker server as well you should now be able to access the UI at http://localhost:8000/'; + // eslint-disable-next-line no-console + console.log(info); + }, + }; +} + // https://vitejs.dev/config/ export default defineConfig({ plugins: [ @@ -18,10 +31,15 @@ export default defineConfig({ Components({ resolvers: IconsResolver(), }), + woodpeckerInfoPlugin(), ], resolve: { alias: { '~/': `${path.resolve(__dirname, 'src')}/`, }, }, + logLevel: 'warn', + server: { + port: 8010, + }, });