From 936c9bdb0da3940f05de39451c497b9f6b1790b2 Mon Sep 17 00:00:00 2001 From: 6543 Date: Mon, 18 Dec 2023 22:37:38 +0100 Subject: [PATCH] Export changed files via builtin environment variables (#2935) add **`CI_PIPELINE_FILES`** to builtin env vars close #853 --- *Sponsored by Kithara Software GmbH* --- docs/docs/20-usage/50-environment.md | 175 +++++++++--------- .../version-2.0/20-usage/50-environment.md | 2 +- pipeline/frontend/metadata/environment.go | 15 +- pipeline/frontend/metadata_test.go | 17 +- 4 files changed, 113 insertions(+), 96 deletions(-) diff --git a/docs/docs/20-usage/50-environment.md b/docs/docs/20-usage/50-environment.md index 306b0a651..8deebc715 100644 --- a/docs/docs/20-usage/50-environment.md +++ b/docs/docs/20-usage/50-environment.md @@ -46,93 +46,94 @@ steps: This is the reference list of all environment variables available to your pipeline containers. These are injected into your pipeline step and plugins containers, at runtime. -| NAME | Description | -| -------------------------------- | -------------------------------------------------------------------------------------------- | -| `CI` | CI environment name (value: `woodpecker`) | -| | **Repository** | -| `CI_REPO` | repository full name `/` | -| `CI_REPO_OWNER` | repository owner | -| `CI_REPO_NAME` | repository name | -| `CI_REPO_REMOTE_ID` | repository remote ID, is the UID it has in the forge | -| `CI_REPO_SCM` | repository SCM (git) | -| `CI_REPO_URL` | repository web URL | -| `CI_REPO_CLONE_URL` | repository clone URL | -| `CI_REPO_CLONE_SSH_URL` | repository SSH clone URL | -| `CI_REPO_DEFAULT_BRANCH` | repository default branch (main) | -| `CI_REPO_PRIVATE` | repository is private | -| `CI_REPO_TRUSTED` | repository is trusted | -| | **Current Commit** | -| `CI_COMMIT_SHA` | commit SHA | -| `CI_COMMIT_REF` | commit ref | -| `CI_COMMIT_REFSPEC` | commit ref spec | -| `CI_COMMIT_BRANCH` | commit branch (equals target branch for pull requests) | -| `CI_COMMIT_SOURCE_BRANCH` | commit source branch (empty if event is not `pull_request`) | -| `CI_COMMIT_TARGET_BRANCH` | commit target branch (empty if event is not `pull_request`) | -| `CI_COMMIT_TAG` | commit tag name (empty if event is not `tag`) | -| `CI_COMMIT_PULL_REQUEST` | commit pull request number (empty if event is not `pull_request`) | -| `CI_COMMIT_PULL_REQUEST_LABELS` | labels assigned to pull request (empty if event is not `pull_request`) | -| `CI_COMMIT_MESSAGE` | commit message | -| `CI_COMMIT_AUTHOR` | commit author username | -| `CI_COMMIT_AUTHOR_EMAIL` | commit author email address | -| `CI_COMMIT_AUTHOR_AVATAR` | commit author avatar | -| | **Current pipeline** | -| `CI_PIPELINE_NUMBER` | pipeline number | -| `CI_PIPELINE_PARENT` | number of parent pipeline | -| `CI_PIPELINE_EVENT` | pipeline event (push, pull_request, tag, deployment) | -| `CI_PIPELINE_URL` | link to the web UI for the pipeline | -| `CI_PIPELINE_FORGE_URL` | link to the forge's web UI for the commit(s) or tag that triggered the pipeline | -| `CI_PIPELINE_DEPLOY_TARGET` | pipeline deploy target for `deployment` events (ie production) | -| `CI_PIPELINE_STATUS` | pipeline status (success, failure) | -| `CI_PIPELINE_CREATED` | pipeline created UNIX timestamp | -| `CI_PIPELINE_STARTED` | pipeline started UNIX timestamp | -| `CI_PIPELINE_FINISHED` | pipeline finished UNIX timestamp | -| | **Current workflow** | -| `CI_WORKFLOW_NAME` | workflow name | -| | **Current step** | -| `CI_STEP_NAME` | step name | -| `CI_STEP_NUMBER` | step number | -| `CI_STEP_STATUS` | step status (success, failure) | -| `CI_STEP_STARTED` | step started UNIX timestamp | -| `CI_STEP_FINISHED` | step finished UNIX timestamp | -| `CI_STEP_URL` | URL to step in UI | -| | **Previous commit** | -| `CI_PREV_COMMIT_SHA` | previous commit SHA | -| `CI_PREV_COMMIT_REF` | previous commit ref | -| `CI_PREV_COMMIT_REFSPEC` | previous commit ref spec | -| `CI_PREV_COMMIT_BRANCH` | previous commit branch | -| `CI_PREV_COMMIT_SOURCE_BRANCH` | previous commit source branch | -| `CI_PREV_COMMIT_TARGET_BRANCH` | previous commit target branch | -| `CI_PREV_COMMIT_URL` | previous commit link in forge | -| `CI_PREV_COMMIT_MESSAGE` | previous commit message | -| `CI_PREV_COMMIT_AUTHOR` | previous commit author username | -| `CI_PREV_COMMIT_AUTHOR_EMAIL` | previous commit author email address | -| `CI_PREV_COMMIT_AUTHOR_AVATAR` | previous commit author avatar | -| | **Previous pipeline** | -| `CI_PREV_PIPELINE_NUMBER` | previous pipeline number | -| `CI_PREV_PIPELINE_PARENT` | previous pipeline number of parent pipeline | -| `CI_PREV_PIPELINE_EVENT` | previous pipeline event (push, pull_request, tag, deployment) | -| `CI_PREV_PIPELINE_URL` | previous pipeline link in CI | -| `CI_PREV_PIPELINE_FORGE_URL` | previous pipeline link to event in forge | -| `CI_PREV_PIPELINE_DEPLOY_TARGET` | previous pipeline deploy target for `deployment` events (ie production) | -| `CI_PREV_PIPELINE_STATUS` | previous pipeline status (success, failure) | -| `CI_PREV_PIPELINE_CREATED` | previous pipeline created UNIX timestamp | -| `CI_PREV_PIPELINE_STARTED` | previous pipeline started UNIX timestamp | -| `CI_PREV_PIPELINE_FINISHED` | previous pipeline finished UNIX timestamp | -| |   | -| `CI_WORKSPACE` | Path of the workspace where source code gets cloned to | -| | **System** | -| `CI_SYSTEM_NAME` | name of the CI system: `woodpecker` | -| `CI_SYSTEM_URL` | link to CI system | -| `CI_SYSTEM_HOST` | hostname of CI server | -| `CI_SYSTEM_VERSION` | version of the server | -| | **Forge** | -| `CI_FORGE_TYPE` | name of forge (gitea, github, ...) | -| `CI_FORGE_URL` | root URL of configured forge | -| | **Internal** - Please don't use! | -| `CI_SCRIPT` | Internal script path. Used to call pipeline step commands. | -| `CI_NETRC_USERNAME` | Credentials for private repos to be able to clone data. (Only available for specific images) | -| `CI_NETRC_PASSWORD` | Credentials for private repos to be able to clone data. (Only available for specific images) | -| `CI_NETRC_MACHINE` | Credentials for private repos to be able to clone data. (Only available for specific images) | +| NAME | Description | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| `CI` | CI environment name (value: `woodpecker`) | +| | **Repository** | +| `CI_REPO` | repository full name `/` | +| `CI_REPO_OWNER` | repository owner | +| `CI_REPO_NAME` | repository name | +| `CI_REPO_REMOTE_ID` | repository remote ID, is the UID it has in the forge | +| `CI_REPO_SCM` | repository SCM (git) | +| `CI_REPO_URL` | repository web URL | +| `CI_REPO_CLONE_URL` | repository clone URL | +| `CI_REPO_CLONE_SSH_URL` | repository SSH clone URL | +| `CI_REPO_DEFAULT_BRANCH` | repository default branch (main) | +| `CI_REPO_PRIVATE` | repository is private | +| `CI_REPO_TRUSTED` | repository is trusted | +| | **Current Commit** | +| `CI_COMMIT_SHA` | commit SHA | +| `CI_COMMIT_REF` | commit ref | +| `CI_COMMIT_REFSPEC` | commit ref spec | +| `CI_COMMIT_BRANCH` | commit branch (equals target branch for pull requests) | +| `CI_COMMIT_SOURCE_BRANCH` | commit source branch (empty if event is not `pull_request`) | +| `CI_COMMIT_TARGET_BRANCH` | commit target branch (empty if event is not `pull_request`) | +| `CI_COMMIT_TAG` | commit tag name (empty if event is not `tag`) | +| `CI_COMMIT_PULL_REQUEST` | commit pull request number (empty if event is not `pull_request`) | +| `CI_COMMIT_PULL_REQUEST_LABELS` | labels assigned to pull request (empty if event is not `pull_request`) | +| `CI_COMMIT_MESSAGE` | commit message | +| `CI_COMMIT_AUTHOR` | commit author username | +| `CI_COMMIT_AUTHOR_EMAIL` | commit author email address | +| `CI_COMMIT_AUTHOR_AVATAR` | commit author avatar | +| | **Current pipeline** | +| `CI_PIPELINE_NUMBER` | pipeline number | +| `CI_PIPELINE_PARENT` | number of parent pipeline | +| `CI_PIPELINE_EVENT` | pipeline event (push, pull_request, tag, deployment, cron, manual) | +| `CI_PIPELINE_URL` | link to the web UI for the pipeline | +| `CI_PIPELINE_FORGE_URL` | link to the forge's web UI for the commit(s) or tag that triggered the pipeline | +| `CI_PIPELINE_DEPLOY_TARGET` | pipeline deploy target for `deployment` events (ie production) | +| `CI_PIPELINE_STATUS` | pipeline status (success, failure) | +| `CI_PIPELINE_CREATED` | pipeline created UNIX timestamp | +| `CI_PIPELINE_STARTED` | pipeline started UNIX timestamp | +| `CI_PIPELINE_FINISHED` | pipeline finished UNIX timestamp | +| `CI_PIPELINE_FILES` | changed files (empty if event is not `push` or `pull_request`), it is undefined if more than 500 files are touched | +| | **Current workflow** | +| `CI_WORKFLOW_NAME` | workflow name | +| | **Current step** | +| `CI_STEP_NAME` | step name | +| `CI_STEP_NUMBER` | step number | +| `CI_STEP_STATUS` | step status (success, failure) | +| `CI_STEP_STARTED` | step started UNIX timestamp | +| `CI_STEP_FINISHED` | step finished UNIX timestamp | +| `CI_STEP_URL` | URL to step in UI | +| | **Previous commit** | +| `CI_PREV_COMMIT_SHA` | previous commit SHA | +| `CI_PREV_COMMIT_REF` | previous commit ref | +| `CI_PREV_COMMIT_REFSPEC` | previous commit ref spec | +| `CI_PREV_COMMIT_BRANCH` | previous commit branch | +| `CI_PREV_COMMIT_SOURCE_BRANCH` | previous commit source branch | +| `CI_PREV_COMMIT_TARGET_BRANCH` | previous commit target branch | +| `CI_PREV_COMMIT_URL` | previous commit link in forge | +| `CI_PREV_COMMIT_MESSAGE` | previous commit message | +| `CI_PREV_COMMIT_AUTHOR` | previous commit author username | +| `CI_PREV_COMMIT_AUTHOR_EMAIL` | previous commit author email address | +| `CI_PREV_COMMIT_AUTHOR_AVATAR` | previous commit author avatar | +| | **Previous pipeline** | +| `CI_PREV_PIPELINE_NUMBER` | previous pipeline number | +| `CI_PREV_PIPELINE_PARENT` | previous pipeline number of parent pipeline | +| `CI_PREV_PIPELINE_EVENT` | previous pipeline event (push, pull_request, tag, deployment) | +| `CI_PREV_PIPELINE_URL` | previous pipeline link in CI | +| `CI_PREV_PIPELINE_FORGE_URL` | previous pipeline link to event in forge | +| `CI_PREV_PIPELINE_DEPLOY_TARGET` | previous pipeline deploy target for `deployment` events (ie production) | +| `CI_PREV_PIPELINE_STATUS` | previous pipeline status (success, failure) | +| `CI_PREV_PIPELINE_CREATED` | previous pipeline created UNIX timestamp | +| `CI_PREV_PIPELINE_STARTED` | previous pipeline started UNIX timestamp | +| `CI_PREV_PIPELINE_FINISHED` | previous pipeline finished UNIX timestamp | +| |   | +| `CI_WORKSPACE` | Path of the workspace where source code gets cloned to | +| | **System** | +| `CI_SYSTEM_NAME` | name of the CI system: `woodpecker` | +| `CI_SYSTEM_URL` | link to CI system | +| `CI_SYSTEM_HOST` | hostname of CI server | +| `CI_SYSTEM_VERSION` | version of the server | +| | **Forge** | +| `CI_FORGE_TYPE` | name of forge (gitea, github, ...) | +| `CI_FORGE_URL` | root URL of configured forge | +| | **Internal** - Please don't use! | +| `CI_SCRIPT` | Internal script path. Used to call pipeline step commands. | +| `CI_NETRC_USERNAME` | Credentials for private repos to be able to clone data. (Only available for specific images) | +| `CI_NETRC_PASSWORD` | Credentials for private repos to be able to clone data. (Only available for specific images) | +| `CI_NETRC_MACHINE` | Credentials for private repos to be able to clone data. (Only available for specific images) | ## Global environment variables diff --git a/docs/versioned_docs/version-2.0/20-usage/50-environment.md b/docs/versioned_docs/version-2.0/20-usage/50-environment.md index 306b0a651..fb1fe3d7c 100644 --- a/docs/versioned_docs/version-2.0/20-usage/50-environment.md +++ b/docs/versioned_docs/version-2.0/20-usage/50-environment.md @@ -78,7 +78,7 @@ This is the reference list of all environment variables available to your pipeli | | **Current pipeline** | | `CI_PIPELINE_NUMBER` | pipeline number | | `CI_PIPELINE_PARENT` | number of parent pipeline | -| `CI_PIPELINE_EVENT` | pipeline event (push, pull_request, tag, deployment) | +| `CI_PIPELINE_EVENT` | pipeline event (push, pull_request, tag, deployment, cron, manual) | | `CI_PIPELINE_URL` | link to the web UI for the pipeline | | `CI_PIPELINE_FORGE_URL` | link to the forge's web UI for the commit(s) or tag that triggered the pipeline | | `CI_PIPELINE_DEPLOY_TARGET` | pipeline deploy target for `deployment` events (ie production) | diff --git a/pipeline/frontend/metadata/environment.go b/pipeline/frontend/metadata/environment.go index 1e8393095..ff84c172d 100644 --- a/pipeline/frontend/metadata/environment.go +++ b/pipeline/frontend/metadata/environment.go @@ -15,6 +15,7 @@ package metadata import ( + "encoding/json" "fmt" "path" "regexp" @@ -22,7 +23,10 @@ import ( "strings" ) -var pullRegexp = regexp.MustCompile(`\d+`) +var ( + pullRegexp = regexp.MustCompile(`\d+`) + maxChangedFiles = 500 +) // Environ returns the metadata as a map of environment variables. func (m *Metadata) Environ() map[string]string { @@ -127,6 +131,15 @@ func (m *Metadata) Environ() map[string]string { params["CI_COMMIT_PULL_REQUEST_LABELS"] = strings.Join(m.Curr.Commit.PullRequestLabels, ",") } + // Only export changed files if maxChangedFiles is not exceeded + if len(m.Curr.Commit.ChangedFiles) == 0 { + params["CI_PIPELINE_FILES"] = "[]" + } else if len(m.Curr.Commit.ChangedFiles) <= maxChangedFiles { + // we have to use json, as other separators like ;, or space are valid filename chars + changedFiles, _ := json.Marshal(m.Curr.Commit.ChangedFiles) + params["CI_PIPELINE_FILES"] = string(changedFiles) + } + return params } diff --git a/pipeline/frontend/metadata_test.go b/pipeline/frontend/metadata_test.go index 8729e4f0e..ed8519ac7 100644 --- a/pipeline/frontend/metadata_test.go +++ b/pipeline/frontend/metadata_test.go @@ -74,7 +74,7 @@ func TestMetadataFromStruct(t *testing.T) { "CI_COMMIT_AUTHOR": "", "CI_COMMIT_AUTHOR_AVATAR": "", "CI_COMMIT_AUTHOR_EMAIL": "", "CI_COMMIT_BRANCH": "", "CI_COMMIT_MESSAGE": "", "CI_COMMIT_PULL_REQUEST": "", "CI_COMMIT_PULL_REQUEST_LABELS": "", "CI_COMMIT_REF": "", "CI_COMMIT_REFSPEC": "", "CI_COMMIT_SHA": "", "CI_COMMIT_SOURCE_BRANCH": "", "CI_COMMIT_TAG": "", "CI_COMMIT_TARGET_BRANCH": "", "CI_COMMIT_URL": "", "CI_FORGE_TYPE": "", "CI_FORGE_URL": "", - "CI_PIPELINE_CREATED": "0", "CI_PIPELINE_DEPLOY_TARGET": "", "CI_PIPELINE_EVENT": "", "CI_PIPELINE_FINISHED": "0", "CI_PIPELINE_NUMBER": "0", + "CI_PIPELINE_CREATED": "0", "CI_PIPELINE_DEPLOY_TARGET": "", "CI_PIPELINE_EVENT": "", "CI_PIPELINE_FINISHED": "0", "CI_PIPELINE_FILES": "[]", "CI_PIPELINE_NUMBER": "0", "CI_PIPELINE_PARENT": "0", "CI_PIPELINE_STARTED": "0", "CI_PIPELINE_STATUS": "", "CI_PIPELINE_URL": "/repos/0/pipeline/0", "CI_PIPELINE_FORGE_URL": "", "CI_PREV_COMMIT_AUTHOR": "", "CI_PREV_COMMIT_AUTHOR_AVATAR": "", "CI_PREV_COMMIT_AUTHOR_EMAIL": "", "CI_PREV_COMMIT_BRANCH": "", "CI_PREV_COMMIT_MESSAGE": "", "CI_PREV_COMMIT_REF": "", "CI_PREV_COMMIT_REFSPEC": "", "CI_PREV_COMMIT_SHA": "", "CI_PREV_COMMIT_URL": "", "CI_PREV_PIPELINE_CREATED": "0", @@ -89,15 +89,18 @@ func TestMetadataFromStruct(t *testing.T) { name: "Test with forge", forge: forge, repo: &model.Repo{FullName: "testUser/testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", Clone: "https://gitea.com/testUser/testRepo.git", CloneSSH: "git@gitea.com:testUser/testRepo.git", Branch: "main", IsSCMPrivate: true}, - pipeline: &model.Pipeline{Number: 3}, + pipeline: &model.Pipeline{Number: 3, ChangedFiles: []string{"test.go", "markdown file.md"}}, last: &model.Pipeline{Number: 2}, workflow: &model.Workflow{Name: "hello"}, sysURL: "https://example.com", expectedMetadata: metadata.Metadata{ - Forge: metadata.Forge{Type: "gitea", URL: "https://gitea.com"}, - Sys: metadata.System{Name: "woodpecker", Host: "example.com", URL: "https://example.com"}, - Repo: metadata.Repo{Owner: "testUser", Name: "testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", CloneURL: "https://gitea.com/testUser/testRepo.git", CloneSSHURL: "git@gitea.com:testUser/testRepo.git", Branch: "main", Private: true}, - Curr: metadata.Pipeline{Number: 3}, + Forge: metadata.Forge{Type: "gitea", URL: "https://gitea.com"}, + Sys: metadata.System{Name: "woodpecker", Host: "example.com", URL: "https://example.com"}, + Repo: metadata.Repo{Owner: "testUser", Name: "testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", CloneURL: "https://gitea.com/testUser/testRepo.git", CloneSSHURL: "git@gitea.com:testUser/testRepo.git", Branch: "main", Private: true}, + Curr: metadata.Pipeline{ + Number: 3, + Commit: metadata.Commit{ChangedFiles: []string{"test.go", "markdown file.md"}}, + }, Prev: metadata.Pipeline{Number: 2}, Workflow: metadata.Workflow{Name: "hello"}, }, @@ -106,7 +109,7 @@ func TestMetadataFromStruct(t *testing.T) { "CI_COMMIT_AUTHOR": "", "CI_COMMIT_AUTHOR_AVATAR": "", "CI_COMMIT_AUTHOR_EMAIL": "", "CI_COMMIT_BRANCH": "", "CI_COMMIT_MESSAGE": "", "CI_COMMIT_PULL_REQUEST": "", "CI_COMMIT_PULL_REQUEST_LABELS": "", "CI_COMMIT_REF": "", "CI_COMMIT_REFSPEC": "", "CI_COMMIT_SHA": "", "CI_COMMIT_SOURCE_BRANCH": "", "CI_COMMIT_TAG": "", "CI_COMMIT_TARGET_BRANCH": "", "CI_COMMIT_URL": "", "CI_FORGE_TYPE": "gitea", "CI_FORGE_URL": "https://gitea.com", - "CI_PIPELINE_CREATED": "0", "CI_PIPELINE_DEPLOY_TARGET": "", "CI_PIPELINE_EVENT": "", "CI_PIPELINE_FINISHED": "0", + "CI_PIPELINE_CREATED": "0", "CI_PIPELINE_DEPLOY_TARGET": "", "CI_PIPELINE_EVENT": "", "CI_PIPELINE_FINISHED": "0", "CI_PIPELINE_FILES": `["test.go","markdown file.md"]`, "CI_PIPELINE_NUMBER": "3", "CI_PIPELINE_PARENT": "0", "CI_PIPELINE_STARTED": "0", "CI_PIPELINE_STATUS": "", "CI_PIPELINE_URL": "https://example.com/repos/0/pipeline/3", "CI_PIPELINE_FORGE_URL": "", "CI_PREV_COMMIT_AUTHOR": "", "CI_PREV_COMMIT_AUTHOR_AVATAR": "", "CI_PREV_COMMIT_AUTHOR_EMAIL": "", "CI_PREV_COMMIT_BRANCH": "", "CI_PREV_COMMIT_MESSAGE": "", "CI_PREV_COMMIT_REF": "", "CI_PREV_COMMIT_REFSPEC": "", "CI_PREV_COMMIT_SHA": "", "CI_PREV_COMMIT_URL": "", "CI_PREV_PIPELINE_CREATED": "0",