From e89a2f38fd41a48ec0d5c76039c1d576cf7e2cfd Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 16 Sep 2024 22:03:24 +0200 Subject: [PATCH] Make cli exec metadata on pair with build in server generated metadata (#4119) remove some old environment and add all missing options to set the whole build-in environment on `cli exec` via flags --- *Sponsored by Kithara Software GmbH* Co-authored-by: qwerty287 <80460567+qwerty287@users.noreply.github.com> --- cli/exec/exec.go | 5 +- cli/exec/flags.go | 48 +++++++++++++++++--- cli/exec/metadata.go | 24 +++++++++- pipeline/frontend/metadata/environment.go | 2 +- pipeline/frontend/metadata/types.go | 1 + server/pipeline/stepbuilder/metadata.go | 1 + server/pipeline/stepbuilder/metadata_test.go | 6 +-- 7 files changed, 74 insertions(+), 13 deletions(-) diff --git a/cli/exec/exec.go b/cli/exec/exec.go index c705de267..59bbe47e6 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -129,7 +129,10 @@ func runExec(ctx context.Context, c *cli.Command, file, repoPath string) error { } func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, axis matrix.Axis) error { - metadata := metadataFromContext(ctx, c, axis) + metadata, err := metadataFromContext(ctx, c, axis) + if err != nil { + return fmt.Errorf("could not create metadata: %w", err) + } environ := metadata.Environ() var secrets []compiler.Secret for key, val := range metadata.Workflow.Matrix { diff --git a/cli/exec/flags.go b/cli/exec/flags.go index 51bcb082f..1acc44ee3 100644 --- a/cli/exec/flags.go +++ b/cli/exec/flags.go @@ -126,6 +126,10 @@ var flags = []cli.Flag{ Sources: cli.EnvVars("CI_SYSTEM_PLATFORM"), Name: "system-platform", }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_SYSTEM_HOST"), + Name: "system-host", + }, &cli.StringFlag{ Sources: cli.EnvVars("CI_SYSTEM_NAME"), Name: "system-name", @@ -149,6 +153,16 @@ var flags = []cli.Flag{ Sources: cli.EnvVars("CI_REPO_URL"), Name: "repo-url", }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_REPO_SCM"), + Name: "repo-scm", + Value: "git", + }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_REPO_DEFAULT_BRANCH"), + Name: "repo-default-branch", + Value: "main", + }, &cli.StringFlag{ Sources: cli.EnvVars("CI_REPO_CLONE_URL"), Name: "repo-clone-url", @@ -187,17 +201,22 @@ var flags = []cli.Flag{ Value: "manual", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_PIPELINE_URL"), + Sources: cli.EnvVars("CI_PIPELINE_FORGE_URL"), Name: "pipeline-url", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TARGET", "CI_PIPELINE_TARGET"), // TODO: remove CI_PIPELINE_TARGET in 3.x + Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TARGET"), Name: "pipeline-deploy-to", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TASK", "CI_PIPELINE_TASK"), // TODO: remove CI_PIPELINE_TASK in 3.x + Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TASK"), Name: "pipeline-deploy-task", }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_PIPELINE_FILES"), + Usage: "either json formatted list of strings, or comma separated string list", + Name: "pipeline-files", + }, &cli.StringFlag{ Sources: cli.EnvVars("CI_COMMIT_SHA"), Name: "commit-sha", @@ -213,13 +232,14 @@ var flags = []cli.Flag{ &cli.StringFlag{ Sources: cli.EnvVars("CI_COMMIT_BRANCH"), Name: "commit-branch", + Value: "main", }, &cli.StringFlag{ Sources: cli.EnvVars("CI_COMMIT_MESSAGE"), Name: "commit-message", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_COMMIT_AUTHOR_NAME"), + Sources: cli.EnvVars("CI_COMMIT_AUTHOR"), Name: "commit-author-name", }, &cli.StringFlag{ @@ -230,6 +250,14 @@ var flags = []cli.Flag{ Sources: cli.EnvVars("CI_COMMIT_AUTHOR_EMAIL"), Name: "commit-author-email", }, + &cli.StringSliceFlag{ + Sources: cli.EnvVars("CI_COMMIT_PULL_REQUEST_LABELS"), + Name: "commit-pull-labels", + }, + &cli.BoolFlag{ + Sources: cli.EnvVars("CI_COMMIT_PRERELEASE"), + Name: "commit-release-is-pre", + }, &cli.IntFlag{ Sources: cli.EnvVars("CI_PREV_PIPELINE_NUMBER"), Name: "prev-pipeline-number", @@ -255,9 +283,17 @@ var flags = []cli.Flag{ Name: "prev-pipeline-event", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_PREV_PIPELINE_URL"), + Sources: cli.EnvVars("CI_PREV_PIPELINE_FORGE_URL"), Name: "prev-pipeline-url", }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_PREV_PIPELINE_DEPLOY_TARGET"), + Name: "prev-pipeline-deploy-to", + }, + &cli.StringFlag{ + Sources: cli.EnvVars("CI_PREV_PIPELINE_DEPLOY_TASK"), + Name: "prev-pipeline-deploy-task", + }, &cli.StringFlag{ Sources: cli.EnvVars("CI_PREV_COMMIT_SHA"), Name: "prev-commit-sha", @@ -279,7 +315,7 @@ var flags = []cli.Flag{ Name: "prev-commit-message", }, &cli.StringFlag{ - Sources: cli.EnvVars("CI_PREV_COMMIT_AUTHOR_NAME"), + Sources: cli.EnvVars("CI_PREV_COMMIT_AUTHOR"), Name: "prev-commit-author-name", }, &cli.StringFlag{ diff --git a/cli/exec/metadata.go b/cli/exec/metadata.go index 20a89da66..dff5f7947 100644 --- a/cli/exec/metadata.go +++ b/cli/exec/metadata.go @@ -16,6 +16,8 @@ package exec import ( "context" + "encoding/json" + "fmt" "runtime" "strings" @@ -27,7 +29,7 @@ import ( ) // return the metadata from the cli context. -func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) metadata.Metadata { +func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) (metadata.Metadata, error) { platform := c.String("system-platform") if platform == "" { platform = runtime.GOOS + "/" + runtime.GOARCH @@ -41,12 +43,26 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me repoName = fullRepoName[idx+1:] } + var changedFiles []string + changedFilesRaw := c.String("pipeline-files") + if changedFilesRaw[0] == '[' { + if err := json.Unmarshal([]byte(changedFilesRaw), &changedFiles); err != nil { + return metadata.Metadata{}, fmt.Errorf("pipeline-files detected json but could not parse it: %w", err) + } + } else { + for _, file := range strings.Split(changedFilesRaw, ",") { + changedFiles = append(changedFiles, strings.TrimSpace(file)) + } + } + return metadata.Metadata{ Repo: metadata.Repo{ Name: repoName, Owner: repoOwner, RemoteID: c.String("repo-remote-id"), ForgeURL: c.String("repo-url"), + SCM: c.String("repo-scm"), + Branch: c.String("repo-default-branch"), CloneURL: c.String("repo-clone-url"), CloneSSHURL: c.String("repo-clone-ssh-url"), Private: c.Bool("repo-private"), @@ -74,6 +90,9 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me Email: c.String("commit-author-email"), Avatar: c.String("commit-author-avatar"), }, + PullRequestLabels: c.StringSlice("commit-pull-labels"), + IsPrerelease: c.Bool("commit-release-is-pre"), + ChangedFiles: changedFiles, }, }, Prev: metadata.Pipeline{ @@ -109,6 +128,7 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me Sys: metadata.System{ Name: c.String("system-name"), URL: c.String("system-url"), + Host: c.String("system-host"), Platform: platform, Version: version.Version, }, @@ -116,5 +136,5 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me Type: c.String("forge-type"), URL: c.String("forge-url"), }, - } + }, nil } diff --git a/pipeline/frontend/metadata/environment.go b/pipeline/frontend/metadata/environment.go index c48efb258..1a018867f 100644 --- a/pipeline/frontend/metadata/environment.go +++ b/pipeline/frontend/metadata/environment.go @@ -56,7 +56,7 @@ func (m *Metadata) Environ() map[string]string { "CI_REPO_NAME": m.Repo.Name, "CI_REPO_OWNER": m.Repo.Owner, "CI_REPO_REMOTE_ID": m.Repo.RemoteID, - "CI_REPO_SCM": "git", + "CI_REPO_SCM": m.Repo.SCM, "CI_REPO_URL": m.Repo.ForgeURL, "CI_REPO_CLONE_URL": m.Repo.CloneURL, "CI_REPO_CLONE_SSH_URL": m.Repo.CloneSSHURL, diff --git a/pipeline/frontend/metadata/types.go b/pipeline/frontend/metadata/types.go index 9cf7e64a1..fcbd0185b 100644 --- a/pipeline/frontend/metadata/types.go +++ b/pipeline/frontend/metadata/types.go @@ -34,6 +34,7 @@ type ( Owner string `json:"owner,omitempty"` RemoteID string `json:"remote_id,omitempty"` ForgeURL string `json:"forge_url,omitempty"` + SCM string `json:"scm,omitempty"` CloneURL string `json:"clone_url,omitempty"` CloneSSHURL string `json:"clone_url_ssh,omitempty"` Private bool `json:"private,omitempty"` diff --git a/server/pipeline/stepbuilder/metadata.go b/server/pipeline/stepbuilder/metadata.go index 6e48e4158..3ea13f9e4 100644 --- a/server/pipeline/stepbuilder/metadata.go +++ b/server/pipeline/stepbuilder/metadata.go @@ -48,6 +48,7 @@ func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, Owner: repo.Owner, RemoteID: fmt.Sprint(repo.ForgeRemoteID), ForgeURL: repo.ForgeURL, + SCM: string(repo.SCMKind), CloneURL: repo.Clone, CloneSSHURL: repo.CloneSSH, Private: repo.IsSCMPrivate, diff --git a/server/pipeline/stepbuilder/metadata_test.go b/server/pipeline/stepbuilder/metadata_test.go index fd85f8d07..368eae47e 100644 --- a/server/pipeline/stepbuilder/metadata_test.go +++ b/server/pipeline/stepbuilder/metadata_test.go @@ -53,7 +53,7 @@ func TestMetadataFromStruct(t *testing.T) { "CI_PREV_COMMIT_MESSAGE": "", "CI_PREV_COMMIT_REF": "", "CI_PREV_COMMIT_REFSPEC": "", "CI_PREV_COMMIT_SHA": "", "CI_PREV_COMMIT_URL": "", "CI_PREV_PIPELINE_CREATED": "0", "CI_PREV_PIPELINE_DEPLOY_TARGET": "", "CI_PREV_PIPELINE_DEPLOY_TASK": "", "CI_PREV_PIPELINE_EVENT": "", "CI_PREV_PIPELINE_FINISHED": "0", "CI_PREV_PIPELINE_NUMBER": "0", "CI_PREV_PIPELINE_PARENT": "0", "CI_PREV_PIPELINE_STARTED": "0", "CI_PREV_PIPELINE_STATUS": "", "CI_PREV_PIPELINE_URL": "/repos/0/pipeline/0", "CI_PREV_PIPELINE_FORGE_URL": "", "CI_REPO": "", "CI_REPO_CLONE_URL": "", "CI_REPO_CLONE_SSH_URL": "", "CI_REPO_DEFAULT_BRANCH": "", "CI_REPO_REMOTE_ID": "", - "CI_REPO_NAME": "", "CI_REPO_OWNER": "", "CI_REPO_PRIVATE": "false", "CI_REPO_SCM": "git", "CI_REPO_TRUSTED": "false", "CI_REPO_URL": "", + "CI_REPO_NAME": "", "CI_REPO_OWNER": "", "CI_REPO_PRIVATE": "false", "CI_REPO_SCM": "", "CI_REPO_TRUSTED": "false", "CI_REPO_URL": "", "CI_STEP_NAME": "", "CI_STEP_NUMBER": "0", "CI_STEP_STARTED": "", "CI_STEP_URL": "/repos/0/pipeline/0", "CI_SYSTEM_HOST": "", "CI_SYSTEM_NAME": "woodpecker", "CI_SYSTEM_PLATFORM": "", "CI_SYSTEM_URL": "", "CI_SYSTEM_VERSION": "", "CI_WORKFLOW_NAME": "", "CI_WORKFLOW_NUMBER": "0", }, @@ -61,7 +61,7 @@ func TestMetadataFromStruct(t *testing.T) { { name: "Test with forge", forge: forge, - repo: &model.Repo{FullName: "testUser/testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", Clone: "https://gitea.com/testUser/testRepo.git", CloneSSH: "git@gitea.com:testUser/testRepo.git", Branch: "main", IsSCMPrivate: true}, + repo: &model.Repo{FullName: "testUser/testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", Clone: "https://gitea.com/testUser/testRepo.git", CloneSSH: "git@gitea.com:testUser/testRepo.git", Branch: "main", IsSCMPrivate: true, SCMKind: "git"}, pipeline: &model.Pipeline{Number: 3, ChangedFiles: []string{"test.go", "markdown file.md"}}, last: &model.Pipeline{Number: 2}, workflow: &model.Workflow{Name: "hello"}, @@ -69,7 +69,7 @@ func TestMetadataFromStruct(t *testing.T) { expectedMetadata: metadata.Metadata{ Forge: metadata.Forge{Type: "gitea", URL: "https://gitea.com"}, Sys: metadata.System{Name: "woodpecker", Host: "example.com", URL: "https://example.com"}, - Repo: metadata.Repo{Owner: "testUser", Name: "testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", CloneURL: "https://gitea.com/testUser/testRepo.git", CloneSSHURL: "git@gitea.com:testUser/testRepo.git", Branch: "main", Private: true}, + Repo: metadata.Repo{Owner: "testUser", Name: "testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", CloneURL: "https://gitea.com/testUser/testRepo.git", CloneSSHURL: "git@gitea.com:testUser/testRepo.git", Branch: "main", Private: true, SCM: "git"}, Curr: metadata.Pipeline{ Number: 3, Commit: metadata.Commit{ChangedFiles: []string{"test.go", "markdown file.md"}},