Make cli exec metadata on pair with build in server generated metadata (#4119)

remove some old environment and add all missing options to set the whole build-in environment on `cli exec` via flags


---
*Sponsored by Kithara Software GmbH*

Co-authored-by: qwerty287 <80460567+qwerty287@users.noreply.github.com>

cli/exec/exec.go

@@ -129,7 +129,10 @@ func runExec(ctx context.Context, c *cli.Command, file, repoPath string) error {
 }
 
 func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, axis matrix.Axis) error {
-	metadata := metadataFromContext(ctx, c, axis)
+	metadata, err := metadataFromContext(ctx, c, axis)
+	if err != nil {
+		return fmt.Errorf("could not create metadata: %w", err)
+	}
 	environ := metadata.Environ()
 	var secrets []compiler.Secret
 	for key, val := range metadata.Workflow.Matrix {

cli/exec/flags.go

@@ -126,6 +126,10 @@ var flags = []cli.Flag{
 		Sources: cli.EnvVars("CI_SYSTEM_PLATFORM"),
 		Name:    "system-platform",
 	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("CI_SYSTEM_HOST"),
+		Name:    "system-host",
+	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("CI_SYSTEM_NAME"),
 		Name:    "system-name",
@@ -149,6 +153,16 @@ var flags = []cli.Flag{
 		Sources: cli.EnvVars("CI_REPO_URL"),
 		Name:    "repo-url",
 	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("CI_REPO_SCM"),
+		Name:    "repo-scm",
+		Value:   "git",
+	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("CI_REPO_DEFAULT_BRANCH"),
+		Name:    "repo-default-branch",
+		Value:   "main",
+	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("CI_REPO_CLONE_URL"),
 		Name:    "repo-clone-url",
@@ -187,17 +201,22 @@ var flags = []cli.Flag{
 		Value:   "manual",
 	},
 	&cli.StringFlag{
-		Sources: cli.EnvVars("CI_PIPELINE_URL"),
+		Sources: cli.EnvVars("CI_PIPELINE_FORGE_URL"),
 		Name:    "pipeline-url",
 	},
 	&cli.StringFlag{
-		Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TARGET", "CI_PIPELINE_TARGET"), // TODO: remove CI_PIPELINE_TARGET in 3.x
+		Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TARGET"),
 		Name:    "pipeline-deploy-to",
 	},
 	&cli.StringFlag{
-		Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TASK", "CI_PIPELINE_TASK"), // TODO: remove CI_PIPELINE_TASK in 3.x
+		Sources: cli.EnvVars("CI_PIPELINE_DEPLOY_TASK"),
 		Name:    "pipeline-deploy-task",
 	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("CI_PIPELINE_FILES"),
+		Usage:   "either json formatted list of strings, or comma separated string list",
+		Name:    "pipeline-files",
+	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("CI_COMMIT_SHA"),
 		Name:    "commit-sha",
@@ -213,13 +232,14 @@ var flags = []cli.Flag{
 	&cli.StringFlag{
 		Sources: cli.EnvVars("CI_COMMIT_BRANCH"),
 		Name:    "commit-branch",
+		Value:   "main",
 	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("CI_COMMIT_MESSAGE"),
 		Name:    "commit-message",
 	},
 	&cli.StringFlag{
-		Sources: cli.EnvVars("CI_COMMIT_AUTHOR_NAME"),
+		Sources: cli.EnvVars("CI_COMMIT_AUTHOR"),
 		Name:    "commit-author-name",
 	},
 	&cli.StringFlag{
@@ -230,6 +250,14 @@ var flags = []cli.Flag{
 		Sources: cli.EnvVars("CI_COMMIT_AUTHOR_EMAIL"),
 		Name:    "commit-author-email",
 	},
+	&cli.StringSliceFlag{
+		Sources: cli.EnvVars("CI_COMMIT_PULL_REQUEST_LABELS"),
+		Name:    "commit-pull-labels",
+	},
+	&cli.BoolFlag{
+		Sources: cli.EnvVars("CI_COMMIT_PRERELEASE"),
+		Name:    "commit-release-is-pre",
+	},
 	&cli.IntFlag{
 		Sources: cli.EnvVars("CI_PREV_PIPELINE_NUMBER"),
 		Name:    "prev-pipeline-number",
@@ -255,9 +283,17 @@ var flags = []cli.Flag{
 		Name:    "prev-pipeline-event",
 	},
 	&cli.StringFlag{
-		Sources: cli.EnvVars("CI_PREV_PIPELINE_URL"),
+		Sources: cli.EnvVars("CI_PREV_PIPELINE_FORGE_URL"),
 		Name:    "prev-pipeline-url",
 	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("CI_PREV_PIPELINE_DEPLOY_TARGET"),
+		Name:    "prev-pipeline-deploy-to",
+	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("CI_PREV_PIPELINE_DEPLOY_TASK"),
+		Name:    "prev-pipeline-deploy-task",
+	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("CI_PREV_COMMIT_SHA"),
 		Name:    "prev-commit-sha",
@@ -279,7 +315,7 @@ var flags = []cli.Flag{
 		Name:    "prev-commit-message",
 	},
 	&cli.StringFlag{
-		Sources: cli.EnvVars("CI_PREV_COMMIT_AUTHOR_NAME"),
+		Sources: cli.EnvVars("CI_PREV_COMMIT_AUTHOR"),
 		Name:    "prev-commit-author-name",
 	},
 	&cli.StringFlag{

cli/exec/metadata.go

@@ -16,6 +16,8 @@ package exec
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"runtime"
 	"strings"
 
@@ -27,7 +29,7 @@ import (
 )
 
 // return the metadata from the cli context.
-func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) metadata.Metadata {
+func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) (metadata.Metadata, error) {
 	platform := c.String("system-platform")
 	if platform == "" {
 		platform = runtime.GOOS + "/" + runtime.GOARCH
@@ -41,12 +43,26 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me
 		repoName = fullRepoName[idx+1:]
 	}
 
+	var changedFiles []string
+	changedFilesRaw := c.String("pipeline-files")
+	if changedFilesRaw[0] == '[' {
+		if err := json.Unmarshal([]byte(changedFilesRaw), &changedFiles); err != nil {
+			return metadata.Metadata{}, fmt.Errorf("pipeline-files detected json but could not parse it: %w", err)
+		}
+	} else {
+		for _, file := range strings.Split(changedFilesRaw, ",") {
+			changedFiles = append(changedFiles, strings.TrimSpace(file))
+		}
+	}
+
 	return metadata.Metadata{
 		Repo: metadata.Repo{
 			Name:        repoName,
 			Owner:       repoOwner,
 			RemoteID:    c.String("repo-remote-id"),
 			ForgeURL:    c.String("repo-url"),
+			SCM:         c.String("repo-scm"),
+			Branch:      c.String("repo-default-branch"),
 			CloneURL:    c.String("repo-clone-url"),
 			CloneSSHURL: c.String("repo-clone-ssh-url"),
 			Private:     c.Bool("repo-private"),
@@ -74,6 +90,9 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me
 					Email:  c.String("commit-author-email"),
 					Avatar: c.String("commit-author-avatar"),
 				},
+				PullRequestLabels: c.StringSlice("commit-pull-labels"),
+				IsPrerelease:      c.Bool("commit-release-is-pre"),
+				ChangedFiles:      changedFiles,
 			},
 		},
 		Prev: metadata.Pipeline{
@@ -109,6 +128,7 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me
 		Sys: metadata.System{
 			Name:     c.String("system-name"),
 			URL:      c.String("system-url"),
+			Host:     c.String("system-host"),
 			Platform: platform,
 			Version:  version.Version,
 		},
@@ -116,5 +136,5 @@ func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) me
 			Type: c.String("forge-type"),
 			URL:  c.String("forge-url"),
 		},
-	}
+	}, nil
 }

pipeline/frontend/metadata/environment.go

@@ -56,7 +56,7 @@ func (m *Metadata) Environ() map[string]string {
 		"CI_REPO_NAME":           m.Repo.Name,
 		"CI_REPO_OWNER":          m.Repo.Owner,
 		"CI_REPO_REMOTE_ID":      m.Repo.RemoteID,
-		"CI_REPO_SCM":            "git",
+		"CI_REPO_SCM":            m.Repo.SCM,
 		"CI_REPO_URL":            m.Repo.ForgeURL,
 		"CI_REPO_CLONE_URL":      m.Repo.CloneURL,
 		"CI_REPO_CLONE_SSH_URL":  m.Repo.CloneSSHURL,

pipeline/frontend/metadata/types.go

@@ -34,6 +34,7 @@ type (
 		Owner       string   `json:"owner,omitempty"`
 		RemoteID    string   `json:"remote_id,omitempty"`
 		ForgeURL    string   `json:"forge_url,omitempty"`
+		SCM         string   `json:"scm,omitempty"`
 		CloneURL    string   `json:"clone_url,omitempty"`
 		CloneSSHURL string   `json:"clone_url_ssh,omitempty"`
 		Private     bool     `json:"private,omitempty"`

server/pipeline/stepbuilder/metadata.go

@@ -48,6 +48,7 @@ func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline,
 			Owner:       repo.Owner,
 			RemoteID:    fmt.Sprint(repo.ForgeRemoteID),
 			ForgeURL:    repo.ForgeURL,
+			SCM:         string(repo.SCMKind),
 			CloneURL:    repo.Clone,
 			CloneSSHURL: repo.CloneSSH,
 			Private:     repo.IsSCMPrivate,

server/pipeline/stepbuilder/metadata_test.go

@@ -53,7 +53,7 @@ func TestMetadataFromStruct(t *testing.T) {
 				"CI_PREV_COMMIT_MESSAGE": "", "CI_PREV_COMMIT_REF": "", "CI_PREV_COMMIT_REFSPEC": "", "CI_PREV_COMMIT_SHA": "", "CI_PREV_COMMIT_URL": "", "CI_PREV_PIPELINE_CREATED": "0",
 				"CI_PREV_PIPELINE_DEPLOY_TARGET": "", "CI_PREV_PIPELINE_DEPLOY_TASK": "", "CI_PREV_PIPELINE_EVENT": "", "CI_PREV_PIPELINE_FINISHED": "0", "CI_PREV_PIPELINE_NUMBER": "0", "CI_PREV_PIPELINE_PARENT": "0",
 				"CI_PREV_PIPELINE_STARTED": "0", "CI_PREV_PIPELINE_STATUS": "", "CI_PREV_PIPELINE_URL": "/repos/0/pipeline/0", "CI_PREV_PIPELINE_FORGE_URL": "", "CI_REPO": "", "CI_REPO_CLONE_URL": "", "CI_REPO_CLONE_SSH_URL": "", "CI_REPO_DEFAULT_BRANCH": "", "CI_REPO_REMOTE_ID": "",
-				"CI_REPO_NAME": "", "CI_REPO_OWNER": "", "CI_REPO_PRIVATE": "false", "CI_REPO_SCM": "git", "CI_REPO_TRUSTED": "false", "CI_REPO_URL": "",
+				"CI_REPO_NAME": "", "CI_REPO_OWNER": "", "CI_REPO_PRIVATE": "false", "CI_REPO_SCM": "", "CI_REPO_TRUSTED": "false", "CI_REPO_URL": "",
 				"CI_STEP_NAME": "", "CI_STEP_NUMBER": "0", "CI_STEP_STARTED": "", "CI_STEP_URL": "/repos/0/pipeline/0", "CI_SYSTEM_HOST": "", "CI_SYSTEM_NAME": "woodpecker",
 				"CI_SYSTEM_PLATFORM": "", "CI_SYSTEM_URL": "", "CI_SYSTEM_VERSION": "", "CI_WORKFLOW_NAME": "", "CI_WORKFLOW_NUMBER": "0",
 			},
@@ -61,7 +61,7 @@ func TestMetadataFromStruct(t *testing.T) {
 		{
 			name:     "Test with forge",
 			forge:    forge,
-			repo:     &model.Repo{FullName: "testUser/testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", Clone: "https://gitea.com/testUser/testRepo.git", CloneSSH: "git@gitea.com:testUser/testRepo.git", Branch: "main", IsSCMPrivate: true},
+			repo:     &model.Repo{FullName: "testUser/testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", Clone: "https://gitea.com/testUser/testRepo.git", CloneSSH: "git@gitea.com:testUser/testRepo.git", Branch: "main", IsSCMPrivate: true, SCMKind: "git"},
 			pipeline: &model.Pipeline{Number: 3, ChangedFiles: []string{"test.go", "markdown file.md"}},
 			last:     &model.Pipeline{Number: 2},
 			workflow: &model.Workflow{Name: "hello"},
@@ -69,7 +69,7 @@ func TestMetadataFromStruct(t *testing.T) {
 			expectedMetadata: metadata.Metadata{
 				Forge: metadata.Forge{Type: "gitea", URL: "https://gitea.com"},
 				Sys:   metadata.System{Name: "woodpecker", Host: "example.com", URL: "https://example.com"},
-				Repo:  metadata.Repo{Owner: "testUser", Name: "testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", CloneURL: "https://gitea.com/testUser/testRepo.git", CloneSSHURL: "git@gitea.com:testUser/testRepo.git", Branch: "main", Private: true},
+				Repo:  metadata.Repo{Owner: "testUser", Name: "testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", CloneURL: "https://gitea.com/testUser/testRepo.git", CloneSSHURL: "git@gitea.com:testUser/testRepo.git", Branch: "main", Private: true, SCM: "git"},
 				Curr: metadata.Pipeline{
 					Number: 3,
 					Commit: metadata.Commit{ChangedFiles: []string{"test.go", "markdown file.md"}},