From 0efd7130f04132c98d2561de8b499063fde15c9f Mon Sep 17 00:00:00 2001 From: katsu Date: Mon, 23 Dec 2024 05:32:28 +0800 Subject: [PATCH 1/5] fix trailing comma not matched in the case of alphanumeric issue (#32945) Fix #32428. Patch the regex to match `,`besides `.` `"` `'` `:` and space. (cherry picked from commit fb75151fb14e866cab8d893c588f5673902b278b) --- modules/references/references.go | 2 +- modules/references/references_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/references/references.go b/modules/references/references.go index c61d06d5dc..3b4bcb3706 100644 --- a/modules/references/references.go +++ b/modules/references/references.go @@ -32,7 +32,7 @@ var ( // issueNumericPattern matches string that references to a numeric issue, e.g. #1287 issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\'|\")([#!][0-9]+)(?:\s|$|\)|\]|\'|\"|[:;,.?!]\s|[:;,.?!]$)`) // issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234 - issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\"|\')([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$)|\"|\')`) + issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\"|\')([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$)|\"|\'|,)`) // crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository // e.g. org/repo#12345 crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+[#!][0-9]+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`) diff --git a/modules/references/references_test.go b/modules/references/references_test.go index ffa7f993e3..d5f7c4b4c5 100644 --- a/modules/references/references_test.go +++ b/modules/references/references_test.go @@ -466,6 +466,7 @@ func TestRegExp_issueAlphanumericPattern(t *testing.T) { "ABC-123:", "\"ABC-123\"", "'ABC-123'", + "ABC-123, unknown PR", } falseTestCases := []string{ "RC-08", From 2ffa9a5e6e24715ab62a571ca1ca9a531944ca86 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 24 Dec 2024 21:52:46 -0800 Subject: [PATCH 2/5] demilestone should not include milestone (#32923) Fix #32887 (cherry picked from commit f44712f22bc7bfce049c64c27f60453ff1e41a5c) Conflicts: services/issue/milestone_test.go trivial conflicts (require vs assert) --- services/issue/milestone.go | 4 ++++ services/issue/milestone_test.go | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/services/issue/milestone.go b/services/issue/milestone.go index 31490c7b03..407ad0a59b 100644 --- a/services/issue/milestone.go +++ b/services/issue/milestone.go @@ -85,6 +85,10 @@ func changeMilestoneAssign(ctx context.Context, doer *user_model.User, issue *is } } + if issue.MilestoneID == 0 { + issue.Milestone = nil + } + return nil } diff --git a/services/issue/milestone_test.go b/services/issue/milestone_test.go index 1c06572f8e..e75f64550c 100644 --- a/services/issue/milestone_test.go +++ b/services/issue/milestone_test.go @@ -24,6 +24,7 @@ func TestChangeMilestoneAssign(t *testing.T) { oldMilestoneID := issue.MilestoneID issue.MilestoneID = 2 + require.NoError(t, issue.LoadMilestone(db.DefaultContext)) require.NoError(t, ChangeMilestoneAssign(db.DefaultContext, issue, doer, oldMilestoneID)) unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ IssueID: issue.ID, @@ -32,4 +33,11 @@ func TestChangeMilestoneAssign(t *testing.T) { OldMilestoneID: oldMilestoneID, }) unittest.CheckConsistencyFor(t, &issues_model.Milestone{}, &issues_model.Issue{}) + assert.NotNil(t, issue.Milestone) + + oldMilestoneID = issue.MilestoneID + issue.MilestoneID = 0 + require.NoError(t, ChangeMilestoneAssign(db.DefaultContext, issue, doer, oldMilestoneID)) + assert.EqualValues(t, 0, issue.MilestoneID) + assert.Nil(t, issue.Milestone) } From 92ac337263ad84b76958677238f51464f5ee759d Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Fri, 27 Dec 2024 03:14:49 +0800 Subject: [PATCH 3/5] Support org labels when adding labels by label names (#32988) Fix #32891 (cherry picked from commit 44b4fb21a4e99f327303f66cc7e48f7ca7ba09e1) --- models/fixtures/label.yml | 11 +++++++++++ models/issues/label.go | 11 +++++++++++ routers/api/v1/repo/issue_label.go | 14 ++++++++++++- tests/integration/api_issue_label_test.go | 24 ++++++++++++++--------- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/models/fixtures/label.yml b/models/fixtures/label.yml index 2242b90dcd..acfac74968 100644 --- a/models/fixtures/label.yml +++ b/models/fixtures/label.yml @@ -96,3 +96,14 @@ num_issues: 0 num_closed_issues: 0 archived_unix: 0 + +- + id: 10 + repo_id: 3 + org_id: 0 + name: repo3label1 + color: '#112233' + exclusive: false + num_issues: 0 + num_closed_issues: 0 + archived_unix: 0 diff --git a/models/issues/label.go b/models/issues/label.go index 804a118e7a..259c87459e 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -353,6 +353,17 @@ func GetLabelIDsInRepoByNames(ctx context.Context, repoID int64, labelNames []st Find(&labelIDs) } +// GetLabelIDsInOrgByNames returns a list of labelIDs by names in a given org. +func GetLabelIDsInOrgByNames(ctx context.Context, orgID int64, labelNames []string) ([]int64, error) { + labelIDs := make([]int64, 0, len(labelNames)) + return labelIDs, db.GetEngine(ctx).Table("label"). + Where("org_id = ?", orgID). + In("name", labelNames). + Asc("name"). + Cols("id"). + Find(&labelIDs) +} + // BuildLabelNamesIssueIDsCondition returns a builder where get issue ids match label names func BuildLabelNamesIssueIDsCondition(labelNames []string) *builder.Builder { return builder.Select("issue_label.issue_id"). diff --git a/routers/api/v1/repo/issue_label.go b/routers/api/v1/repo/issue_label.go index ae05544365..6458fbf514 100644 --- a/routers/api/v1/repo/issue_label.go +++ b/routers/api/v1/repo/issue_label.go @@ -350,6 +350,9 @@ func prepareForReplaceOrAdd(ctx *context.APIContext, form api.IssueLabelsOption) labelIDs = append(labelIDs, int64(rv.Float())) case reflect.String: labelNames = append(labelNames, rv.String()) + default: + ctx.Error(http.StatusBadRequest, "InvalidLabel", "a label must be an integer or a string") + return nil, nil, fmt.Errorf("invalid label") } } if len(labelIDs) > 0 && len(labelNames) > 0 { @@ -357,11 +360,20 @@ func prepareForReplaceOrAdd(ctx *context.APIContext, form api.IssueLabelsOption) return nil, nil, fmt.Errorf("invalid labels") } if len(labelNames) > 0 { - labelIDs, err = issues_model.GetLabelIDsInRepoByNames(ctx, ctx.Repo.Repository.ID, labelNames) + repoLabelIDs, err := issues_model.GetLabelIDsInRepoByNames(ctx, ctx.Repo.Repository.ID, labelNames) if err != nil { ctx.Error(http.StatusInternalServerError, "GetLabelIDsInRepoByNames", err) return nil, nil, err } + labelIDs = append(labelIDs, repoLabelIDs...) + if ctx.Repo.Owner.IsOrganization() { + orgLabelIDs, err := issues_model.GetLabelIDsInOrgByNames(ctx, ctx.Repo.Owner.ID, labelNames) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetLabelIDsInOrgByNames", err) + return nil, nil, err + } + labelIDs = append(labelIDs, orgLabelIDs...) + } } labels, err := issues_model.GetLabelsByIDs(ctx, labelIDs, "id", "repo_id", "org_id", "name", "exclusive") diff --git a/tests/integration/api_issue_label_test.go b/tests/integration/api_issue_label_test.go index 29da419380..ebcf29a13d 100644 --- a/tests/integration/api_issue_label_test.go +++ b/tests/integration/api_issue_label_test.go @@ -120,27 +120,33 @@ func TestAPIAddIssueLabels(t *testing.T) { func TestAPIAddIssueLabelsWithLabelNames(t *testing.T) { require.NoError(t, unittest.LoadFixtures()) - repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{RepoID: repo.ID}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) + issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 6, RepoID: repo.ID}) owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}) + repoLabel := unittest.AssertExistsAndLoadBean(t, &issues_model.Label{ID: 10, RepoID: repo.ID}) + orgLabel := unittest.AssertExistsAndLoadBean(t, &issues_model.Label{ID: 4, OrgID: owner.ID}) - session := loginUser(t, owner.Name) - token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteIssue) - urlStr := fmt.Sprintf("/api/v1/repos/%s/%s/issues/%d/labels", - repo.OwnerName, repo.Name, issue.Index) + user1Session := loginUser(t, "user1") + token := getTokenForLoggedInUser(t, user1Session, auth_model.AccessTokenScopeWriteIssue) + + // add the org label and the repo label to the issue + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s/issues/%d/labels", owner.Name, repo.Name, issue.Index) req := NewRequestWithJSON(t, "POST", urlStr, &api.IssueLabelsOption{ - Labels: []any{"label1", "label2"}, + Labels: []any{repoLabel.Name, orgLabel.Name}, }).AddTokenAuth(token) resp := MakeRequest(t, req, http.StatusOK) var apiLabels []*api.Label DecodeJSON(t, resp, &apiLabels) assert.Len(t, apiLabels, unittest.GetCount(t, &issues_model.IssueLabel{IssueID: issue.ID})) - var apiLabelNames []string for _, label := range apiLabels { apiLabelNames = append(apiLabelNames, label.Name) } - assert.ElementsMatch(t, apiLabelNames, []string{"label1", "label2"}) + assert.ElementsMatch(t, apiLabelNames, []string{repoLabel.Name, orgLabel.Name}) + + // delete labels + req = NewRequest(t, "DELETE", urlStr).AddTokenAuth(token) + MakeRequest(t, req, http.StatusNoContent) } func TestAPIAddIssueLabelsAutoDate(t *testing.T) { From b18dcd69f233e032f0564efed7b257180de393e0 Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Fri, 27 Dec 2024 06:10:30 +0800 Subject: [PATCH 4/5] Improve Actions test (#32883) This PR adds a mock runner to test more actions features. (cherry picked from commit df98452c0de9d01338f00aa5d85757623523b1fc) --- tests/integration/actions_job_test.go | 410 +++++++++++++++++++++++ tests/integration/actions_log_test.go | 159 +++++++++ tests/integration/actions_runner_test.go | 157 +++++++++ 3 files changed, 726 insertions(+) create mode 100644 tests/integration/actions_job_test.go create mode 100644 tests/integration/actions_log_test.go create mode 100644 tests/integration/actions_runner_test.go diff --git a/tests/integration/actions_job_test.go b/tests/integration/actions_job_test.go new file mode 100644 index 0000000000..e13277678d --- /dev/null +++ b/tests/integration/actions_job_test.go @@ -0,0 +1,410 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/url" + "testing" + "time" + + actions_model "code.gitea.io/gitea/models/actions" + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + api "code.gitea.io/gitea/modules/structs" + + runnerv1 "code.gitea.io/actions-proto-go/runner/v1" + "github.com/stretchr/testify/assert" +) + +func TestJobWithNeeds(t *testing.T) { + testCases := []struct { + treePath string + fileContent string + outcomes map[string]*mockTaskOutcome + expectedStatuses map[string]string + }{ + { + treePath: ".gitea/workflows/job-with-needs.yml", + fileContent: `name: job-with-needs +on: + push: + paths: + - '.gitea/workflows/job-with-needs.yml' +jobs: + job1: + runs-on: ubuntu-latest + steps: + - run: echo job1 + job2: + runs-on: ubuntu-latest + needs: [job1] + steps: + - run: echo job2 +`, + outcomes: map[string]*mockTaskOutcome{ + "job1": { + result: runnerv1.Result_RESULT_SUCCESS, + }, + "job2": { + result: runnerv1.Result_RESULT_SUCCESS, + }, + }, + expectedStatuses: map[string]string{ + "job1": actions_model.StatusSuccess.String(), + "job2": actions_model.StatusSuccess.String(), + }, + }, + { + treePath: ".gitea/workflows/job-with-needs-fail.yml", + fileContent: `name: job-with-needs-fail +on: + push: + paths: + - '.gitea/workflows/job-with-needs-fail.yml' +jobs: + job1: + runs-on: ubuntu-latest + steps: + - run: echo job1 + job2: + runs-on: ubuntu-latest + needs: [job1] + steps: + - run: echo job2 +`, + outcomes: map[string]*mockTaskOutcome{ + "job1": { + result: runnerv1.Result_RESULT_FAILURE, + }, + }, + expectedStatuses: map[string]string{ + "job1": actions_model.StatusFailure.String(), + "job2": actions_model.StatusSkipped.String(), + }, + }, + { + treePath: ".gitea/workflows/job-with-needs-fail-if.yml", + fileContent: `name: job-with-needs-fail-if +on: + push: + paths: + - '.gitea/workflows/job-with-needs-fail-if.yml' +jobs: + job1: + runs-on: ubuntu-latest + steps: + - run: echo job1 + job2: + runs-on: ubuntu-latest + if: ${{ always() }} + needs: [job1] + steps: + - run: echo job2 +`, + outcomes: map[string]*mockTaskOutcome{ + "job1": { + result: runnerv1.Result_RESULT_FAILURE, + }, + "job2": { + result: runnerv1.Result_RESULT_SUCCESS, + }, + }, + expectedStatuses: map[string]string{ + "job1": actions_model.StatusFailure.String(), + "job2": actions_model.StatusSuccess.String(), + }, + }, + } + onGiteaRun(t, func(t *testing.T, u *url.URL) { + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + session := loginUser(t, user2.Name) + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) + + apiRepo := createActionsTestRepo(t, token, "actions-jobs-with-needs", false) + runner := newMockRunner() + runner.registerAsRepoRunner(t, user2.Name, apiRepo.Name, "mock-runner", []string{"ubuntu-latest"}) + + for _, tc := range testCases { + t.Run(fmt.Sprintf("test %s", tc.treePath), func(t *testing.T) { + // create the workflow file + opts := getWorkflowCreateFileOptions(user2, apiRepo.DefaultBranch, fmt.Sprintf("create %s", tc.treePath), tc.fileContent) + fileResp := createWorkflowFile(t, token, user2.Name, apiRepo.Name, tc.treePath, opts) + + // fetch and execute task + for i := 0; i < len(tc.outcomes); i++ { + task := runner.fetchTask(t) + jobName := getTaskJobNameByTaskID(t, token, user2.Name, apiRepo.Name, task.Id) + outcome := tc.outcomes[jobName] + assert.NotNil(t, outcome) + runner.execTask(t, task, outcome) + } + + // check result + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/actions/tasks", user2.Name, apiRepo.Name)). + AddTokenAuth(token) + resp := MakeRequest(t, req, http.StatusOK) + var actionTaskRespAfter api.ActionTaskResponse + DecodeJSON(t, resp, &actionTaskRespAfter) + for _, apiTask := range actionTaskRespAfter.Entries { + if apiTask.HeadSHA != fileResp.Commit.SHA { + continue + } + status := apiTask.Status + assert.Equal(t, status, tc.expectedStatuses[apiTask.Name]) + } + }) + } + + httpContext := NewAPITestContext(t, user2.Name, apiRepo.Name, auth_model.AccessTokenScopeWriteRepository) + doAPIDeleteRepository(httpContext)(t) + }) +} + +func TestJobNeedsMatrix(t *testing.T) { + testCases := []struct { + treePath string + fileContent string + outcomes map[string]*mockTaskOutcome + expectedTaskNeeds map[string]*runnerv1.TaskNeed // jobID => TaskNeed + }{ + { + treePath: ".gitea/workflows/jobs-outputs-with-matrix.yml", + fileContent: `name: jobs-outputs-with-matrix +on: + push: + paths: + - '.gitea/workflows/jobs-outputs-with-matrix.yml' +jobs: + job1: + runs-on: ubuntu-latest + outputs: + output_1: ${{ steps.gen_output.outputs.output_1 }} + output_2: ${{ steps.gen_output.outputs.output_2 }} + output_3: ${{ steps.gen_output.outputs.output_3 }} + strategy: + matrix: + version: [1, 2, 3] + steps: + - name: Generate output + id: gen_output + run: | + version="${{ matrix.version }}" + echo "output_${version}=${version}" >> "$GITHUB_OUTPUT" + job2: + runs-on: ubuntu-latest + needs: [job1] + steps: + - run: echo '${{ toJSON(needs.job1.outputs) }}' +`, + outcomes: map[string]*mockTaskOutcome{ + "job1 (1)": { + result: runnerv1.Result_RESULT_SUCCESS, + outputs: map[string]string{ + "output_1": "1", + "output_2": "", + "output_3": "", + }, + }, + "job1 (2)": { + result: runnerv1.Result_RESULT_SUCCESS, + outputs: map[string]string{ + "output_1": "", + "output_2": "2", + "output_3": "", + }, + }, + "job1 (3)": { + result: runnerv1.Result_RESULT_SUCCESS, + outputs: map[string]string{ + "output_1": "", + "output_2": "", + "output_3": "3", + }, + }, + }, + expectedTaskNeeds: map[string]*runnerv1.TaskNeed{ + "job1": { + Result: runnerv1.Result_RESULT_SUCCESS, + Outputs: map[string]string{ + "output_1": "1", + "output_2": "2", + "output_3": "3", + }, + }, + }, + }, + { + treePath: ".gitea/workflows/jobs-outputs-with-matrix-failure.yml", + fileContent: `name: jobs-outputs-with-matrix-failure +on: + push: + paths: + - '.gitea/workflows/jobs-outputs-with-matrix-failure.yml' +jobs: + job1: + runs-on: ubuntu-latest + outputs: + output_1: ${{ steps.gen_output.outputs.output_1 }} + output_2: ${{ steps.gen_output.outputs.output_2 }} + output_3: ${{ steps.gen_output.outputs.output_3 }} + strategy: + matrix: + version: [1, 2, 3] + steps: + - name: Generate output + id: gen_output + run: | + version="${{ matrix.version }}" + echo "output_${version}=${version}" >> "$GITHUB_OUTPUT" + job2: + runs-on: ubuntu-latest + if: ${{ always() }} + needs: [job1] + steps: + - run: echo '${{ toJSON(needs.job1.outputs) }}' +`, + outcomes: map[string]*mockTaskOutcome{ + "job1 (1)": { + result: runnerv1.Result_RESULT_SUCCESS, + outputs: map[string]string{ + "output_1": "1", + "output_2": "", + "output_3": "", + }, + }, + "job1 (2)": { + result: runnerv1.Result_RESULT_FAILURE, + outputs: map[string]string{ + "output_1": "", + "output_2": "", + "output_3": "", + }, + }, + "job1 (3)": { + result: runnerv1.Result_RESULT_SUCCESS, + outputs: map[string]string{ + "output_1": "", + "output_2": "", + "output_3": "3", + }, + }, + }, + expectedTaskNeeds: map[string]*runnerv1.TaskNeed{ + "job1": { + Result: runnerv1.Result_RESULT_FAILURE, + Outputs: map[string]string{ + "output_1": "1", + "output_2": "", + "output_3": "3", + }, + }, + }, + }, + } + onGiteaRun(t, func(t *testing.T, u *url.URL) { + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + session := loginUser(t, user2.Name) + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) + + apiRepo := createActionsTestRepo(t, token, "actions-jobs-outputs-with-matrix", false) + runner := newMockRunner() + runner.registerAsRepoRunner(t, user2.Name, apiRepo.Name, "mock-runner", []string{"ubuntu-latest"}) + + for _, tc := range testCases { + t.Run(fmt.Sprintf("test %s", tc.treePath), func(t *testing.T) { + opts := getWorkflowCreateFileOptions(user2, apiRepo.DefaultBranch, fmt.Sprintf("create %s", tc.treePath), tc.fileContent) + createWorkflowFile(t, token, user2.Name, apiRepo.Name, tc.treePath, opts) + + for i := 0; i < len(tc.outcomes); i++ { + task := runner.fetchTask(t) + jobName := getTaskJobNameByTaskID(t, token, user2.Name, apiRepo.Name, task.Id) + outcome := tc.outcomes[jobName] + assert.NotNil(t, outcome) + runner.execTask(t, task, outcome) + } + + task := runner.fetchTask(t) + actualTaskNeeds := task.Needs + assert.Len(t, actualTaskNeeds, len(tc.expectedTaskNeeds)) + for jobID, tn := range tc.expectedTaskNeeds { + actualNeed := actualTaskNeeds[jobID] + assert.Equal(t, tn.Result, actualNeed.Result) + assert.Len(t, actualNeed.Outputs, len(tn.Outputs)) + for outputKey, outputValue := range tn.Outputs { + assert.Equal(t, outputValue, actualNeed.Outputs[outputKey]) + } + } + }) + } + + httpContext := NewAPITestContext(t, user2.Name, apiRepo.Name, auth_model.AccessTokenScopeWriteRepository) + doAPIDeleteRepository(httpContext)(t) + }) +} + +func createActionsTestRepo(t *testing.T, authToken, repoName string, isPrivate bool) *api.Repository { + req := NewRequestWithJSON(t, "POST", "/api/v1/user/repos", &api.CreateRepoOption{ + Name: repoName, + Private: isPrivate, + Readme: "Default", + AutoInit: true, + DefaultBranch: "main", + }).AddTokenAuth(authToken) + resp := MakeRequest(t, req, http.StatusCreated) + var apiRepo api.Repository + DecodeJSON(t, resp, &apiRepo) + return &apiRepo +} + +func getWorkflowCreateFileOptions(u *user_model.User, branch, msg, content string) *api.CreateFileOptions { + return &api.CreateFileOptions{ + FileOptions: api.FileOptions{ + BranchName: branch, + Message: msg, + Author: api.Identity{ + Name: u.Name, + Email: u.Email, + }, + Committer: api.Identity{ + Name: u.Name, + Email: u.Email, + }, + Dates: api.CommitDateOptions{ + Author: time.Now(), + Committer: time.Now(), + }, + }, + ContentBase64: base64.StdEncoding.EncodeToString([]byte(content)), + } +} + +func createWorkflowFile(t *testing.T, authToken, ownerName, repoName, treePath string, opts *api.CreateFileOptions) *api.FileResponse { + req := NewRequestWithJSON(t, "POST", fmt.Sprintf("/api/v1/repos/%s/%s/contents/%s", ownerName, repoName, treePath), opts). + AddTokenAuth(authToken) + resp := MakeRequest(t, req, http.StatusCreated) + var fileResponse api.FileResponse + DecodeJSON(t, resp, &fileResponse) + return &fileResponse +} + +// getTaskJobNameByTaskID get the job name of the task by task ID +// there is currently not an API for querying a task by ID so we have to list all the tasks +func getTaskJobNameByTaskID(t *testing.T, authToken, ownerName, repoName string, taskID int64) string { + // FIXME: we may need to query several pages + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/actions/tasks", ownerName, repoName)). + AddTokenAuth(authToken) + resp := MakeRequest(t, req, http.StatusOK) + var taskRespBefore api.ActionTaskResponse + DecodeJSON(t, resp, &taskRespBefore) + for _, apiTask := range taskRespBefore.Entries { + if apiTask.ID == taskID { + return apiTask.Name + } + } + return "" +} diff --git a/tests/integration/actions_log_test.go b/tests/integration/actions_log_test.go new file mode 100644 index 0000000000..fd055fc4c4 --- /dev/null +++ b/tests/integration/actions_log_test.go @@ -0,0 +1,159 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "fmt" + "net/http" + "net/url" + "strings" + "testing" + "time" + + auth_model "code.gitea.io/gitea/models/auth" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/test" + + runnerv1 "code.gitea.io/actions-proto-go/runner/v1" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestDownloadTaskLogs(t *testing.T) { + now := time.Now() + testCases := []struct { + treePath string + fileContent string + outcome *mockTaskOutcome + zstdEnabled bool + }{ + { + treePath: ".gitea/workflows/download-task-logs-zstd.yml", + fileContent: `name: download-task-logs-zstd +on: + push: + paths: + - '.gitea/workflows/download-task-logs-zstd.yml' +jobs: + job1: + runs-on: ubuntu-latest + steps: + - run: echo job1 with zstd enabled +`, + outcome: &mockTaskOutcome{ + result: runnerv1.Result_RESULT_SUCCESS, + logRows: []*runnerv1.LogRow{ + { + Time: timestamppb.New(now.Add(1 * time.Second)), + Content: " \U0001F433 docker create image", + }, + { + Time: timestamppb.New(now.Add(2 * time.Second)), + Content: "job1 zstd enabled", + }, + { + Time: timestamppb.New(now.Add(3 * time.Second)), + Content: "\U0001F3C1 Job succeeded", + }, + }, + }, + zstdEnabled: true, + }, + { + treePath: ".gitea/workflows/download-task-logs-no-zstd.yml", + fileContent: `name: download-task-logs-no-zstd +on: + push: + paths: + - '.gitea/workflows/download-task-logs-no-zstd.yml' +jobs: + job1: + runs-on: ubuntu-latest + steps: + - run: echo job1 with zstd disabled +`, + outcome: &mockTaskOutcome{ + result: runnerv1.Result_RESULT_SUCCESS, + logRows: []*runnerv1.LogRow{ + { + Time: timestamppb.New(now.Add(4 * time.Second)), + Content: " \U0001F433 docker create image", + }, + { + Time: timestamppb.New(now.Add(5 * time.Second)), + Content: "job1 zstd disabled", + }, + { + Time: timestamppb.New(now.Add(6 * time.Second)), + Content: "\U0001F3C1 Job succeeded", + }, + }, + }, + zstdEnabled: false, + }, + } + onGiteaRun(t, func(t *testing.T, u *url.URL) { + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + session := loginUser(t, user2.Name) + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) + + apiRepo := createActionsTestRepo(t, token, "actions-download-task-logs", false) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: apiRepo.ID}) + runner := newMockRunner() + runner.registerAsRepoRunner(t, user2.Name, repo.Name, "mock-runner", []string{"ubuntu-latest"}) + + for _, tc := range testCases { + t.Run(fmt.Sprintf("test %s", tc.treePath), func(t *testing.T) { + var resetFunc func() + if tc.zstdEnabled { + resetFunc = test.MockVariableValue(&setting.Actions.LogCompression, "zstd") + assert.True(t, setting.Actions.LogCompression.IsZstd()) + } else { + resetFunc = test.MockVariableValue(&setting.Actions.LogCompression, "none") + assert.False(t, setting.Actions.LogCompression.IsZstd()) + } + + // create the workflow file + opts := getWorkflowCreateFileOptions(user2, repo.DefaultBranch, fmt.Sprintf("create %s", tc.treePath), tc.fileContent) + createWorkflowFile(t, token, user2.Name, repo.Name, tc.treePath, opts) + + // fetch and execute task + task := runner.fetchTask(t) + runner.execTask(t, task, tc.outcome) + + // check whether the log file exists + logFileName := fmt.Sprintf("%s/%02x/%d.log", repo.FullName(), task.Id%256, task.Id) + if setting.Actions.LogCompression.IsZstd() { + logFileName += ".zst" + } + _, err := storage.Actions.Stat(logFileName) + assert.NoError(t, err) + + // download task logs and check content + runIndex := task.Context.GetFields()["run_number"].GetStringValue() + req := NewRequest(t, "GET", fmt.Sprintf("/%s/%s/actions/runs/%s/jobs/0/logs", user2.Name, repo.Name, runIndex)). + AddTokenAuth(token) + resp := MakeRequest(t, req, http.StatusOK) + logTextLines := strings.Split(strings.TrimSpace(resp.Body.String()), "\n") + assert.Len(t, logTextLines, len(tc.outcome.logRows)) + for idx, lr := range tc.outcome.logRows { + assert.Equal( + t, + fmt.Sprintf("%s %s", lr.Time.AsTime().Format("2006-01-02T15:04:05.0000000Z07:00"), lr.Content), + logTextLines[idx], + ) + } + + resetFunc() + }) + } + + httpContext := NewAPITestContext(t, user2.Name, repo.Name, auth_model.AccessTokenScopeWriteRepository) + doAPIDeleteRepository(httpContext)(t) + }) +} diff --git a/tests/integration/actions_runner_test.go b/tests/integration/actions_runner_test.go new file mode 100644 index 0000000000..355ea1705e --- /dev/null +++ b/tests/integration/actions_runner_test.go @@ -0,0 +1,157 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/modules/setting" + + pingv1 "code.gitea.io/actions-proto-go/ping/v1" + "code.gitea.io/actions-proto-go/ping/v1/pingv1connect" + runnerv1 "code.gitea.io/actions-proto-go/runner/v1" + "code.gitea.io/actions-proto-go/runner/v1/runnerv1connect" + "connectrpc.com/connect" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type mockRunner struct { + client *mockRunnerClient +} + +type mockRunnerClient struct { + pingServiceClient pingv1connect.PingServiceClient + runnerServiceClient runnerv1connect.RunnerServiceClient +} + +func newMockRunner() *mockRunner { + client := newMockRunnerClient("", "") + return &mockRunner{client: client} +} + +func newMockRunnerClient(uuid, token string) *mockRunnerClient { + baseURL := fmt.Sprintf("%sapi/actions", setting.AppURL) + + opt := connect.WithInterceptors(connect.UnaryInterceptorFunc(func(next connect.UnaryFunc) connect.UnaryFunc { + return func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) { + if uuid != "" { + req.Header().Set("x-runner-uuid", uuid) + } + if token != "" { + req.Header().Set("x-runner-token", token) + } + return next(ctx, req) + } + })) + + client := &mockRunnerClient{ + pingServiceClient: pingv1connect.NewPingServiceClient(http.DefaultClient, baseURL, opt), + runnerServiceClient: runnerv1connect.NewRunnerServiceClient(http.DefaultClient, baseURL, opt), + } + + return client +} + +func (r *mockRunner) doPing(t *testing.T) { + resp, err := r.client.pingServiceClient.Ping(context.Background(), connect.NewRequest(&pingv1.PingRequest{ + Data: "mock-runner", + })) + assert.NoError(t, err) + assert.Equal(t, "Hello, mock-runner!", resp.Msg.Data) +} + +func (r *mockRunner) doRegister(t *testing.T, name, token string, labels []string) { + r.doPing(t) + resp, err := r.client.runnerServiceClient.Register(context.Background(), connect.NewRequest(&runnerv1.RegisterRequest{ + Name: name, + Token: token, + Version: "mock-runner-version", + Labels: labels, + })) + assert.NoError(t, err) + r.client = newMockRunnerClient(resp.Msg.Runner.Uuid, resp.Msg.Runner.Token) +} + +func (r *mockRunner) registerAsRepoRunner(t *testing.T, ownerName, repoName, runnerName string, labels []string) { + session := loginUser(t, ownerName) + token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository) + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/actions/runners/registration-token", ownerName, repoName)).AddTokenAuth(token) + resp := MakeRequest(t, req, http.StatusOK) + var registrationToken struct { + Token string `json:"token"` + } + DecodeJSON(t, resp, ®istrationToken) + r.doRegister(t, runnerName, registrationToken.Token, labels) +} + +func (r *mockRunner) fetchTask(t *testing.T, timeout ...time.Duration) *runnerv1.Task { + fetchTimeout := 10 * time.Second + if len(timeout) > 0 { + fetchTimeout = timeout[0] + } + ddl := time.Now().Add(fetchTimeout) + var task *runnerv1.Task + for time.Now().Before(ddl) { + resp, err := r.client.runnerServiceClient.FetchTask(context.Background(), connect.NewRequest(&runnerv1.FetchTaskRequest{ + TasksVersion: 0, + })) + assert.NoError(t, err) + if resp.Msg.Task != nil { + task = resp.Msg.Task + break + } + time.Sleep(time.Second) + } + assert.NotNil(t, task, "failed to fetch a task") + return task +} + +type mockTaskOutcome struct { + result runnerv1.Result + outputs map[string]string + logRows []*runnerv1.LogRow + execTime time.Duration +} + +func (r *mockRunner) execTask(t *testing.T, task *runnerv1.Task, outcome *mockTaskOutcome) { + for idx, lr := range outcome.logRows { + resp, err := r.client.runnerServiceClient.UpdateLog(context.Background(), connect.NewRequest(&runnerv1.UpdateLogRequest{ + TaskId: task.Id, + Index: int64(idx), + Rows: []*runnerv1.LogRow{lr}, + NoMore: idx == len(outcome.logRows)-1, + })) + assert.NoError(t, err) + assert.EqualValues(t, idx+1, resp.Msg.AckIndex) + } + sentOutputKeys := make([]string, 0, len(outcome.outputs)) + for outputKey, outputValue := range outcome.outputs { + resp, err := r.client.runnerServiceClient.UpdateTask(context.Background(), connect.NewRequest(&runnerv1.UpdateTaskRequest{ + State: &runnerv1.TaskState{ + Id: task.Id, + Result: runnerv1.Result_RESULT_UNSPECIFIED, + }, + Outputs: map[string]string{outputKey: outputValue}, + })) + assert.NoError(t, err) + sentOutputKeys = append(sentOutputKeys, outputKey) + assert.ElementsMatch(t, sentOutputKeys, resp.Msg.SentOutputs) + } + time.Sleep(outcome.execTime) + resp, err := r.client.runnerServiceClient.UpdateTask(context.Background(), connect.NewRequest(&runnerv1.UpdateTaskRequest{ + State: &runnerv1.TaskState{ + Id: task.Id, + Result: outcome.result, + StoppedAt: timestamppb.Now(), + }, + })) + assert.NoError(t, err) + assert.Equal(t, outcome.result, resp.Msg.State.Result) +} From 37f05617a5fd7582b535d3a91069445de51f91da Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Sat, 28 Dec 2024 19:38:11 +0000 Subject: [PATCH 5/5] Improve Actions test (#32883) (followup) --- tests/integration/actions_job_test.go | 21 ++++++++++++++------- tests/integration/actions_log_test.go | 10 +++++++--- tests/integration/actions_runner_test.go | 19 ++++++++++++------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/tests/integration/actions_job_test.go b/tests/integration/actions_job_test.go index e13277678d..5af3519b93 100644 --- a/tests/integration/actions_job_test.go +++ b/tests/integration/actions_job_test.go @@ -15,6 +15,7 @@ import ( auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" runnerv1 "code.gitea.io/actions-proto-go/runner/v1" @@ -22,6 +23,9 @@ import ( ) func TestJobWithNeeds(t *testing.T) { + if !setting.Database.Type.IsSQLite3() { + t.Skip() + } testCases := []struct { treePath string fileContent string @@ -31,7 +35,7 @@ func TestJobWithNeeds(t *testing.T) { { treePath: ".gitea/workflows/job-with-needs.yml", fileContent: `name: job-with-needs -on: +on: push: paths: - '.gitea/workflows/job-with-needs.yml' @@ -62,7 +66,7 @@ jobs: { treePath: ".gitea/workflows/job-with-needs-fail.yml", fileContent: `name: job-with-needs-fail -on: +on: push: paths: - '.gitea/workflows/job-with-needs-fail.yml' @@ -90,7 +94,7 @@ jobs: { treePath: ".gitea/workflows/job-with-needs-fail-if.yml", fileContent: `name: job-with-needs-fail-if -on: +on: push: paths: - '.gitea/workflows/job-with-needs-fail-if.yml' @@ -166,6 +170,9 @@ jobs: } func TestJobNeedsMatrix(t *testing.T) { + if !setting.Database.Type.IsSQLite3() { + t.Skip() + } testCases := []struct { treePath string fileContent string @@ -175,7 +182,7 @@ func TestJobNeedsMatrix(t *testing.T) { { treePath: ".gitea/workflows/jobs-outputs-with-matrix.yml", fileContent: `name: jobs-outputs-with-matrix -on: +on: push: paths: - '.gitea/workflows/jobs-outputs-with-matrix.yml' @@ -194,7 +201,7 @@ jobs: id: gen_output run: | version="${{ matrix.version }}" - echo "output_${version}=${version}" >> "$GITHUB_OUTPUT" + echo "output_${version}=${version}" >> "$GITHUB_OUTPUT" job2: runs-on: ubuntu-latest needs: [job1] @@ -241,7 +248,7 @@ jobs: { treePath: ".gitea/workflows/jobs-outputs-with-matrix-failure.yml", fileContent: `name: jobs-outputs-with-matrix-failure -on: +on: push: paths: - '.gitea/workflows/jobs-outputs-with-matrix-failure.yml' @@ -260,7 +267,7 @@ jobs: id: gen_output run: | version="${{ matrix.version }}" - echo "output_${version}=${version}" >> "$GITHUB_OUTPUT" + echo "output_${version}=${version}" >> "$GITHUB_OUTPUT" job2: runs-on: ubuntu-latest if: ${{ always() }} diff --git a/tests/integration/actions_log_test.go b/tests/integration/actions_log_test.go index fd055fc4c4..d66eada950 100644 --- a/tests/integration/actions_log_test.go +++ b/tests/integration/actions_log_test.go @@ -21,10 +21,14 @@ import ( runnerv1 "code.gitea.io/actions-proto-go/runner/v1" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/timestamppb" ) func TestDownloadTaskLogs(t *testing.T) { + if !setting.Database.Type.IsSQLite3() { + t.Skip() + } now := time.Now() testCases := []struct { treePath string @@ -35,7 +39,7 @@ func TestDownloadTaskLogs(t *testing.T) { { treePath: ".gitea/workflows/download-task-logs-zstd.yml", fileContent: `name: download-task-logs-zstd -on: +on: push: paths: - '.gitea/workflows/download-task-logs-zstd.yml' @@ -67,7 +71,7 @@ jobs: { treePath: ".gitea/workflows/download-task-logs-no-zstd.yml", fileContent: `name: download-task-logs-no-zstd -on: +on: push: paths: - '.gitea/workflows/download-task-logs-no-zstd.yml' @@ -132,7 +136,7 @@ jobs: logFileName += ".zst" } _, err := storage.Actions.Stat(logFileName) - assert.NoError(t, err) + require.NoError(t, err) // download task logs and check content runIndex := task.Context.GetFields()["run_number"].GetStringValue() diff --git a/tests/integration/actions_runner_test.go b/tests/integration/actions_runner_test.go index 355ea1705e..0ffd97a208 100644 --- a/tests/integration/actions_runner_test.go +++ b/tests/integration/actions_runner_test.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/actions-proto-go/runner/v1/runnerv1connect" "connectrpc.com/connect" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -63,8 +64,8 @@ func (r *mockRunner) doPing(t *testing.T) { resp, err := r.client.pingServiceClient.Ping(context.Background(), connect.NewRequest(&pingv1.PingRequest{ Data: "mock-runner", })) - assert.NoError(t, err) - assert.Equal(t, "Hello, mock-runner!", resp.Msg.Data) + require.NoError(t, err) + require.Equal(t, "Hello, mock-runner!", resp.Msg.Data) } func (r *mockRunner) doRegister(t *testing.T, name, token string, labels []string) { @@ -75,11 +76,15 @@ func (r *mockRunner) doRegister(t *testing.T, name, token string, labels []strin Version: "mock-runner-version", Labels: labels, })) - assert.NoError(t, err) + require.NoError(t, err) r.client = newMockRunnerClient(resp.Msg.Runner.Uuid, resp.Msg.Runner.Token) } func (r *mockRunner) registerAsRepoRunner(t *testing.T, ownerName, repoName, runnerName string, labels []string) { + if !setting.Database.Type.IsSQLite3() { + // registering a mock runner when using a database other than SQLite leaves leftovers + t.FailNow() + } session := loginUser(t, ownerName) token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository) req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/actions/runners/registration-token", ownerName, repoName)).AddTokenAuth(token) @@ -102,7 +107,7 @@ func (r *mockRunner) fetchTask(t *testing.T, timeout ...time.Duration) *runnerv1 resp, err := r.client.runnerServiceClient.FetchTask(context.Background(), connect.NewRequest(&runnerv1.FetchTaskRequest{ TasksVersion: 0, })) - assert.NoError(t, err) + require.NoError(t, err) if resp.Msg.Task != nil { task = resp.Msg.Task break @@ -128,7 +133,7 @@ func (r *mockRunner) execTask(t *testing.T, task *runnerv1.Task, outcome *mockTa Rows: []*runnerv1.LogRow{lr}, NoMore: idx == len(outcome.logRows)-1, })) - assert.NoError(t, err) + require.NoError(t, err) assert.EqualValues(t, idx+1, resp.Msg.AckIndex) } sentOutputKeys := make([]string, 0, len(outcome.outputs)) @@ -140,7 +145,7 @@ func (r *mockRunner) execTask(t *testing.T, task *runnerv1.Task, outcome *mockTa }, Outputs: map[string]string{outputKey: outputValue}, })) - assert.NoError(t, err) + require.NoError(t, err) sentOutputKeys = append(sentOutputKeys, outputKey) assert.ElementsMatch(t, sentOutputKeys, resp.Msg.SentOutputs) } @@ -152,6 +157,6 @@ func (r *mockRunner) execTask(t *testing.T, task *runnerv1.Task, outcome *mockTa StoppedAt: timestamppb.Now(), }, })) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, outcome.result, resp.Msg.State.Result) }