From 5149d3eda47643e8c9d73f8435469a325a7acd69 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Sat, 30 Nov 2024 13:57:59 +0100 Subject: [PATCH] Add pipeline purge command to cli (#4470) --- cli/pipeline/kill.go | 2 +- cli/pipeline/list.go | 4 +- cli/pipeline/list_test.go | 4 +- cli/pipeline/pipeline.go | 1 + cli/pipeline/purge.go | 156 +++++++++++++++++++++++ cli/pipeline/purge_test.go | 105 +++++++++++++++ woodpecker-go/woodpecker/interface.go | 5 +- woodpecker-go/woodpecker/mocks/client.go | 6 +- woodpecker-go/woodpecker/repo.go | 14 +- 9 files changed, 279 insertions(+), 18 deletions(-) create mode 100644 cli/pipeline/purge.go create mode 100644 cli/pipeline/purge_test.go diff --git a/cli/pipeline/kill.go b/cli/pipeline/kill.go index 872719765..839d7c75e 100644 --- a/cli/pipeline/kill.go +++ b/cli/pipeline/kill.go @@ -48,7 +48,7 @@ func pipelineKill(ctx context.Context, c *cli.Command) (err error) { return err } - err = client.PipelineKill(repoID, number) + err = client.PipelineDelete(repoID, number) if err != nil { return err } diff --git a/cli/pipeline/list.go b/cli/pipeline/list.go index ceadc9777..bfd24981b 100644 --- a/cli/pipeline/list.go +++ b/cli/pipeline/list.go @@ -77,14 +77,14 @@ func List(ctx context.Context, c *cli.Command) error { if err != nil { return err } - resources, err := pipelineList(ctx, c, client) + resources, err := pipelineList(c, client) if err != nil { return err } return pipelineOutput(c, resources) } -func pipelineList(_ context.Context, c *cli.Command, client woodpecker.Client) ([]woodpecker.Pipeline, error) { +func pipelineList(c *cli.Command, client woodpecker.Client) ([]woodpecker.Pipeline, error) { resources := make([]woodpecker.Pipeline, 0) repoIDOrFullName := c.Args().First() diff --git a/cli/pipeline/list_test.go b/cli/pipeline/list_test.go index 20ebefb30..e73bffe57 100644 --- a/cli/pipeline/list_test.go +++ b/cli/pipeline/list_test.go @@ -112,8 +112,8 @@ func TestPipelineList(t *testing.T) { command := buildPipelineListCmd() command.Writer = io.Discard - command.Action = func(ctx context.Context, c *cli.Command) error { - pipelines, err := pipelineList(ctx, c, mockClient) + command.Action = func(_ context.Context, c *cli.Command) error { + pipelines, err := pipelineList(c, mockClient) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) return nil diff --git a/cli/pipeline/pipeline.go b/cli/pipeline/pipeline.go index abc892600..866158e88 100644 --- a/cli/pipeline/pipeline.go +++ b/cli/pipeline/pipeline.go @@ -46,6 +46,7 @@ var Command = &cli.Command{ pipelineCreateCmd, log.Command, deploy.Command, + pipelinePurgeCmd, }, } diff --git a/cli/pipeline/purge.go b/cli/pipeline/purge.go new file mode 100644 index 000000000..759b9cbf2 --- /dev/null +++ b/cli/pipeline/purge.go @@ -0,0 +1,156 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipeline + +import ( + "context" + "fmt" + "time" + + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v3" + + "go.woodpecker-ci.org/woodpecker/v2/cli/internal" + shared_utils "go.woodpecker-ci.org/woodpecker/v2/shared/utils" + "go.woodpecker-ci.org/woodpecker/v2/woodpecker-go/woodpecker" +) + +//nolint:mnd +var pipelinePurgeCmd = &cli.Command{ + Name: "purge", + Usage: "purge pipelines", + ArgsUsage: "", + Action: Purge, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "older-than", + Usage: "remove pipelines older than the specified time limit", + Required: true, + }, + &cli.IntFlag{ + Name: "keep-min", + Usage: "minimum number of pipelines to keep", + Value: 10, + }, + &cli.BoolFlag{ + Name: "dry-run", + Usage: "disable non-read api calls", + Value: false, + }, + }, +} + +func Purge(ctx context.Context, c *cli.Command) error { + client, err := internal.NewClient(ctx, c) + if err != nil { + return err + } + return pipelinePurge(c, client) +} + +func pipelinePurge(c *cli.Command, client woodpecker.Client) (err error) { + repoIDOrFullName := c.Args().First() + if len(repoIDOrFullName) == 0 { + return fmt.Errorf("missing required argument repo-id / repo-full-name") + } + repoID, err := internal.ParseRepo(client, repoIDOrFullName) + if err != nil { + return fmt.Errorf("invalid repo '%s': %w", repoIDOrFullName, err) + } + + olderThan := c.String("older-than") + keepMin := c.Int("keep-min") + dryRun := c.Bool("dry-run") + + duration, err := time.ParseDuration(olderThan) + if err != nil { + return err + } + + var pipelinesKeep []*woodpecker.Pipeline + + if keepMin > 0 { + pipelinesKeep, err = fetchPipelinesToKeep(client, repoID, int(keepMin)) + if err != nil { + return err + } + } + + pipelines, err := fetchPipelines(client, repoID, duration) + if err != nil { + return err + } + + // Create a map of pipeline IDs to keep + keepMap := make(map[int64]struct{}) + for _, p := range pipelinesKeep { + keepMap[p.ID] = struct{}{} + } + + // Filter pipelines to only include those not in keepMap + var pipelinesToPurge []*woodpecker.Pipeline + for _, p := range pipelines { + if _, exists := keepMap[p.ID]; !exists { + pipelinesToPurge = append(pipelinesToPurge, p) + } + } + + msgPrefix := "" + if dryRun { + msgPrefix = "DRY-RUN: " + } + + for i, p := range pipelinesToPurge { + log.Debug().Msgf("%sprune %v/%v pipelines from repo '%v'", msgPrefix, i+1, len(pipelinesToPurge), repoIDOrFullName) + if dryRun { + continue + } + + err := client.PipelineDelete(repoID, p.ID) + if err != nil { + return err + } + } + + return nil +} + +func fetchPipelinesToKeep(client woodpecker.Client, repoID int64, keepMin int) ([]*woodpecker.Pipeline, error) { + if keepMin <= 0 { + return nil, nil + } + return shared_utils.Paginate(func(page int) ([]*woodpecker.Pipeline, error) { + return client.PipelineList(repoID, + woodpecker.PipelineListOptions{ + ListOptions: woodpecker.ListOptions{ + Page: page, + }, + }, + ) + }, keepMin) +} + +func fetchPipelines(client woodpecker.Client, repoID int64, duration time.Duration) ([]*woodpecker.Pipeline, error) { + return shared_utils.Paginate(func(page int) ([]*woodpecker.Pipeline, error) { + return client.PipelineList(repoID, + woodpecker.PipelineListOptions{ + ListOptions: woodpecker.ListOptions{ + Page: page, + }, + After: time.Now().Add(-duration), + }, + ) + }, -1) +} diff --git a/cli/pipeline/purge_test.go b/cli/pipeline/purge_test.go new file mode 100644 index 000000000..be185277b --- /dev/null +++ b/cli/pipeline/purge_test.go @@ -0,0 +1,105 @@ +package pipeline + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/urfave/cli/v3" + + "go.woodpecker-ci.org/woodpecker/v2/woodpecker-go/woodpecker" + "go.woodpecker-ci.org/woodpecker/v2/woodpecker-go/woodpecker/mocks" +) + +func TestPipelinePurge(t *testing.T) { + tests := []struct { + name string + repoID int64 + args []string + pipelinesKeep []*woodpecker.Pipeline + pipelines []*woodpecker.Pipeline + wantDelete int + wantErr error + }{ + { + name: "success with no pipelines to purge", + repoID: 1, + args: []string{"purge", "--older-than", "1h", "repo/name"}, + pipelinesKeep: []*woodpecker.Pipeline{ + {ID: 1}, + }, + pipelines: []*woodpecker.Pipeline{}, + }, + { + name: "success with pipelines to purge", + repoID: 1, + args: []string{"purge", "--older-than", "1h", "repo/name"}, + pipelinesKeep: []*woodpecker.Pipeline{ + {ID: 1}, + }, + pipelines: []*woodpecker.Pipeline{ + {ID: 1}, + {ID: 2}, + {ID: 3}, + }, + wantDelete: 2, + }, + { + name: "error on invalid duration", + repoID: 1, + args: []string{"purge", "--older-than", "invalid", "repo/name"}, + wantErr: errors.New("time: invalid duration \"invalid\""), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockClient := mocks.NewClient(t) + mockClient.On("RepoLookup", mock.Anything).Maybe().Return(&woodpecker.Repo{ID: tt.repoID}, nil) + + mockClient.On("PipelineList", mock.Anything, mock.Anything).Return(func(_ int64, opt woodpecker.PipelineListOptions) ([]*woodpecker.Pipeline, error) { + // Return keep pipelines for first call + if opt.After.IsZero() { + if opt.Page == 1 { + return tt.pipelinesKeep, nil + } + return []*woodpecker.Pipeline{}, nil + } + + // Return pipelines to purge for calls with After filter + if !opt.After.IsZero() { + if opt.Page == 1 { + return tt.pipelines, nil + } + return []*woodpecker.Pipeline{}, nil + } + + return []*woodpecker.Pipeline{}, nil + }).Maybe() + + if tt.wantDelete > 0 { + mockClient.On("PipelineDelete", tt.repoID, mock.Anything).Return(nil).Times(tt.wantDelete) + } + + command := pipelinePurgeCmd + command.Writer = io.Discard + command.Action = func(_ context.Context, c *cli.Command) error { + err := pipelinePurge(c, mockClient) + + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + return nil + } + + assert.NoError(t, err) + + return nil + } + + _ = command.Run(context.Background(), tt.args) + }) + } +} diff --git a/woodpecker-go/woodpecker/interface.go b/woodpecker-go/woodpecker/interface.go index 611318e31..4f29717fd 100644 --- a/woodpecker-go/woodpecker/interface.go +++ b/woodpecker-go/woodpecker/interface.go @@ -84,6 +84,8 @@ type Client interface { // the specified repository. PipelineList(repoID int64, opt PipelineListOptions) ([]*Pipeline, error) + PipelineDelete(repoID, pipeline int64) error + // PipelineQueue returns a list of enqueued pipelines. PipelineQueue() ([]*Feed, error) @@ -102,9 +104,6 @@ type Client interface { // PipelineDecline declines a blocked pipeline. PipelineDecline(repoID, pipeline int64) (*Pipeline, error) - // PipelineKill force kills the running pipeline. - PipelineKill(repoID, pipeline int64) error - // PipelineMetadata returns metadata for a pipeline. PipelineMetadata(repoID int64, pipelineNumber int) ([]byte, error) diff --git a/woodpecker-go/woodpecker/mocks/client.go b/woodpecker-go/woodpecker/mocks/client.go index 0ce17b6e0..0a2e72f87 100644 --- a/woodpecker-go/woodpecker/mocks/client.go +++ b/woodpecker-go/woodpecker/mocks/client.go @@ -1133,12 +1133,12 @@ func (_m *Client) PipelineDecline(repoID int64, pipeline int64) (*woodpecker.Pip return r0, r1 } -// PipelineKill provides a mock function with given fields: repoID, pipeline -func (_m *Client) PipelineKill(repoID int64, pipeline int64) error { +// PipelineDelete provides a mock function with given fields: repoID, pipeline +func (_m *Client) PipelineDelete(repoID int64, pipeline int64) error { ret := _m.Called(repoID, pipeline) if len(ret) == 0 { - panic("no return value specified for PipelineKill") + panic("no return value specified for PipelineDelete") } var r0 error diff --git a/woodpecker-go/woodpecker/repo.go b/woodpecker-go/woodpecker/repo.go index af5d0ca95..5c24b228d 100644 --- a/woodpecker-go/woodpecker/repo.go +++ b/woodpecker-go/woodpecker/repo.go @@ -321,6 +321,13 @@ func (c *client) PipelineList(repoID int64, opt PipelineListOptions) ([]*Pipelin return out, err } +// PipelineDelete deletes a pipeline by the specified repository ID and pipeline ID. +func (c *client) PipelineDelete(repoID, pipeline int64) error { + uri := fmt.Sprintf(pathPipeline, c.addr, repoID, pipeline) + err := c.delete(uri) + return err +} + // PipelineCreate creates a new pipeline for the specified repository. func (c *client) PipelineCreate(repoID int64, options *PipelineOptions) (*Pipeline, error) { var out *Pipeline @@ -361,13 +368,6 @@ func (c *client) PipelineDecline(repoID, pipeline int64) (*Pipeline, error) { return out, err } -// PipelineKill force kills the running pipeline. -func (c *client) PipelineKill(repoID, pipeline int64) error { - uri := fmt.Sprintf(pathPipeline, c.addr, repoID, pipeline) - err := c.delete(uri) - return err -} - // LogsPurge purges the pipeline all steps logs for the specified pipeline. func (c *client) LogsPurge(repoID, pipeline int64) error { uri := fmt.Sprintf(pathPipelineLogs, c.addr, repoID, pipeline)