mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-05-20 09:18:15 +00:00
Code Issues Projects Releases Wiki Activity

Add support for superseding runs (#831)

Browse source 
closes #11

Added support:
1. Environment variable `WOODPECKER_DELETE_MULTIPLE_RUNS_ON_EVENTS` (Default pull_request, push)
2. Builds will be marked as killed when they "override" another build
This commit is contained in:
Zav Shotan 2022-05-09 05:26:09 -04:00 committed by GitHub
parent a127745a23
commit 7313de2b1d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 254 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cmd/server/flags.go
View file

@ -97,6 +97,12 @@ var flags = []cli.Flag{
Name: "authenticate-public-repos",
Usage: "Always use authentication to clone repositories even if they are public. Needed if the SCM requires to always authenticate as used by many companies.",
},
&cli.StringSliceFlag{
EnvVars: []string{"WOODPECKER_DEFAULT_CANCEL_PREVIOUS_PIPELINE_EVENTS"},
Name: "default-cancel-previous-pipeline-events",
Usage: "List of event names that will be canceled when a new pipeline for the same context (tag, branch) is created.",
Value: cli.NewStringSlice("push", "pull_request"),
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_DEFAULT_CLONE_IMAGE"},
Name: "default-clone-image",

9
cmd/server/server.go
View file

@ -41,6 +41,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server"
woodpeckerGrpcServer "github.com/woodpecker-ci/woodpecker/server/grpc"
"github.com/woodpecker-ci/woodpecker/server/logging"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/plugins/configuration"
"github.com/woodpecker-ci/woodpecker/server/plugins/sender"
"github.com/woodpecker-ci/woodpecker/server/pubsub"
@ -287,6 +288,14 @@ func setupEvilGlobals(c *cli.Context, v store.Store, r remote.Remote) {
// Cloning
server.Config.Pipeline.DefaultCloneImage = c.String("default-clone-image")
// Execution
_events := c.StringSlice("default-cancel-previous-pipeline-events")
events := make([]model.WebhookEvent, len(_events))
for _, v := range _events {
events = append(events, model.WebhookEvent(v))
}
server.Config.Pipeline.DefaultCancelPreviousPipelineEvents = events
// limits
server.Config.Pipeline.Limits.MemSwapLimit = c.Int64("limit-mem-swap")
server.Config.Pipeline.Limits.MemLimit = c.Int64("limit-mem")

3
docs/docs/20-usage/71-project-settings.md
View file

@ -40,3 +40,6 @@ You can change the visibility of your project by this setting. If a user has acc
After this timeout a pipeline has to finish or will be treated as timed out.
## Cancel previous pipelines
By enabling this option for a pipeline event previous pipelines of the same event and context will be canceled before starting the newly triggered one.

5
docs/docs/30-administration/10-server-config.md
View file

@ -197,6 +197,11 @@ Link to documentation in the UI.
Always use authentication to clone repositories even if they are public. Needed if the forge requires to always authenticate as used by many companies.
### `WOODPECKER_DEFAULT_CANCEL_PREVIOUS_PIPELINE_EVENTS`
> Default: `pull_request, push`
List of event names that will be canceled when a new pipeline for the same context (tag, branch) is created.
### `WOODPECKER_DEFAULT_CLONE_IMAGE`
> Default: `woodpeckerci/plugin-git:latest`

161
server/api/build.go
View file

@ -216,45 +216,60 @@ func DeleteBuild(c *gin.Context) {
return
}
procs, err := _store.ProcList(build)
if err != nil {
_ = c.AbortWithError(http.StatusNotFound, err)
return
}
if build.Status != model.StatusRunning && build.Status != model.StatusPending {
c.String(http.StatusBadRequest, "Cannot cancel a non-running or non-pending build")
return
}
code, err := cancelBuild(c, _store, repo, build)
if err != nil {
_ = c.AbortWithError(code, err)
return
}
c.String(code, "")
}
// Cancel the build and returns the status.
func cancelBuild(
ctx context.Context,
_store store.Store,
repo *model.Repo,
build *model.Build,
) (int, error) {
procs, err := _store.ProcList(build)
if err != nil {
return http.StatusNotFound, err
}
// First cancel/evict procs in the queue in one go
var (
procToCancel []string
procToEvict []string
procsToCancel []string
procsToEvict []string
)
for _, proc := range procs {
if proc.PPID != 0 {
continue
}
if proc.State == model.StatusRunning {
procToCancel = append(procToCancel, fmt.Sprint(proc.ID))
procsToCancel = append(procsToCancel, fmt.Sprint(proc.ID))
}
if proc.State == model.StatusPending {
procToEvict = append(procToEvict, fmt.Sprint(proc.ID))
procsToEvict = append(procsToEvict, fmt.Sprint(proc.ID))
}
}
if len(procToEvict) != 0 {
if err := server.Config.Services.Queue.EvictAtOnce(c, procToEvict); err != nil {
log.Error().Err(err).Msgf("queue: evict_at_once: %v", procToEvict)
if len(procsToEvict) != 0 {
if err := server.Config.Services.Queue.EvictAtOnce(ctx, procsToEvict); err != nil {
log.Error().Err(err).Msgf("queue: evict_at_once: %v", procsToEvict)
}
if err := server.Config.Services.Queue.ErrorAtOnce(c, procToEvict, queue.ErrCancel); err != nil {
log.Error().Err(err).Msgf("queue: evict_at_once: %v", procToEvict)
if err := server.Config.Services.Queue.ErrorAtOnce(ctx, procsToEvict, queue.ErrCancel); err != nil {
log.Error().Err(err).Msgf("queue: evict_at_once: %v", procsToEvict)
}
}
if len(procToCancel) != 0 {
if err := server.Config.Services.Queue.ErrorAtOnce(c, procToCancel, queue.ErrCancel); err != nil {
log.Error().Err(err).Msgf("queue: evict_at_once: %v", procToCancel)
if len(procsToCancel) != 0 {
if err := server.Config.Services.Queue.ErrorAtOnce(ctx, procsToCancel, queue.ErrCancel); err != nil {
log.Error().Err(err).Msgf("queue: evict_at_once: %v", procsToCancel)
}
}
@ -277,8 +292,7 @@ func DeleteBuild(c *gin.Context) {
killedBuild, err := shared.UpdateToStatusKilled(_store, *build)
if err != nil {
log.Error().Err(err).Msgf("UpdateToStatusKilled: %v", build)
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
return http.StatusInternalServerError, err
}
// For pending builds, we stream the UI the latest state.
@ -286,19 +300,17 @@ func DeleteBuild(c *gin.Context) {
if build.Status == model.StatusPending {
procs, err = _store.ProcList(killedBuild)
if err != nil {
_ = c.AbortWithError(404, err)
return
return http.StatusNotFound, err
}
if killedBuild.Procs, err = model.Tree(procs); err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
return http.StatusInternalServerError, err
}
if err := publishToTopic(c, killedBuild, repo); err != nil {
if err := publishToTopic(ctx, killedBuild, repo); err != nil {
log.Error().Err(err).Msg("publishToTopic")
}
}
c.String(204, "")
return http.StatusNoContent, nil
}
func PostApproval(c *gin.Context) {
@ -651,26 +663,109 @@ func createBuildItems(ctx context.Context, store store.Store, build *model.Build
return build, buildItems, nil
}
func startBuild(ctx context.Context, store store.Store, build *model.Build, user *model.User, repo *model.Repo, buildItems []*shared.BuildItem) (*model.Build, error) {
if err := store.ProcCreate(build.Procs); err != nil {
log.Error().Err(err).Str("repo", repo.FullName).Msgf("error persisting procs for %s#%d", repo.FullName, build.Number)
func cancelPreviousPipelines(
ctx context.Context,
_store store.Store,
build *model.Build,
user *model.User,
repo *model.Repo,
) error {
// check this event should cancel previous pipelines
eventIncluded := false
for _, ev := range repo.CancelPreviousPipelineEvents {
if ev == build.Event {
eventIncluded = true
break
}
}
if !eventIncluded {
return nil
}
// get all active activeBuilds
activeBuilds, err := _store.GetActiveBuildList(repo, -1)
if err != nil {
return err
}
buildNeedsCancel := func(active *model.Build) (bool, error) {
// always filter on same event
if active.Event != build.Event {
return false, nil
}
// find events for the same context
switch build.Event {
case model.EventPush:
return build.Branch == active.Branch, nil
default:
return build.Refspec == active.Refspec, nil
}
}
for _, active := range activeBuilds {
if active.ID == build.ID {
// same build. e.g. self
continue
}
cancel, err := buildNeedsCancel(active)
if err != nil {
log.Error().
Err(err).
Str("Ref", active.Ref).
Msg("Error while trying to cancel build, skipping")
continue
}
if !cancel {
continue
}
_, err = cancelBuild(ctx, _store, repo, active)
if err != nil {
log.Error().
Err(err).
Str("Ref", active.Ref).
Int64("ID", active.ID).
Msg("Failed to cancel build")
}
}
return nil
}
func startBuild(
ctx context.Context,
store store.Store,
activeBuild *model.Build,
user *model.User,
repo *model.Repo,
buildItems []*shared.BuildItem,
) (*model.Build, error) {
// call to cancel previous builds if needed
if err := cancelPreviousPipelines(ctx, store, activeBuild, user, repo); err != nil {
// should be not breaking
log.Error().Err(err).Msg("Failed to cancel previous builds")
}
if err := store.ProcCreate(activeBuild.Procs); err != nil {
log.Error().Err(err).Str("repo", repo.FullName).Msgf("error persisting procs for %s#%d", repo.FullName, activeBuild.Number)
return nil, err
}
if err := publishToTopic(ctx, build, repo); err != nil {
if err := publishToTopic(ctx, activeBuild, repo); err != nil {
log.Error().Err(err).Msg("publishToTopic")
}
if err := queueBuild(build, repo, buildItems); err != nil {
if err := queueBuild(activeBuild, repo, buildItems); err != nil {
log.Error().Err(err).Msg("queueBuild")
return nil, err
}
if err := updateBuildStatus(ctx, build, repo, user); err != nil {
if err := updateBuildStatus(ctx, activeBuild, repo, user); err != nil {
log.Error().Err(err).Msg("updateBuildStatus")
}
return build, nil
return activeBuild, nil
}
func updateBuildStatus(ctx context.Context, build *model.Build, repo *model.Repo, user *model.User) error {

4
server/api/repo.go
View file

@ -51,6 +51,7 @@ func PostRepo(c *gin.Context) {
repo.IsActive = true
repo.UserID = user.ID
repo.AllowPull = true
repo.CancelPreviousPipelineEvents = server.Config.Pipeline.DefaultCancelPreviousPipelineEvents
if repo.Visibility == "" {
repo.Visibility = model.VisibilityPublic
@ -140,6 +141,9 @@ func PatchRepo(c *gin.Context) {
if in.Config != nil {
repo.Config = *in.Config
}
if in.CancelPreviousPipelineEvents != nil {
repo.CancelPreviousPipelineEvents = *in.CancelPreviousPipelineEvents
}
if in.Visibility != nil {
switch *in.Visibility {
case string(model.VisibilityInternal), string(model.VisibilityPrivate), string(model.VisibilityPublic):

13
server/config.go
View file

@ -68,12 +68,13 @@ var Config = struct {
AuthToken string
}
Pipeline struct {
AuthenticatePublicRepos bool
DefaultCloneImage string
Limits model.ResourceLimit
Volumes []string
Networks []string
Privileged []string
AuthenticatePublicRepos bool
DefaultCancelPreviousPipelineEvents []model.WebhookEvent
DefaultCloneImage string
Limits model.ResourceLimit
Volumes []string
Networks []string
Privileged []string
}
FlatPermissions bool // TODO(485) temporary workaround to not hit api rate limits
}{}

56
server/model/repo.go
View file

@ -24,27 +24,28 @@ import (
//
// swagger:model repo
type Repo struct {
ID int64 `json:"id,omitempty" xorm:"pk autoincr 'repo_id'"`
UserID int64 `json:"-" xorm:"repo_user_id"`
Owner string `json:"owner" xorm:"UNIQUE(name) 'repo_owner'"`
Name string `json:"name" xorm:"UNIQUE(name) 'repo_name'"`
FullName string `json:"full_name" xorm:"UNIQUE 'repo_full_name'"`
Avatar string `json:"avatar_url,omitempty" xorm:"varchar(500) 'repo_avatar'"`
Link string `json:"link_url,omitempty" xorm:"varchar(1000) 'repo_link'"`
Clone string `json:"clone_url,omitempty" xorm:"varchar(1000) 'repo_clone'"`
Branch string `json:"default_branch,omitempty" xorm:"varchar(500) 'repo_branch'"`
SCMKind SCMKind `json:"scm,omitempty" xorm:"varchar(50) 'repo_scm'"`
Timeout int64 `json:"timeout,omitempty" xorm:"repo_timeout"`
Visibility RepoVisibly `json:"visibility" xorm:"varchar(10) 'repo_visibility'"`
IsSCMPrivate bool `json:"private" xorm:"repo_private"`
IsTrusted bool `json:"trusted" xorm:"repo_trusted"`
IsStarred bool `json:"starred,omitempty" xorm:"-"`
IsGated bool `json:"gated" xorm:"repo_gated"`
IsActive bool `json:"active" xorm:"repo_active"`
AllowPull bool `json:"allow_pr" xorm:"repo_allow_pr"`
Config string `json:"config_file" xorm:"varchar(500) 'repo_config_path'"`
Hash string `json:"-" xorm:"varchar(500) 'repo_hash'"`
Perm *Perm `json:"-" xorm:"-"`
ID int64 `json:"id,omitempty" xorm:"pk autoincr 'repo_id'"`
UserID int64 `json:"-" xorm:"repo_user_id"`
Owner string `json:"owner" xorm:"UNIQUE(name) 'repo_owner'"`
Name string `json:"name" xorm:"UNIQUE(name) 'repo_name'"`
FullName string `json:"full_name" xorm:"UNIQUE 'repo_full_name'"`
Avatar string `json:"avatar_url,omitempty" xorm:"varchar(500) 'repo_avatar'"`
Link string `json:"link_url,omitempty" xorm:"varchar(1000) 'repo_link'"`
Clone string `json:"clone_url,omitempty" xorm:"varchar(1000) 'repo_clone'"`
Branch string `json:"default_branch,omitempty" xorm:"varchar(500) 'repo_branch'"`
SCMKind SCMKind `json:"scm,omitempty" xorm:"varchar(50) 'repo_scm'"`
Timeout int64 `json:"timeout,omitempty" xorm:"repo_timeout"`
Visibility RepoVisibly `json:"visibility" xorm:"varchar(10) 'repo_visibility'"`
IsSCMPrivate bool `json:"private" xorm:"repo_private"`
IsTrusted bool `json:"trusted" xorm:"repo_trusted"`
IsStarred bool `json:"starred,omitempty" xorm:"-"`
IsGated bool `json:"gated" xorm:"repo_gated"`
IsActive bool `json:"active" xorm:"repo_active"`
AllowPull bool `json:"allow_pr" xorm:"repo_allow_pr"`
Config string `json:"config_file" xorm:"varchar(500) 'repo_config_path'"`
Hash string `json:"-" xorm:"varchar(500) 'repo_hash'"`
Perm *Perm `json:"-" xorm:"-"`
CancelPreviousPipelineEvents []WebhookEvent `json:"cancel_previous_pipeline_events" xorm:"json 'cancel_previous_pipeline_events'"`
}
// TableName return database table name for xorm
@ -90,10 +91,11 @@ func (r *Repo) Update(from *Repo) {
// RepoPatch represents a repository patch object.
type RepoPatch struct {
Config *string `json:"config_file,omitempty"`
IsTrusted *bool `json:"trusted,omitempty"`
IsGated *bool `json:"gated,omitempty"`
Timeout *int64 `json:"timeout,omitempty"`
Visibility *string `json:"visibility,omitempty"`
AllowPull *bool `json:"allow_pr,omitempty"`
Config *string `json:"config_file,omitempty"`
IsTrusted *bool `json:"trusted,omitempty"`
IsGated *bool `json:"gated,omitempty"`
Timeout *int64 `json:"timeout,omitempty"`
Visibility *string `json:"visibility,omitempty"`
AllowPull *bool `json:"allow_pr,omitempty"`
CancelPreviousPipelineEvents *[]WebhookEvent `json:"cancel_previous_pipeline_events"`
}

12
server/store/datastore/build.go
View file

@ -80,6 +80,18 @@ func (s storage) GetBuildList(repo *model.Repo, page int) ([]*model.Build, error
Find(&builds)
}
func (s storage) GetActiveBuildList(repo *model.Repo, page int) ([]*model.Build, error) {
builds := make([]*model.Build, 0, perPage)
query := s.engine.
Where("build_repo_id = ?", repo.ID).
Where("build_status = ? or build_status = ?", "pending", "running").
Desc("build_number")
if page > 0 {
query = query.Limit(perPage, perPage*(page-1))
}
return builds, query.Find(&builds)
}
func (s storage) GetBuildCount() (int64, error) {
return s.engine.Count(new(model.Build))
}

2
server/store/store.go
View file

@ -70,6 +70,8 @@ type Store interface {
// GetBuildList gets a list of builds for the repository
// TODO: paginate
GetBuildList(*model.Repo, int) ([]*model.Build, error)
// GetBuildList gets a list of the active builds for the repository
GetActiveBuildList(repo *model.Repo, page int) ([]*model.Build, error)
// GetBuildQueue gets a list of build in queue.
GetBuildQueue() ([]*model.Feed, error)
// GetBuildCount gets a count of all builds in the system.

31
web/src/components/repo/settings/GeneralTab.vue
View file

@ -50,6 +50,18 @@
</div>
</InputField>
<InputField label="Cancel previous pipelines" docs-url="docs/usage/project-settings#cancel-previous-pipelines">
<CheckboxesField
v-model="repoSettings.cancel_previous_pipeline_events"
:options="cancelPreviousBuildEventsOptions"
/>
<template #description>
<p class="text-sm text-gray-400 dark:text-gray-600">
Enable to cancel running pipelines of the same event and context before starting the newly triggered one.
</p>
</template>
</InputField>
<Button class="mr-auto" color="green" text="Save settings" :is-loading="isSaving" @click="saveRepoSettings" />
</div>
</Panel>
@ -60,7 +72,8 @@ import { defineComponent, inject, onMounted, Ref, ref } from 'vue';
import Button from '~/components/atomic/Button.vue';
import Checkbox from '~/components/form/Checkbox.vue';
import { RadioOption } from '~/components/form/form.types';
import CheckboxesField from '~/components/form/CheckboxesField.vue';
import { CheckboxOption, RadioOption } from '~/components/form/form.types';
import InputField from '~/components/form/InputField.vue';
import NumberField from '~/components/form/NumberField.vue';
import RadioField from '~/components/form/RadioField.vue';
@ -70,7 +83,7 @@ import useApiClient from '~/compositions/useApiClient';
import { useAsyncAction } from '~/compositions/useAsyncAction';
import useAuthentication from '~/compositions/useAuthentication';
import useNotifications from '~/compositions/useNotifications';
import { Repo, RepoSettings, RepoVisibility } from '~/lib/api/types';
import { Repo, RepoSettings, RepoVisibility, WebhookEvents } from '~/lib/api/types';
import RepoStore from '~/store/repos';
const projectVisibilityOptions: RadioOption[] = [
@ -91,10 +104,20 @@ const projectVisibilityOptions: RadioOption[] = [
},
];
const cancelPreviousBuildEventsOptions: CheckboxOption[] = [
{ value: WebhookEvents.Push, text: 'Push' },
{ value: WebhookEvents.Tag, text: 'Tag' },
{
value: WebhookEvents.PullRequest,
text: 'Pull Request',
},
{ value: WebhookEvents.Deploy, text: 'Deploy' },
];
export default defineComponent({
name: 'GeneralTab',
components: { Button, Panel, InputField, TextField, RadioField, NumberField, Checkbox },
components: { Button, Panel, InputField, TextField, RadioField, NumberField, Checkbox, CheckboxesField },
setup() {
const apiClient = useApiClient();
@ -117,6 +140,7 @@ export default defineComponent({
gated: repo.value.gated,
trusted: repo.value.trusted,
allow_pr: repo.value.allow_pr,
cancel_previous_pipeline_events: repo.value.cancel_previous_pipeline_events || [],
};
}
@ -153,6 +177,7 @@ export default defineComponent({
isSaving,
saveRepoSettings,
projectVisibilityOptions,
cancelPreviousBuildEventsOptions,
};
},
});

36
web/src/lib/api/types/repo.ts
View file

@ -1,51 +1,51 @@
// A version control repository.
export type Repo = {
active: boolean;
// Is the repo currently active or not
active: boolean;
id: number;
// The unique identifier for the repository.
id: number;
scm: string;
// The source control management being used.
// Currently this is either 'git' or 'hg' (Mercurial).
scm: string;
owner: string;
// The owner of the repository.
owner: string;
name: string;
// The name of the repository.
name: string;
full_name: string;
// The full name of the repository.
// This is created from the owner and name of the repository.
full_name: string;
avatar_url: string;
// The url for the avatar image.
avatar_url: string;
link_url: string;
// The link to view the repository.
link_url: string;
clone_url: string;
// The url used to clone the repository.
clone_url: string;
default_branch: string;
// The default branch of the repository.
default_branch: string;
private: boolean;
// Whether the repository is publicly visible.
private: boolean;
trusted: boolean;
// Whether the repository has trusted access for builds.
// If the repository is trusted then the host network can be used and
// volumes can be created.
trusted: boolean;
timeout: number;
// x-dart-type: Duration
// The amount of time in minutes before the build is killed.
timeout: number;
allow_pr: boolean;
// Whether pull requests should trigger a build.
allow_pr: boolean;
config_file: string;
@ -54,6 +54,9 @@ export type Repo = {
last_build: number;
gated: boolean;
// Events that will cancel running pipelines before starting a new one
cancel_previous_pipeline_events: string[];
};
export enum RepoVisibility {
@ -62,7 +65,10 @@ export enum RepoVisibility {
Internal = 'internal',
}
export type RepoSettings = Pick<Repo, 'config_file' | 'timeout' | 'visibility' | 'trusted' | 'gated' | 'allow_pr'>;
export type RepoSettings = Pick<
Repo,
'config_file' | 'timeout' | 'visibility' | 'trusted' | 'gated' | 'allow_pr' | 'cancel_previous_pipeline_events'
>;
export type RepoPermissions = {
pull: boolean;