Add queue details UI for admins (#1632)

# Changes
- Adds an admin view to see the whole work-queue of the server. 
- The admin can also pause / resume the queue. 
- The view is reloading data every 5 seconds automatically.
- The task model from queue got removed in favor of the one from models.
This commit is contained in:
Anbraten 2023-03-20 04:50:56 +01:00 committed by GitHub
parent 4d5c59556e
commit 2337f1854a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 432 additions and 265 deletions

View file

@ -16,6 +16,7 @@ Some versions need some changes to the server configuration or the pipeline conf
- Renamed config env `WOODPECKER_MAX_PROCS` to `WOODPECKER_MAX_WORKFLOWS` (still available as fallback) - Renamed config env `WOODPECKER_MAX_PROCS` to `WOODPECKER_MAX_WORKFLOWS` (still available as fallback)
- The pipelines are now also read from `.yaml` files, the new default order is `.woodpecker/*.yml` and `.woodpecker/*.yaml` (without any prioritization) -> `.woodpecker.yml` -> `.woodpecker.yaml` -> `.drone.yml` - The pipelines are now also read from `.yaml` files, the new default order is `.woodpecker/*.yml` and `.woodpecker/*.yaml` (without any prioritization) -> `.woodpecker.yml` -> `.woodpecker.yaml` -> `.drone.yml`
- Dropped support for [Coding](https://coding.net/). - Dropped support for [Coding](https://coding.net/).
- `/api/queue/resume` & `/api/queue/pause` endpoint methods were changed from `GET` to `POST`
## 0.15.0 ## 0.15.0

View file

@ -16,11 +16,12 @@ package grpc
import ( import (
"github.com/woodpecker-ci/woodpecker/pipeline/rpc" "github.com/woodpecker-ci/woodpecker/pipeline/rpc"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/queue" "github.com/woodpecker-ci/woodpecker/server/queue"
) )
func createFilterFunc(agentFilter rpc.Filter) (queue.FilterFn, error) { func createFilterFunc(agentFilter rpc.Filter) (queue.FilterFn, error) {
return func(task *queue.Task) bool { return func(task *model.Task) bool {
for taskLabel, taskLabelValue := range task.Labels { for taskLabel, taskLabelValue := range task.Labels {
// if a task label is empty it will be ignored // if a task label is empty it will be ignored
if taskLabelValue == "" { if taskLabelValue == "" {

View file

@ -20,7 +20,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/woodpecker-ci/woodpecker/pipeline/rpc" "github.com/woodpecker-ci/woodpecker/pipeline/rpc"
"github.com/woodpecker-ci/woodpecker/server/queue" "github.com/woodpecker-ci/woodpecker/server/model"
) )
func TestCreateFilterFunc(t *testing.T) { func TestCreateFilterFunc(t *testing.T) {
@ -29,13 +29,13 @@ func TestCreateFilterFunc(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
agentLabels map[string]string agentLabels map[string]string
task queue.Task task model.Task
exp bool exp bool
}{ }{
{ {
name: "agent with missing labels", name: "agent with missing labels",
agentLabels: map[string]string{"repo": "test/woodpecker"}, agentLabels: map[string]string{"repo": "test/woodpecker"},
task: queue.Task{ task: model.Task{
Labels: map[string]string{"platform": "linux/amd64", "repo": "test/woodpecker"}, Labels: map[string]string{"platform": "linux/amd64", "repo": "test/woodpecker"},
}, },
exp: false, exp: false,
@ -43,7 +43,7 @@ func TestCreateFilterFunc(t *testing.T) {
{ {
name: "agent with wrong labels", name: "agent with wrong labels",
agentLabels: map[string]string{"platform": "linux/arm64"}, agentLabels: map[string]string{"platform": "linux/arm64"},
task: queue.Task{ task: model.Task{
Labels: map[string]string{"platform": "linux/amd64"}, Labels: map[string]string{"platform": "linux/amd64"},
}, },
exp: false, exp: false,
@ -51,7 +51,7 @@ func TestCreateFilterFunc(t *testing.T) {
{ {
name: "agent with correct labels", name: "agent with correct labels",
agentLabels: map[string]string{"platform": "linux/amd64", "location": "europe"}, agentLabels: map[string]string{"platform": "linux/amd64", "location": "europe"},
task: queue.Task{ task: model.Task{
Labels: map[string]string{"platform": "linux/amd64", "location": "europe"}, Labels: map[string]string{"platform": "linux/amd64", "location": "europe"},
}, },
exp: true, exp: true,
@ -59,7 +59,7 @@ func TestCreateFilterFunc(t *testing.T) {
{ {
name: "agent with additional labels", name: "agent with additional labels",
agentLabels: map[string]string{"platform": "linux/amd64", "location": "europe"}, agentLabels: map[string]string{"platform": "linux/amd64", "location": "europe"},
task: queue.Task{ task: model.Task{
Labels: map[string]string{"platform": "linux/amd64"}, Labels: map[string]string{"platform": "linux/amd64"},
}, },
exp: true, exp: true,
@ -67,7 +67,7 @@ func TestCreateFilterFunc(t *testing.T) {
{ {
name: "agent with wildcard label", name: "agent with wildcard label",
agentLabels: map[string]string{"platform": "linux/amd64", "location": "*"}, agentLabels: map[string]string{"platform": "linux/amd64", "location": "*"},
task: queue.Task{ task: model.Task{
Labels: map[string]string{"platform": "linux/amd64", "location": "america"}, Labels: map[string]string{"platform": "linux/amd64", "location": "america"},
}, },
exp: true, exp: true,
@ -75,7 +75,7 @@ func TestCreateFilterFunc(t *testing.T) {
{ {
name: "agent with platform label and task without", name: "agent with platform label and task without",
agentLabels: map[string]string{"platform": "linux/amd64"}, agentLabels: map[string]string{"platform": "linux/amd64"},
task: queue.Task{ task: model.Task{
Labels: map[string]string{"platform": ""}, Labels: map[string]string{"platform": ""},
}, },
exp: true, exp: true,

View file

@ -14,6 +14,11 @@
package model package model
import (
"fmt"
"strings"
)
// TaskStore defines storage for scheduled Tasks. // TaskStore defines storage for scheduled Tasks.
type TaskStore interface { type TaskStore interface {
TaskList() ([]*Task, error) TaskList() ([]*Task, error)
@ -21,17 +26,80 @@ type TaskStore interface {
TaskDelete(string) error TaskDelete(string) error
} }
type TaskStatusValue string
const (
TaskStatusSkipped TaskStatusValue = "skipped"
TaskStatusSuccess TaskStatusValue = "success"
TaskStatusFailure TaskStatusValue = "failure"
)
// Task defines scheduled pipeline Task. // Task defines scheduled pipeline Task.
type Task struct { type Task struct {
ID string `xorm:"PK UNIQUE 'task_id'"` ID string `json:"id" xorm:"PK UNIQUE 'task_id'"`
Data []byte `xorm:"'task_data'"` Data []byte `json:"data" xorm:"'task_data'"`
Labels map[string]string `xorm:"json 'task_labels'"` Labels map[string]string `json:"labels" xorm:"json 'task_labels'"`
Dependencies []string `xorm:"json 'task_dependencies'"` Dependencies []string `json:"dependencies" xorm:"json 'task_dependencies'"`
RunOn []string `xorm:"json 'task_run_on'"` RunOn []string `json:"run_on" xorm:"json 'task_run_on'"`
DepStatus map[string]string `xorm:"json 'task_dep_status'"` DepStatus map[string]StatusValue `json:"dep_status" xorm:"json 'task_dep_status'"`
} }
// TableName return database table name for xorm // TableName return database table name for xorm
func (Task) TableName() string { func (Task) TableName() string {
return "tasks" return "tasks"
} }
func (t *Task) String() string {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("%s (%s) - %s", t.ID, t.Dependencies, t.DepStatus))
return sb.String()
}
// ShouldRun tells if a task should be run or skipped, based on dependencies
func (t *Task) ShouldRun() bool {
if t.runsOnFailure() && t.runsOnSuccess() {
return true
}
if !t.runsOnFailure() && t.runsOnSuccess() {
for _, status := range t.DepStatus {
if status != StatusSuccess {
return false
}
}
return true
}
if t.runsOnFailure() && !t.runsOnSuccess() {
for _, status := range t.DepStatus {
if status == StatusSuccess {
return false
}
}
return true
}
return false
}
func (t *Task) runsOnFailure() bool {
for _, status := range t.RunOn {
if status == string(StatusFailure) {
return true
}
}
return false
}
func (t *Task) runsOnSuccess() bool {
if len(t.RunOn) == 0 {
return true
}
for _, status := range t.RunOn {
if status == string(StatusSuccess) {
return true
}
}
return false
}

View file

@ -23,16 +23,15 @@ import (
"github.com/woodpecker-ci/woodpecker/pipeline/rpc" "github.com/woodpecker-ci/woodpecker/pipeline/rpc"
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/queue"
) )
func queuePipeline(repo *model.Repo, pipelineItems []*pipeline.Item) error { func queuePipeline(repo *model.Repo, pipelineItems []*pipeline.Item) error {
var tasks []*queue.Task var tasks []*model.Task
for _, item := range pipelineItems { for _, item := range pipelineItems {
if item.Step.State == model.StatusSkipped { if item.Step.State == model.StatusSkipped {
continue continue
} }
task := new(queue.Task) task := new(model.Task)
task.ID = fmt.Sprint(item.Step.ID) task.ID = fmt.Sprint(item.Step.ID)
task.Labels = map[string]string{} task.Labels = map[string]string{}
for k, v := range item.Labels { for k, v := range item.Labels {
@ -42,7 +41,7 @@ func queuePipeline(repo *model.Repo, pipelineItems []*pipeline.Item) error {
task.Labels["repo"] = repo.FullName task.Labels["repo"] = repo.FullName
task.Dependencies = taskIds(item.DependsOn, pipelineItems) task.Dependencies = taskIds(item.DependsOn, pipelineItems)
task.RunOn = item.RunsOn task.RunOn = item.RunsOn
task.DepStatus = make(map[string]string) task.DepStatus = make(map[string]model.StatusValue)
task.Data, _ = json.Marshal(rpc.Pipeline{ task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Step.ID), ID: fmt.Sprint(item.Step.ID),

View file

@ -26,14 +26,8 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
const (
StatusSkipped = "skipped"
StatusSuccess = "success"
StatusFailure = "failure"
)
type entry struct { type entry struct {
item *Task item *model.Task
done chan bool done chan bool
error error error error
deadline time.Time deadline time.Time
@ -41,7 +35,7 @@ type entry struct {
type worker struct { type worker struct {
filter FilterFn filter FilterFn
channel chan *Task channel chan *model.Task
} }
type fifo struct { type fifo struct {
@ -68,7 +62,7 @@ func New(_ context.Context) Queue {
} }
// Push pushes an item to the tail of this queue. // Push pushes an item to the tail of this queue.
func (q *fifo) Push(_ context.Context, task *Task) error { func (q *fifo) Push(_ context.Context, task *model.Task) error {
q.Lock() q.Lock()
q.pending.PushBack(task) q.pending.PushBack(task)
q.Unlock() q.Unlock()
@ -77,7 +71,7 @@ func (q *fifo) Push(_ context.Context, task *Task) error {
} }
// Push pushes an item to the tail of this queue. // Push pushes an item to the tail of this queue.
func (q *fifo) PushAtOnce(_ context.Context, tasks []*Task) error { func (q *fifo) PushAtOnce(_ context.Context, tasks []*model.Task) error {
q.Lock() q.Lock()
for _, task := range tasks { for _, task := range tasks {
q.pending.PushBack(task) q.pending.PushBack(task)
@ -88,10 +82,10 @@ func (q *fifo) PushAtOnce(_ context.Context, tasks []*Task) error {
} }
// Poll retrieves and removes the head of this queue. // Poll retrieves and removes the head of this queue.
func (q *fifo) Poll(c context.Context, f FilterFn) (*Task, error) { func (q *fifo) Poll(c context.Context, f FilterFn) (*model.Task, error) {
q.Lock() q.Lock()
w := &worker{ w := &worker{
channel: make(chan *Task, 1), channel: make(chan *model.Task, 1),
filter: f, filter: f,
} }
q.workers[w] = struct{}{} q.workers[w] = struct{}{}
@ -113,20 +107,20 @@ func (q *fifo) Poll(c context.Context, f FilterFn) (*Task, error) {
// Done signals that the item is done executing. // Done signals that the item is done executing.
func (q *fifo) Done(_ context.Context, id string, exitStatus model.StatusValue) error { func (q *fifo) Done(_ context.Context, id string, exitStatus model.StatusValue) error {
return q.finished([]string{id}, string(exitStatus), nil) return q.finished([]string{id}, exitStatus, nil)
} }
// Error signals that the item is done executing with error. // Error signals that the item is done executing with error.
func (q *fifo) Error(_ context.Context, id string, err error) error { func (q *fifo) Error(_ context.Context, id string, err error) error {
return q.finished([]string{id}, StatusFailure, err) return q.finished([]string{id}, model.StatusFailure, err)
} }
// Error signals that the item is done executing with error. // Error signals that the item is done executing with error.
func (q *fifo) ErrorAtOnce(_ context.Context, id []string, err error) error { func (q *fifo) ErrorAtOnce(_ context.Context, id []string, err error) error {
return q.finished(id, StatusFailure, err) return q.finished(id, model.StatusFailure, err)
} }
func (q *fifo) finished(ids []string, exitStatus string, err error) error { func (q *fifo) finished(ids []string, exitStatus model.StatusValue, err error) error {
q.Lock() q.Lock()
for _, id := range ids { for _, id := range ids {
@ -159,7 +153,7 @@ func (q *fifo) EvictAtOnce(_ context.Context, ids []string) error {
var next *list.Element var next *list.Element
for e := q.pending.Front(); e != nil; e = next { for e := q.pending.Front(); e != nil; e = next {
next = e.Next() next = e.Next()
task, ok := e.Value.(*Task) task, ok := e.Value.(*model.Task)
if ok && task.ID == id { if ok && task.ID == id {
q.pending.Remove(e) q.pending.Remove(e)
return nil return nil
@ -205,12 +199,13 @@ func (q *fifo) Info(_ context.Context) InfoT {
stats.Stats.Pending = q.pending.Len() stats.Stats.Pending = q.pending.Len()
stats.Stats.WaitingOnDeps = q.waitingOnDeps.Len() stats.Stats.WaitingOnDeps = q.waitingOnDeps.Len()
stats.Stats.Running = len(q.running) stats.Stats.Running = len(q.running)
stats.Stats.Complete = 0 // TODO: implement this
for e := q.pending.Front(); e != nil; e = e.Next() { for e := q.pending.Front(); e != nil; e = e.Next() {
stats.Pending = append(stats.Pending, e.Value.(*Task)) stats.Pending = append(stats.Pending, e.Value.(*model.Task))
} }
for e := q.waitingOnDeps.Front(); e != nil; e = e.Next() { for e := q.waitingOnDeps.Front(); e != nil; e = e.Next() {
stats.WaitingOnDeps = append(stats.WaitingOnDeps, e.Value.(*Task)) stats.WaitingOnDeps = append(stats.WaitingOnDeps, e.Value.(*model.Task))
} }
for _, entry := range q.running { for _, entry := range q.running {
stats.Running = append(stats.Running, entry.item) stats.Running = append(stats.Running, entry.item)
@ -258,7 +253,7 @@ func (q *fifo) process() {
q.resubmitExpiredPipelines() q.resubmitExpiredPipelines()
q.filterWaiting() q.filterWaiting()
for pending, worker := q.assignToWorker(); pending != nil && worker != nil; pending, worker = q.assignToWorker() { for pending, worker := q.assignToWorker(); pending != nil && worker != nil; pending, worker = q.assignToWorker() {
task := pending.Value.(*Task) task := pending.Value.(*model.Task)
delete(q.workers, worker) delete(q.workers, worker)
q.pending.Remove(pending) q.pending.Remove(pending)
q.running[task.ID] = &entry{ q.running[task.ID] = &entry{
@ -275,7 +270,7 @@ func (q *fifo) filterWaiting() {
var nextWaiting *list.Element var nextWaiting *list.Element
for e := q.waitingOnDeps.Front(); e != nil; e = nextWaiting { for e := q.waitingOnDeps.Front(); e != nil; e = nextWaiting {
nextWaiting = e.Next() nextWaiting = e.Next()
task := e.Value.(*Task) task := e.Value.(*model.Task)
q.pending.PushBack(task) q.pending.PushBack(task)
} }
@ -285,7 +280,7 @@ func (q *fifo) filterWaiting() {
var nextPending *list.Element var nextPending *list.Element
for e := q.pending.Front(); e != nil; e = nextPending { for e := q.pending.Front(); e != nil; e = nextPending {
nextPending = e.Next() nextPending = e.Next()
task := e.Value.(*Task) task := e.Value.(*model.Task)
if q.depsInQueue(task) { if q.depsInQueue(task) {
log.Debug().Msgf("queue: waiting due to unmet dependencies %v", task.ID) log.Debug().Msgf("queue: waiting due to unmet dependencies %v", task.ID)
q.waitingOnDeps.PushBack(task) q.waitingOnDeps.PushBack(task)
@ -303,7 +298,7 @@ func (q *fifo) assignToWorker() (*list.Element, *worker) {
var next *list.Element var next *list.Element
for e := q.pending.Front(); e != nil; e = next { for e := q.pending.Front(); e != nil; e = next {
next = e.Next() next = e.Next()
task := e.Value.(*Task) task := e.Value.(*model.Task)
log.Debug().Msgf("queue: trying to assign task: %v with deps %v", task.ID, task.Dependencies) log.Debug().Msgf("queue: trying to assign task: %v with deps %v", task.ID, task.Dependencies)
for w := range q.workers { for w := range q.workers {
@ -327,11 +322,11 @@ func (q *fifo) resubmitExpiredPipelines() {
} }
} }
func (q *fifo) depsInQueue(task *Task) bool { func (q *fifo) depsInQueue(task *model.Task) bool {
var next *list.Element var next *list.Element
for e := q.pending.Front(); e != nil; e = next { for e := q.pending.Front(); e != nil; e = next {
next = e.Next() next = e.Next()
possibleDep, ok := e.Value.(*Task) possibleDep, ok := e.Value.(*model.Task)
log.Debug().Msgf("queue: pending right now: %v", possibleDep.ID) log.Debug().Msgf("queue: pending right now: %v", possibleDep.ID)
for _, dep := range task.Dependencies { for _, dep := range task.Dependencies {
if ok && possibleDep.ID == dep { if ok && possibleDep.ID == dep {
@ -350,11 +345,11 @@ func (q *fifo) depsInQueue(task *Task) bool {
return false return false
} }
func (q *fifo) updateDepStatusInQueue(taskID, status string) { func (q *fifo) updateDepStatusInQueue(taskID string, status model.StatusValue) {
var next *list.Element var next *list.Element
for e := q.pending.Front(); e != nil; e = next { for e := q.pending.Front(); e != nil; e = next {
next = e.Next() next = e.Next()
pending, ok := e.Value.(*Task) pending, ok := e.Value.(*model.Task)
for _, dep := range pending.Dependencies { for _, dep := range pending.Dependencies {
if ok && taskID == dep { if ok && taskID == dep {
pending.DepStatus[dep] = status pending.DepStatus[dep] = status
@ -372,7 +367,7 @@ func (q *fifo) updateDepStatusInQueue(taskID, status string) {
for e := q.waitingOnDeps.Front(); e != nil; e = next { for e := q.waitingOnDeps.Front(); e != nil; e = next {
next = e.Next() next = e.Next()
waiting, ok := e.Value.(*Task) waiting, ok := e.Value.(*model.Task)
for _, dep := range waiting.Dependencies { for _, dep := range waiting.Dependencies {
if ok && taskID == dep { if ok && taskID == dep {
waiting.DepStatus[dep] = status waiting.DepStatus[dep] = status
@ -386,7 +381,7 @@ func (q *fifo) removeFromPending(taskID string) {
var next *list.Element var next *list.Element
for e := q.pending.Front(); e != nil; e = next { for e := q.pending.Front(); e != nil; e = next {
next = e.Next() next = e.Next()
task := e.Value.(*Task) task := e.Value.(*model.Task)
if task.ID == taskID { if task.ID == taskID {
log.Debug().Msgf("queue: %s is removed from pending", taskID) log.Debug().Msgf("queue: %s is removed from pending", taskID)
q.pending.Remove(e) q.pending.Remove(e)

View file

@ -9,12 +9,13 @@ import (
"time" "time"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/woodpecker-ci/woodpecker/server/model"
) )
var noContext = context.Background() var noContext = context.Background()
func TestFifo(t *testing.T) { func TestFifo(t *testing.T) {
want := &Task{ID: "1"} want := &model.Task{ID: "1"}
q := New(context.Background()) q := New(context.Background())
assert.NoError(t, q.Push(noContext, want)) assert.NoError(t, q.Push(noContext, want))
@ -24,7 +25,7 @@ func TestFifo(t *testing.T) {
return return
} }
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
if got != want { if got != want {
t.Errorf("expect task returned form queue") t.Errorf("expect task returned form queue")
return return
@ -40,7 +41,7 @@ func TestFifo(t *testing.T) {
return return
} }
assert.NoError(t, q.Done(noContext, got.ID, StatusSuccess)) assert.NoError(t, q.Done(noContext, got.ID, model.StatusSuccess))
info = q.Info(noContext) info = q.Info(noContext)
if len(info.Pending) != 0 { if len(info.Pending) != 0 {
t.Errorf("expect task removed from pending queue") t.Errorf("expect task removed from pending queue")
@ -53,7 +54,7 @@ func TestFifo(t *testing.T) {
} }
func TestFifoExpire(t *testing.T) { func TestFifoExpire(t *testing.T) {
want := &Task{ID: "1"} want := &model.Task{ID: "1"}
q := New(context.Background()).(*fifo) q := New(context.Background()).(*fifo)
q.extension = 0 q.extension = 0
@ -64,7 +65,7 @@ func TestFifoExpire(t *testing.T) {
return return
} }
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
if got != want { if got != want {
t.Errorf("expect task returned form queue") t.Errorf("expect task returned form queue")
return return
@ -78,12 +79,12 @@ func TestFifoExpire(t *testing.T) {
} }
func TestFifoWait(t *testing.T) { func TestFifoWait(t *testing.T) {
want := &Task{ID: "1"} want := &model.Task{ID: "1"}
q := New(context.Background()).(*fifo) q := New(context.Background()).(*fifo)
assert.NoError(t, q.Push(noContext, want)) assert.NoError(t, q.Push(noContext, want))
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
if got != want { if got != want {
t.Errorf("expect task returned form queue") t.Errorf("expect task returned form queue")
return return
@ -97,12 +98,12 @@ func TestFifoWait(t *testing.T) {
}() }()
<-time.After(time.Millisecond) <-time.After(time.Millisecond)
assert.NoError(t, q.Done(noContext, got.ID, StatusSuccess)) assert.NoError(t, q.Done(noContext, got.ID, model.StatusSuccess))
wg.Wait() wg.Wait()
} }
func TestFifoEvict(t *testing.T) { func TestFifoEvict(t *testing.T) {
t1 := &Task{ID: "1"} t1 := &model.Task{ID: "1"}
q := New(context.Background()) q := New(context.Background())
assert.NoError(t, q.Push(noContext, t1)) assert.NoError(t, q.Push(noContext, t1))
@ -123,28 +124,28 @@ func TestFifoEvict(t *testing.T) {
} }
func TestFifoDependencies(t *testing.T) { func TestFifoDependencies(t *testing.T) {
task1 := &Task{ task1 := &model.Task{
ID: "1", ID: "1",
} }
task2 := &Task{ task2 := &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
} }
q := New(context.Background()).(*fifo) q := New(context.Background()).(*fifo)
assert.NoError(t, q.PushAtOnce(noContext, []*Task{task2, task1})) assert.NoError(t, q.PushAtOnce(noContext, []*model.Task{task2, task1}))
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
if got != task1 { if got != task1 {
t.Errorf("expect task1 returned from queue as task2 depends on it") t.Errorf("expect task1 returned from queue as task2 depends on it")
return return
} }
assert.NoError(t, q.Done(noContext, got.ID, StatusSuccess)) assert.NoError(t, q.Done(noContext, got.ID, model.StatusSuccess))
got, _ = q.Poll(noContext, func(*Task) bool { return true }) got, _ = q.Poll(noContext, func(*model.Task) bool { return true })
if got != task2 { if got != task2 {
t.Errorf("expect task2 returned from queue") t.Errorf("expect task2 returned from queue")
return return
@ -152,27 +153,27 @@ func TestFifoDependencies(t *testing.T) {
} }
func TestFifoErrors(t *testing.T) { func TestFifoErrors(t *testing.T) {
task1 := &Task{ task1 := &model.Task{
ID: "1", ID: "1",
} }
task2 := &Task{ task2 := &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
} }
task3 := &Task{ task3 := &model.Task{
ID: "3", ID: "3",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
RunOn: []string{"success", "failure"}, RunOn: []string{"success", "failure"},
} }
q := New(context.Background()).(*fifo) q := New(context.Background()).(*fifo)
assert.NoError(t, q.PushAtOnce(noContext, []*Task{task2, task3, task1})) assert.NoError(t, q.PushAtOnce(noContext, []*model.Task{task2, task3, task1}))
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
if got != task1 { if got != task1 {
t.Errorf("expect task1 returned from queue as task2 depends on it") t.Errorf("expect task1 returned from queue as task2 depends on it")
return return
@ -180,7 +181,7 @@ func TestFifoErrors(t *testing.T) {
assert.NoError(t, q.Error(noContext, got.ID, fmt.Errorf("exitcode 1, there was an error"))) assert.NoError(t, q.Error(noContext, got.ID, fmt.Errorf("exitcode 1, there was an error")))
got, _ = q.Poll(noContext, func(*Task) bool { return true }) got, _ = q.Poll(noContext, func(*model.Task) bool { return true })
if got != task2 { if got != task2 {
t.Errorf("expect task2 returned from queue") t.Errorf("expect task2 returned from queue")
return return
@ -191,7 +192,7 @@ func TestFifoErrors(t *testing.T) {
return return
} }
got, _ = q.Poll(noContext, func(*Task) bool { return true }) got, _ = q.Poll(noContext, func(*model.Task) bool { return true })
if got != task3 { if got != task3 {
t.Errorf("expect task3 returned from queue") t.Errorf("expect task3 returned from queue")
return return
@ -204,39 +205,39 @@ func TestFifoErrors(t *testing.T) {
} }
func TestFifoErrors2(t *testing.T) { func TestFifoErrors2(t *testing.T) {
task1 := &Task{ task1 := &model.Task{
ID: "1", ID: "1",
} }
task2 := &Task{ task2 := &model.Task{
ID: "2", ID: "2",
} }
task3 := &Task{ task3 := &model.Task{
ID: "3", ID: "3",
Dependencies: []string{"1", "2"}, Dependencies: []string{"1", "2"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
} }
q := New(context.Background()).(*fifo) q := New(context.Background()).(*fifo)
assert.NoError(t, q.PushAtOnce(noContext, []*Task{task2, task3, task1})) assert.NoError(t, q.PushAtOnce(noContext, []*model.Task{task2, task3, task1}))
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
if got != task1 && got != task2 { if got != task1 && got != task2 {
t.Errorf("expect task1 or task2 returned from queue as task3 depends on them") t.Errorf("expect task1 or task2 returned from queue as task3 depends on them")
return return
} }
if got != task1 { if got != task1 {
assert.NoError(t, q.Done(noContext, got.ID, StatusSuccess)) assert.NoError(t, q.Done(noContext, got.ID, model.StatusSuccess))
} }
if got != task2 { if got != task2 {
assert.NoError(t, q.Error(noContext, got.ID, fmt.Errorf("exitcode 1, there was an error"))) assert.NoError(t, q.Error(noContext, got.ID, fmt.Errorf("exitcode 1, there was an error")))
} }
} }
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
if got != task3 { if got != task3 {
t.Errorf("expect task3 returned from queue") t.Errorf("expect task3 returned from queue")
return return
@ -249,32 +250,32 @@ func TestFifoErrors2(t *testing.T) {
} }
func TestFifoErrorsMultiThread(t *testing.T) { func TestFifoErrorsMultiThread(t *testing.T) {
task1 := &Task{ task1 := &model.Task{
ID: "1", ID: "1",
} }
task2 := &Task{ task2 := &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
} }
task3 := &Task{ task3 := &model.Task{
ID: "3", ID: "3",
Dependencies: []string{"1", "2"}, Dependencies: []string{"1", "2"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
} }
q := New(context.Background()).(*fifo) q := New(context.Background()).(*fifo)
assert.NoError(t, q.PushAtOnce(noContext, []*Task{task2, task3, task1})) assert.NoError(t, q.PushAtOnce(noContext, []*model.Task{task2, task3, task1}))
obtainedWorkCh := make(chan *Task) obtainedWorkCh := make(chan *model.Task)
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
go func(i int) { go func(i int) {
for { for {
fmt.Printf("Worker %d started\n", i) fmt.Printf("Worker %d started\n", i)
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
obtainedWorkCh <- got obtainedWorkCh <- got
} }
}(i) }(i)
@ -298,7 +299,7 @@ func TestFifoErrorsMultiThread(t *testing.T) {
go func() { go func() {
for { for {
fmt.Printf("Worker spawned\n") fmt.Printf("Worker spawned\n")
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
obtainedWorkCh <- got obtainedWorkCh <- got
} }
}() }()
@ -308,11 +309,11 @@ func TestFifoErrorsMultiThread(t *testing.T) {
return return
} }
task2Processed = true task2Processed = true
assert.NoError(t, q.Done(noContext, got.ID, StatusSuccess)) assert.NoError(t, q.Done(noContext, got.ID, model.StatusSuccess))
go func() { go func() {
for { for {
fmt.Printf("Worker spawned\n") fmt.Printf("Worker spawned\n")
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
obtainedWorkCh <- got obtainedWorkCh <- got
} }
}() }()
@ -339,33 +340,33 @@ func TestFifoErrorsMultiThread(t *testing.T) {
} }
func TestFifoTransitiveErrors(t *testing.T) { func TestFifoTransitiveErrors(t *testing.T) {
task1 := &Task{ task1 := &model.Task{
ID: "1", ID: "1",
} }
task2 := &Task{ task2 := &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
} }
task3 := &Task{ task3 := &model.Task{
ID: "3", ID: "3",
Dependencies: []string{"2"}, Dependencies: []string{"2"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
} }
q := New(context.Background()).(*fifo) q := New(context.Background()).(*fifo)
assert.NoError(t, q.PushAtOnce(noContext, []*Task{task2, task3, task1})) assert.NoError(t, q.PushAtOnce(noContext, []*model.Task{task2, task3, task1}))
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
if got != task1 { if got != task1 {
t.Errorf("expect task1 returned from queue as task2 depends on it") t.Errorf("expect task1 returned from queue as task2 depends on it")
return return
} }
assert.NoError(t, q.Error(noContext, got.ID, fmt.Errorf("exitcode 1, there was an error"))) assert.NoError(t, q.Error(noContext, got.ID, fmt.Errorf("exitcode 1, there was an error")))
got, _ = q.Poll(noContext, func(*Task) bool { return true }) got, _ = q.Poll(noContext, func(*model.Task) bool { return true })
if got != task2 { if got != task2 {
t.Errorf("expect task2 returned from queue") t.Errorf("expect task2 returned from queue")
return return
@ -374,9 +375,9 @@ func TestFifoTransitiveErrors(t *testing.T) {
t.Errorf("expect task2 should not run, since task1 failed") t.Errorf("expect task2 should not run, since task1 failed")
return return
} }
assert.NoError(t, q.Done(noContext, got.ID, StatusSkipped)) assert.NoError(t, q.Done(noContext, got.ID, model.StatusSkipped))
got, _ = q.Poll(noContext, func(*Task) bool { return true }) got, _ = q.Poll(noContext, func(*model.Task) bool { return true })
if got != task3 { if got != task3 {
t.Errorf("expect task3 returned from queue") t.Errorf("expect task3 returned from queue")
return return
@ -388,27 +389,27 @@ func TestFifoTransitiveErrors(t *testing.T) {
} }
func TestFifoCancel(t *testing.T) { func TestFifoCancel(t *testing.T) {
task1 := &Task{ task1 := &model.Task{
ID: "1", ID: "1",
} }
task2 := &Task{ task2 := &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
} }
task3 := &Task{ task3 := &model.Task{
ID: "3", ID: "3",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
RunOn: []string{"success", "failure"}, RunOn: []string{"success", "failure"},
} }
q := New(context.Background()).(*fifo) q := New(context.Background()).(*fifo)
assert.NoError(t, q.PushAtOnce(noContext, []*Task{task2, task3, task1})) assert.NoError(t, q.PushAtOnce(noContext, []*model.Task{task2, task3, task1}))
_, _ = q.Poll(noContext, func(*Task) bool { return true }) _, _ = q.Poll(noContext, func(*model.Task) bool { return true })
assert.NoError(t, q.Error(noContext, task1.ID, fmt.Errorf("canceled"))) assert.NoError(t, q.Error(noContext, task1.ID, fmt.Errorf("canceled")))
assert.NoError(t, q.Error(noContext, task2.ID, fmt.Errorf("canceled"))) assert.NoError(t, q.Error(noContext, task2.ID, fmt.Errorf("canceled")))
assert.NoError(t, q.Error(noContext, task3.ID, fmt.Errorf("canceled"))) assert.NoError(t, q.Error(noContext, task3.ID, fmt.Errorf("canceled")))
@ -421,7 +422,7 @@ func TestFifoCancel(t *testing.T) {
} }
func TestFifoPause(t *testing.T) { func TestFifoPause(t *testing.T) {
task1 := &Task{ task1 := &model.Task{
ID: "1", ID: "1",
} }
@ -429,7 +430,7 @@ func TestFifoPause(t *testing.T) {
var wg sync.WaitGroup var wg sync.WaitGroup
wg.Add(1) wg.Add(1)
go func() { go func() {
_, _ = q.Poll(noContext, func(*Task) bool { return true }) _, _ = q.Poll(noContext, func(*model.Task) bool { return true })
wg.Done() wg.Done()
}() }()
@ -449,11 +450,11 @@ func TestFifoPause(t *testing.T) {
q.Pause() q.Pause()
assert.NoError(t, q.Push(noContext, task1)) assert.NoError(t, q.Push(noContext, task1))
q.Resume() q.Resume()
_, _ = q.Poll(noContext, func(*Task) bool { return true }) _, _ = q.Poll(noContext, func(*model.Task) bool { return true })
} }
func TestFifoPauseResume(t *testing.T) { func TestFifoPauseResume(t *testing.T) {
task1 := &Task{ task1 := &model.Task{
ID: "1", ID: "1",
} }
@ -462,31 +463,31 @@ func TestFifoPauseResume(t *testing.T) {
assert.NoError(t, q.Push(noContext, task1)) assert.NoError(t, q.Push(noContext, task1))
q.Resume() q.Resume()
_, _ = q.Poll(noContext, func(*Task) bool { return true }) _, _ = q.Poll(noContext, func(*model.Task) bool { return true })
} }
func TestWaitingVsPending(t *testing.T) { func TestWaitingVsPending(t *testing.T) {
task1 := &Task{ task1 := &model.Task{
ID: "1", ID: "1",
} }
task2 := &Task{ task2 := &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
} }
task3 := &Task{ task3 := &model.Task{
ID: "3", ID: "3",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: make(map[string]string), DepStatus: make(map[string]model.StatusValue),
RunOn: []string{"success", "failure"}, RunOn: []string{"success", "failure"},
} }
q := New(context.Background()).(*fifo) q := New(context.Background()).(*fifo)
assert.NoError(t, q.PushAtOnce(noContext, []*Task{task2, task3, task1})) assert.NoError(t, q.PushAtOnce(noContext, []*model.Task{task2, task3, task1}))
got, _ := q.Poll(noContext, func(*Task) bool { return true }) got, _ := q.Poll(noContext, func(*model.Task) bool { return true })
info := q.Info(noContext) info := q.Info(noContext)
if info.Stats.WaitingOnDeps != 2 { if info.Stats.WaitingOnDeps != 2 {
@ -494,7 +495,7 @@ func TestWaitingVsPending(t *testing.T) {
} }
assert.NoError(t, q.Error(noContext, got.ID, fmt.Errorf("exitcode 1, there was an error"))) assert.NoError(t, q.Error(noContext, got.ID, fmt.Errorf("exitcode 1, there was an error")))
got, err := q.Poll(noContext, func(*Task) bool { return true }) got, err := q.Poll(noContext, func(*model.Task) bool { return true })
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, task2, got) assert.EqualValues(t, task2, got)
@ -508,11 +509,11 @@ func TestWaitingVsPending(t *testing.T) {
} }
func TestShouldRun(t *testing.T) { func TestShouldRun(t *testing.T) {
task := &Task{ task := &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: map[string]string{ DepStatus: map[string]model.StatusValue{
"1": StatusSuccess, "1": model.StatusSuccess,
}, },
RunOn: []string{"failure"}, RunOn: []string{"failure"},
} }
@ -521,11 +522,11 @@ func TestShouldRun(t *testing.T) {
return return
} }
task = &Task{ task = &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: map[string]string{ DepStatus: map[string]model.StatusValue{
"1": StatusSuccess, "1": model.StatusSuccess,
}, },
RunOn: []string{"failure", "success"}, RunOn: []string{"failure", "success"},
} }
@ -534,11 +535,11 @@ func TestShouldRun(t *testing.T) {
return return
} }
task = &Task{ task = &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: map[string]string{ DepStatus: map[string]model.StatusValue{
"1": StatusFailure, "1": model.StatusFailure,
}, },
} }
if task.ShouldRun() { if task.ShouldRun() {
@ -546,11 +547,11 @@ func TestShouldRun(t *testing.T) {
return return
} }
task = &Task{ task = &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: map[string]string{ DepStatus: map[string]model.StatusValue{
"1": StatusSuccess, "1": model.StatusSuccess,
}, },
RunOn: []string{"success"}, RunOn: []string{"success"},
} }
@ -559,11 +560,11 @@ func TestShouldRun(t *testing.T) {
return return
} }
task = &Task{ task = &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: map[string]string{ DepStatus: map[string]model.StatusValue{
"1": StatusFailure, "1": model.StatusFailure,
}, },
RunOn: []string{"failure"}, RunOn: []string{"failure"},
} }
@ -572,23 +573,23 @@ func TestShouldRun(t *testing.T) {
return return
} }
task = &Task{ task = &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: map[string]string{ DepStatus: map[string]model.StatusValue{
"1": StatusSkipped, "1": model.StatusSkipped,
}, },
} }
if task.ShouldRun() { if task.ShouldRun() {
t.Errorf("Tasked should not run if dependency is skipped") t.Errorf("model.Tasked should not run if dependency is skipped")
return return
} }
task = &Task{ task = &model.Task{
ID: "2", ID: "2",
Dependencies: []string{"1"}, Dependencies: []string{"1"},
DepStatus: map[string]string{ DepStatus: map[string]model.StatusValue{
"1": StatusSkipped, "1": model.StatusSkipped,
}, },
RunOn: []string{"failure"}, RunOn: []string{"failure"},
} }

View file

@ -28,18 +28,7 @@ import (
// ensures the task Queue can be restored when the system starts. // ensures the task Queue can be restored when the system starts.
func WithTaskStore(q Queue, s model.TaskStore) Queue { func WithTaskStore(q Queue, s model.TaskStore) Queue {
tasks, _ := s.TaskList() tasks, _ := s.TaskList()
var toEnqueue []*Task if err := q.PushAtOnce(context.Background(), tasks); err != nil {
for _, task := range tasks {
toEnqueue = append(toEnqueue, &Task{
ID: task.ID,
Data: task.Data,
Labels: task.Labels,
Dependencies: task.Dependencies,
RunOn: task.RunOn,
DepStatus: task.DepStatus,
})
}
if err := q.PushAtOnce(context.Background(), toEnqueue); err != nil {
log.Error().Err(err).Msg("PushAtOnce failed") log.Error().Err(err).Msg("PushAtOnce failed")
} }
return &persistentQueue{q, s} return &persistentQueue{q, s}
@ -51,15 +40,8 @@ type persistentQueue struct {
} }
// Push pushes a task to the tail of this queue. // Push pushes a task to the tail of this queue.
func (q *persistentQueue) Push(c context.Context, task *Task) error { func (q *persistentQueue) Push(c context.Context, task *model.Task) error {
if err := q.store.TaskInsert(&model.Task{ if err := q.store.TaskInsert(task); err != nil {
ID: task.ID,
Data: task.Data,
Labels: task.Labels,
Dependencies: task.Dependencies,
RunOn: task.RunOn,
DepStatus: task.DepStatus,
}); err != nil {
return err return err
} }
err := q.Queue.Push(c, task) err := q.Queue.Push(c, task)
@ -72,17 +54,10 @@ func (q *persistentQueue) Push(c context.Context, task *Task) error {
} }
// PushAtOnce pushes multiple tasks to the tail of this queue. // PushAtOnce pushes multiple tasks to the tail of this queue.
func (q *persistentQueue) PushAtOnce(c context.Context, tasks []*Task) error { func (q *persistentQueue) PushAtOnce(c context.Context, tasks []*model.Task) error {
// TODO: invent store.NewSession who return context including a session and make TaskInsert & TaskDelete use it // TODO: invent store.NewSession who return context including a session and make TaskInsert & TaskDelete use it
for _, task := range tasks { for _, task := range tasks {
if err := q.store.TaskInsert(&model.Task{ if err := q.store.TaskInsert(task); err != nil {
ID: task.ID,
Data: task.Data,
Labels: task.Labels,
Dependencies: task.Dependencies,
RunOn: task.RunOn,
DepStatus: task.DepStatus,
}); err != nil {
return err return err
} }
} }
@ -98,7 +73,7 @@ func (q *persistentQueue) PushAtOnce(c context.Context, tasks []*Task) error {
} }
// Poll retrieves and removes a task head of this queue. // Poll retrieves and removes a task head of this queue.
func (q *persistentQueue) Poll(c context.Context, f FilterFn) (*Task, error) { func (q *persistentQueue) Poll(c context.Context, f FilterFn) (*model.Task, error) {
task, err := q.Queue.Poll(c, f) task, err := q.Queue.Poll(c, f)
if task != nil { if task != nil {
log.Debug().Msgf("pull queue item: %s: remove from backup", task.ID) log.Debug().Msgf("pull queue item: %s: remove from backup", task.ID)

View file

@ -3,7 +3,6 @@ package queue
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"strings" "strings"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
@ -17,87 +16,11 @@ var (
ErrNotFound = errors.New("queue: task not found") ErrNotFound = errors.New("queue: task not found")
) )
// Task defines a unit of work in the queue.
type Task struct {
// ID identifies this task.
ID string `json:"id,omitempty"`
// Data is the actual data in the entry.
Data []byte `json:"data"`
// Labels represents the key-value pairs the entry is labeled with.
Labels map[string]string `json:"labels,omitempty"`
// Task IDs this task depend
Dependencies []string
// Dependency's exit status
DepStatus map[string]string
// RunOn failure or success
RunOn []string
}
// ShouldRun tells if a task should be run or skipped, based on dependencies
func (t *Task) ShouldRun() bool {
if runsOnFailure(t.RunOn) && runsOnSuccess(t.RunOn) {
return true
}
if !runsOnFailure(t.RunOn) && runsOnSuccess(t.RunOn) {
for _, status := range t.DepStatus {
if StatusSuccess != status {
return false
}
}
return true
}
if runsOnFailure(t.RunOn) && !runsOnSuccess(t.RunOn) {
for _, status := range t.DepStatus {
if StatusSuccess == status {
return false
}
}
return true
}
return false
}
func (t *Task) String() string {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("%s (%s) - %s", t.ID, t.Dependencies, t.DepStatus))
return sb.String()
}
func runsOnFailure(runsOn []string) bool {
for _, status := range runsOn {
if status == "failure" {
return true
}
}
return false
}
func runsOnSuccess(runsOn []string) bool {
if len(runsOn) == 0 {
return true
}
for _, status := range runsOn {
if status == "success" {
return true
}
}
return false
}
// InfoT provides runtime information. // InfoT provides runtime information.
type InfoT struct { type InfoT struct {
Pending []*Task `json:"pending"` Pending []*model.Task `json:"pending"`
WaitingOnDeps []*Task `json:"waiting_on_deps"` WaitingOnDeps []*model.Task `json:"waiting_on_deps"`
Running []*Task `json:"running"` Running []*model.Task `json:"running"`
Stats struct { Stats struct {
Workers int `json:"worker_count"` Workers int `json:"worker_count"`
Pending int `json:"pending_count"` Pending int `json:"pending_count"`
@ -105,7 +28,7 @@ type InfoT struct {
Running int `json:"running_count"` Running int `json:"running_count"`
Complete int `json:"completed_count"` Complete int `json:"completed_count"`
} `json:"stats"` } `json:"stats"`
Paused bool Paused bool `json:"paused"`
} }
func (t *InfoT) String() string { func (t *InfoT) String() string {
@ -128,19 +51,19 @@ func (t *InfoT) String() string {
// Filter filters tasks in the queue. If the Filter returns false, // Filter filters tasks in the queue. If the Filter returns false,
// the Task is skipped and not returned to the subscriber. // the Task is skipped and not returned to the subscriber.
type FilterFn func(*Task) bool type FilterFn func(*model.Task) bool
// Queue defines a task queue for scheduling tasks among // Queue defines a task queue for scheduling tasks among
// a pool of workers. // a pool of workers.
type Queue interface { type Queue interface {
// Push pushes a task to the tail of this queue. // Push pushes a task to the tail of this queue.
Push(c context.Context, task *Task) error Push(c context.Context, task *model.Task) error
// PushAtOnce pushes a task to the tail of this queue. // PushAtOnce pushes a task to the tail of this queue.
PushAtOnce(c context.Context, tasks []*Task) error PushAtOnce(c context.Context, tasks []*model.Task) error
// Poll retrieves and removes a task head of this queue. // Poll retrieves and removes a task head of this queue.
Poll(c context.Context, f FilterFn) (*Task, error) Poll(c context.Context, f FilterFn) (*model.Task, error)
// Extend extends the deadline for a task. // Extend extends the deadline for a task.
Extend(c context.Context, id string) error Extend(c context.Context, id string) error

View file

@ -145,8 +145,8 @@ func apiRoutes(e *gin.Engine) {
{ {
queue.Use(session.MustAdmin()) queue.Use(session.MustAdmin())
queue.GET("/info", api.GetQueueInfo) queue.GET("/info", api.GetQueueInfo)
queue.GET("/pause", api.PauseQueue) queue.POST("/pause", api.PauseQueue)
queue.GET("/resume", api.ResumeQueue) queue.POST("/resume", api.ResumeQueue)
queue.GET("/norunningpipelines", api.BlockTilQueueHasRunningItem) queue.GET("/norunningpipelines", api.BlockTilQueueHasRunningItem)
} }

View file

@ -30,7 +30,7 @@ func TestTaskList(t *testing.T) {
ID: "some_random_id", ID: "some_random_id",
Data: []byte("foo"), Data: []byte("foo"),
Labels: map[string]string{"foo": "bar"}, Labels: map[string]string{"foo": "bar"},
DepStatus: map[string]string{"test": "dep"}, DepStatus: map[string]model.StatusValue{"test": "dep"},
})) }))
list, err := store.TaskList() list, err := store.TaskList()
@ -41,7 +41,7 @@ func TestTaskList(t *testing.T) {
assert.Len(t, list, 1, "Expected one task in list") assert.Len(t, list, 1, "Expected one task in list")
assert.Equal(t, "some_random_id", list[0].ID) assert.Equal(t, "some_random_id", list[0].ID)
assert.Equal(t, "foo", string(list[0].Data)) assert.Equal(t, "foo", string(list[0].Data))
assert.EqualValues(t, map[string]string{"test": "dep"}, list[0].DepStatus) assert.EqualValues(t, map[string]model.StatusValue{"test": "dep"}, list[0].DepStatus)
err = store.TaskDelete("some_random_id") err = store.TaskDelete("some_random_id")
if err != nil { if err != nil {

3
web/components.d.ts vendored
View file

@ -12,6 +12,7 @@ declare module '@vue/runtime-core' {
ActionsTab: typeof import('./src/components/repo/settings/ActionsTab.vue')['default'] ActionsTab: typeof import('./src/components/repo/settings/ActionsTab.vue')['default']
ActivePipelines: typeof import('./src/components/layout/header/ActivePipelines.vue')['default'] ActivePipelines: typeof import('./src/components/layout/header/ActivePipelines.vue')['default']
AdminAgentsTab: typeof import('./src/components/admin/settings/AdminAgentsTab.vue')['default'] AdminAgentsTab: typeof import('./src/components/admin/settings/AdminAgentsTab.vue')['default']
AdminQueueTab: typeof import('./src/components/admin/settings/AdminQueueTab.vue')['default']
AdminSecretsTab: typeof import('./src/components/admin/settings/AdminSecretsTab.vue')['default'] AdminSecretsTab: typeof import('./src/components/admin/settings/AdminSecretsTab.vue')['default']
AdminUsersTab: typeof import('./src/components/admin/settings/AdminUsersTab.vue')['default'] AdminUsersTab: typeof import('./src/components/admin/settings/AdminUsersTab.vue')['default']
Badge: typeof import('./src/components/atomic/Badge.vue')['default'] Badge: typeof import('./src/components/atomic/Badge.vue')['default']
@ -19,6 +20,7 @@ declare module '@vue/runtime-core' {
Button: typeof import('./src/components/atomic/Button.vue')['default'] Button: typeof import('./src/components/atomic/Button.vue')['default']
Checkbox: typeof import('./src/components/form/Checkbox.vue')['default'] Checkbox: typeof import('./src/components/form/Checkbox.vue')['default']
CheckboxesField: typeof import('./src/components/form/CheckboxesField.vue')['default'] CheckboxesField: typeof import('./src/components/form/CheckboxesField.vue')['default']
copy: typeof import('./src/components/admin/settings/AdminAgentsTab copy.vue')['default']
CronTab: typeof import('./src/components/repo/settings/CronTab.vue')['default'] CronTab: typeof import('./src/components/repo/settings/CronTab.vue')['default']
DeployPipelinePopup: typeof import('./src/components/layout/popups/DeployPipelinePopup.vue')['default'] DeployPipelinePopup: typeof import('./src/components/layout/popups/DeployPipelinePopup.vue')['default']
DocsLink: typeof import('./src/components/atomic/DocsLink.vue')['default'] DocsLink: typeof import('./src/components/atomic/DocsLink.vue')['default']
@ -45,6 +47,7 @@ declare module '@vue/runtime-core' {
IIcBaselineFileDownload: typeof import('~icons/ic/baseline-file-download')['default'] IIcBaselineFileDownload: typeof import('~icons/ic/baseline-file-download')['default']
IIcBaselineFileDownloadOff: typeof import('~icons/ic/baseline-file-download-off')['default'] IIcBaselineFileDownloadOff: typeof import('~icons/ic/baseline-file-download-off')['default']
IIcBaselineHealing: typeof import('~icons/ic/baseline-healing')['default'] IIcBaselineHealing: typeof import('~icons/ic/baseline-healing')['default']
IIcBaselinePause: typeof import('~icons/ic/baseline-pause')['default']
IIcBaselinePlayArrow: typeof import('~icons/ic/baseline-play-arrow')['default'] IIcBaselinePlayArrow: typeof import('~icons/ic/baseline-play-arrow')['default']
IIconoirArrowLeft: typeof import('~icons/iconoir/arrow-left')['default'] IIconoirArrowLeft: typeof import('~icons/iconoir/arrow-left')['default']
IIconParkOutlineAlarmClock: typeof import('~icons/icon-park-outline/alarm-clock')['default'] IIconParkOutlineAlarmClock: typeof import('~icons/icon-park-outline/alarm-clock')['default']

View file

@ -366,6 +366,18 @@
"never": "Never", "never": "Never",
"delete_confirm": "Do you really want to delete this agent? It wont be able to connected to the server anymore." "delete_confirm": "Do you really want to delete this agent? It wont be able to connected to the server anymore."
}, },
"queue": {
"queue": "Queue",
"desc": "Tasks waiting to be executed by agents",
"pause": "Pause",
"resume": "Resume",
"paused": "Queue is paused",
"resumed": "Queue is resumed",
"tasks": "Tasks",
"task_running": "Task is running",
"task_pending": "Task is pending",
"task_waiting_on_deps": "Task is waiting on dependencies"
},
"users": { "users": {
"users": "Users", "users": "Users",
"desc": "Users registered for this server", "desc": "Users registered for this server",

View file

@ -0,0 +1,147 @@
<template>
<Panel>
<div v-if="queueInfo" class="flex flex-row border-b mb-4 pb-4 items-center dark:border-gray-600">
<div class="ml-2">
<h1 class="text-xl text-color">{{ $t('admin.settings.queue.queue') }}</h1>
<p class="text-sm text-color-alt">{{ $t('admin.settings.queue.desc') }}</p>
</div>
<div class="ml-auto flex items-center gap-2">
<Button
v-if="queueInfo.paused"
class="ml-auto"
:text="$t('admin.settings.queue.resume')"
start-icon="play"
@click="resumeQueue"
/>
<Button
v-else
class="ml-auto"
:text="$t('admin.settings.queue.pause')"
start-icon="pause"
@click="pauseQueue"
/>
<Icon
:name="queueInfo.paused ? 'pause' : 'play'"
:class="{
'text-red-400': queueInfo.paused,
'text-lime-400': !queueInfo.paused,
}"
/>
</div>
</div>
<div class="flex flex-col">
<pre>{{ queueInfo?.stats }}</pre>
<div v-if="tasks.length > 0" class="flex flex-col">
<p class="mt-6 mb-2 text-xl">{{ $t('admin.settings.queue.tasks') }}</p>
<ListItem v-for="task in tasks" :key="task.id" class="items-center mb-2">
<div
class="flex items-center"
:title="
task.status === 'pending'
? $t('admin.settings.queue.task_pending')
: task.status === 'running'
? $t('admin.settings.queue.task_running')
: $t('admin.settings.queue.task_waiting_on_deps')
"
>
<Icon
:name="
task.status === 'pending'
? 'status-pending'
: task.status === 'running'
? 'status-running'
: 'status-declined'
"
:class="{
'text-red-400': task.status === 'waiting_on_deps',
'text-lime-400': task.status === 'running',
'text-blue-400': task.status === 'pending',
}"
/>
</div>
<span class="ml-2">{{ task.id }}</span>
<span class="flex ml-auto gap-2">
<span>{{ task.labels }}</span>
<span>{{ task.dependencies }}</span>
</span>
</ListItem>
</div>
</div>
</Panel>
</template>
<script lang="ts" setup>
import { computed, onBeforeUnmount, onMounted, ref } from 'vue';
import { useI18n } from 'vue-i18n';
import Button from '~/components/atomic/Button.vue';
import Icon from '~/components/atomic/Icon.vue';
import ListItem from '~/components/atomic/ListItem.vue';
import Panel from '~/components/layout/Panel.vue';
import useApiClient from '~/compositions/useApiClient';
import useNotifications from '~/compositions/useNotifications';
import { QueueInfo } from '~/lib/api/types/queue';
const apiClient = useApiClient();
const notifications = useNotifications();
const { t } = useI18n();
const queueInfo = ref<QueueInfo>();
const tasks = computed(() => {
const _tasks = [];
if (queueInfo.value?.running) {
_tasks.push(...queueInfo.value.running.map((task) => ({ ...task, status: 'running' })));
}
if (queueInfo.value?.pending) {
_tasks.push(...queueInfo.value.pending.map((task) => ({ ...task, status: 'pending' })));
}
if (queueInfo.value?.waiting_on_deps) {
_tasks.push(...queueInfo.value.waiting_on_deps.map((task) => ({ ...task, status: 'waiting_on_deps' })));
}
return _tasks.sort((a, b) => a.id - b.id);
});
async function loadQueueInfo() {
queueInfo.value = await apiClient.getQueueInfo();
}
async function pauseQueue() {
await apiClient.pauseQueue();
await loadQueueInfo();
notifications.notify({
title: t('admin.settings.queue.paused'),
type: 'success',
});
}
async function resumeQueue() {
await apiClient.resumeQueue();
await loadQueueInfo();
notifications.notify({
title: t('admin.settings.queue.resumed'),
type: 'success',
});
}
const reloadInterval = ref<number>();
onMounted(async () => {
await loadQueueInfo();
reloadInterval.value = window.setInterval(async () => {
await loadQueueInfo();
}, 5000);
});
onBeforeUnmount(() => {
if (reloadInterval.value) {
window.clearInterval(reloadInterval.value);
}
});
</script>

View file

@ -43,6 +43,7 @@
<i-ic-baseline-file-download-off v-else-if="name === 'auto-scroll-off'" class="h-6 w-6" /> <i-ic-baseline-file-download-off v-else-if="name === 'auto-scroll-off'" class="h-6 w-6" />
<i-teenyicons-refresh-outline v-else-if="name === 'refresh'" class="h-6 w-6" /> <i-teenyicons-refresh-outline v-else-if="name === 'refresh'" class="h-6 w-6" />
<i-ic-baseline-play-arrow v-else-if="name === 'play'" class="h-6 w-6" /> <i-ic-baseline-play-arrow v-else-if="name === 'play'" class="h-6 w-6" />
<i-ic-baseline-pause v-else-if="name === 'pause'" class="h-6 w-6" />
<div v-else-if="name === 'blank'" class="h-6 w-6" /> <div v-else-if="name === 'blank'" class="h-6 w-6" />
</template> </template>
@ -92,7 +93,8 @@ export type IconNames =
| 'auto-scroll' | 'auto-scroll'
| 'auto-scroll-off' | 'auto-scroll-off'
| 'refresh' | 'refresh'
| 'play'; | 'play'
| 'pause';
defineProps<{ defineProps<{
name: IconNames; name: IconNames;

View file

@ -9,6 +9,7 @@ import {
PipelineLog, PipelineLog,
PipelineStep, PipelineStep,
PullRequest, PullRequest,
QueueInfo,
Registry, Registry,
Repo, Repo,
RepoPermissions, RepoPermissions,
@ -256,6 +257,18 @@ export default class WoodpeckerClient extends ApiClient {
return this._delete(`/api/agents/${agent.id}`); return this._delete(`/api/agents/${agent.id}`);
} }
getQueueInfo(): Promise<QueueInfo> {
return this._get('/api/queue/info') as Promise<QueueInfo>;
}
pauseQueue(): Promise<unknown> {
return this._post('/api/queue/pause');
}
resumeQueue(): Promise<unknown> {
return this._post('/api/queue/resume');
}
getUsers(): Promise<User[]> { getUsers(): Promise<User[]> {
return this._get('/api/users') as Promise<User[]>; return this._get('/api/users') as Promise<User[]>;
} }

View file

@ -4,6 +4,7 @@ export * from './org';
export * from './pipeline'; export * from './pipeline';
export * from './pipelineConfig'; export * from './pipelineConfig';
export * from './pull_request'; export * from './pull_request';
export * from './queue';
export * from './registry'; export * from './registry';
export * from './repo'; export * from './repo';
export * from './secret'; export * from './secret';

View file

@ -0,0 +1,22 @@
export type Task = {
id: number;
data: string;
labels: { [key: string]: string };
dependencies: string[];
dep_status: { [key: string]: string };
run_on: string[];
};
export type QueueInfo = {
pending: Task[];
waiting_on_deps: Task[];
running: Task[];
stats: {
worker_count: number;
pending_count: number;
waiting_on_deps_count: number;
running_count: number;
completed_count: number;
};
paused: boolean;
};

View file

@ -12,6 +12,9 @@
<Tab id="agents" :title="$t('admin.settings.agents.agents')"> <Tab id="agents" :title="$t('admin.settings.agents.agents')">
<AdminAgentsTab /> <AdminAgentsTab />
</Tab> </Tab>
<Tab id="queue" :title="$t('admin.settings.queue.queue')">
<AdminQueueTab />
</Tab>
</Scaffold> </Scaffold>
</template> </template>
@ -21,6 +24,7 @@ import { useI18n } from 'vue-i18n';
import { useRouter } from 'vue-router'; import { useRouter } from 'vue-router';
import AdminAgentsTab from '~/components/admin/settings/AdminAgentsTab.vue'; import AdminAgentsTab from '~/components/admin/settings/AdminAgentsTab.vue';
import AdminQueueTab from '~/components/admin/settings/AdminQueueTab.vue';
import AdminSecretsTab from '~/components/admin/settings/AdminSecretsTab.vue'; import AdminSecretsTab from '~/components/admin/settings/AdminSecretsTab.vue';
import AdminUsersTab from '~/components/admin/settings/AdminUsersTab.vue'; import AdminUsersTab from '~/components/admin/settings/AdminUsersTab.vue';
import Scaffold from '~/components/layout/scaffold/Scaffold.vue'; import Scaffold from '~/components/layout/scaffold/Scaffold.vue';