mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-11-26 03:41:01 +00:00
Code Issues Projects Releases Wiki Activity

Split and refactor (#1394)

Browse source 
Closes #974
This commit is contained in:
6543 2022-11-06 12:44:04 +01:00 committed by GitHub
parent e901f605b1
commit 18311d4360
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 258 additions and 239 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
server/shared/stepBuilder.go → pipeline/stepBuilder.go
View file

@ -13,14 +13,13 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package shared package pipeline
import ( import (
"fmt" "fmt"
"math/rand" "math/rand"
"net/url" "net/url"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"github.com/drone/envsubst" "github.com/drone/envsubst"
@ -33,12 +32,10 @@ import (
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/linter" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/linter"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/matrix" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/matrix"
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
// TODO(974) move to pipeline/*
// StepBuilder Takes the hook data and the yaml and returns in internal data model // StepBuilder Takes the hook data and the yaml and returns in internal data model
type StepBuilder struct { type StepBuilder struct {
Repo *model.Repo Repo *model.Repo
@ -48,11 +45,11 @@ type StepBuilder struct {
Secs []*model.Secret Secs []*model.Secret
Regs []*model.Registry Regs []*model.Registry
Link string Link string
Yamls []*forge.FileMeta Yamls []*forge_types.FileMeta
Envs map[string]string Envs map[string]string
} }
type PipelineItem struct { type Item struct {
Step *model.Step Step *model.Step
Platform string Platform string
Labels map[string]string Labels map[string]string
@ -61,10 +58,10 @@ type PipelineItem struct {
Config *backend.Config Config *backend.Config
} }
func (b *StepBuilder) Build() ([]*PipelineItem, error) { func (b *StepBuilder) Build() ([]*Item, error) {
var items []*PipelineItem var items []*Item
sort.Sort(forge.ByName(b.Yamls)) b.Yamls = forge_types.SortByName(b.Yamls)
pidSequence := 1 pidSequence := 1
@ -149,7 +146,7 @@ func (b *StepBuilder) Build() ([]*PipelineItem, error) {
continue continue
} }
item := &PipelineItem{ item := &Item{
Step: step, Step: step,
Config: ir, Config: ir,
Labels: parsed.Labels, Labels: parsed.Labels,
@ -176,7 +173,7 @@ func (b *StepBuilder) Build() ([]*PipelineItem, error) {
return items, nil return items, nil
} }
func stepListContainsItemsToRun(items []*PipelineItem) bool { func stepListContainsItemsToRun(items []*Item) bool {
for i := range items { for i := range items {
if items[i].Step.State == model.StatusPending { if items[i].Step.State == model.StatusPending {
return true return true
@ -185,8 +182,8 @@ func stepListContainsItemsToRun(items []*PipelineItem) bool {
return false return false
} }
func filterItemsWithMissingDependencies(items []*PipelineItem) []*PipelineItem { func filterItemsWithMissingDependencies(items []*Item) []*Item {
itemsToRemove := make([]*PipelineItem, 0) itemsToRemove := make([]*Item, 0)
for _, item := range items { for _, item := range items {
for _, dep := range item.DependsOn { for _, dep := range item.DependsOn {
@ -197,7 +194,7 @@ func filterItemsWithMissingDependencies(items []*PipelineItem) []*PipelineItem {
} }
if len(itemsToRemove) > 0 { if len(itemsToRemove) > 0 {
filtered := make([]*PipelineItem, 0) filtered := make([]*Item, 0)
for _, item := range items { for _, item := range items {
if !containsItemWithName(item.Step.Name, itemsToRemove) { if !containsItemWithName(item.Step.Name, itemsToRemove) {
filtered = append(filtered, item) filtered = append(filtered, item)
@ -210,7 +207,7 @@ func filterItemsWithMissingDependencies(items []*PipelineItem) []*PipelineItem {
return items return items
} }
func containsItemWithName(name string, items []*PipelineItem) bool { func containsItemWithName(name string, items []*Item) bool {
for _, item := range items { for _, item := range items {
if name == item.Step.Name { if name == item.Step.Name {
return true return true
@ -293,7 +290,7 @@ func (b *StepBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[
).Compile(parsed) ).Compile(parsed)
} }
func SetPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*PipelineItem) *model.Pipeline { func SetPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*Item) *model.Pipeline {
var pidSequence int var pidSequence int
for _, item := range pipelineItems { for _, item := range pipelineItems {
pipeline.Steps = append(pipeline.Steps, item.Step) pipeline.Steps = append(pipeline.Steps, item.Step)

32
server/shared/stepBuilder_test.go → pipeline/stepBuilder_test.go
View file

@ -13,18 +13,16 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package shared package pipeline
import ( import (
"fmt" "fmt"
"testing" "testing"
"github.com/woodpecker-ci/woodpecker/server/forge" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
// TODO(974) move to pipeline/*
func TestGlobalEnvsubst(t *testing.T) { func TestGlobalEnvsubst(t *testing.T) {
t.Parallel() t.Parallel()
@ -42,7 +40,7 @@ func TestGlobalEnvsubst(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Data: []byte(` {Data: []byte(`
pipeline: pipeline:
build: build:
@ -76,7 +74,7 @@ func TestMissingGlobalEnvsubst(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Data: []byte(` {Data: []byte(`
pipeline: pipeline:
build: build:
@ -107,7 +105,7 @@ bbb`,
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Data: []byte(` {Data: []byte(`
pipeline: pipeline:
xxx: xxx:
@ -141,7 +139,7 @@ func TestMultiPipeline(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Data: []byte(` {Data: []byte(`
pipeline: pipeline:
xxx: xxx:
@ -175,7 +173,7 @@ func TestDependsOn(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Name: "lint", Data: []byte(` {Name: "lint", Data: []byte(`
pipeline: pipeline:
build: build:
@ -221,7 +219,7 @@ func TestRunsOn(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Data: []byte(` {Data: []byte(`
pipeline: pipeline:
deploy: deploy:
@ -257,7 +255,7 @@ func TestPipelineName(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Name: ".woodpecker/lint.yml", Data: []byte(` {Name: ".woodpecker/lint.yml", Data: []byte(`
pipeline: pipeline:
build: build:
@ -292,7 +290,7 @@ func TestBranchFilter(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Data: []byte(` {Data: []byte(`
pipeline: pipeline:
xxx: xxx:
@ -338,7 +336,7 @@ func TestRootWhenFilter(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Data: []byte(` {Data: []byte(`
when: when:
event: event:
@ -386,7 +384,7 @@ func TestZeroSteps(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Data: []byte(` {Data: []byte(`
skip_clone: true skip_clone: true
pipeline: pipeline:
@ -420,7 +418,7 @@ func TestZeroStepsAsMultiPipelineDeps(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Name: "zerostep", Data: []byte(` {Name: "zerostep", Data: []byte(`
skip_clone: true skip_clone: true
pipeline: pipeline:
@ -468,7 +466,7 @@ func TestZeroStepsAsMultiPipelineTransitiveDeps(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Name: "zerostep", Data: []byte(` {Name: "zerostep", Data: []byte(`
skip_clone: true skip_clone: true
pipeline: pipeline:
@ -524,7 +522,7 @@ func TestTree(t *testing.T) {
Secs: []*model.Secret{}, Secs: []*model.Secret{},
Regs: []*model.Registry{}, Regs: []*model.Registry{},
Link: "", Link: "",
Yamls: []*forge.FileMeta{ Yamls: []*forge_types.FileMeta{
{Data: []byte(` {Data: []byte(`
pipeline: pipeline:
build: build:

18
server/api/user.go
View file

@ -25,9 +25,9 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/router/middleware/session" "github.com/woodpecker-ci/woodpecker/server/router/middleware/session"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
"github.com/woodpecker-ci/woodpecker/shared/token" "github.com/woodpecker-ci/woodpecker/shared/token"
) )
@ -38,7 +38,7 @@ func GetSelf(c *gin.Context) {
func GetFeed(c *gin.Context) { func GetFeed(c *gin.Context) {
_store := store.FromContext(c) _store := store.FromContext(c)
forge := server.Config.Services.Forge _forge := server.Config.Services.Forge
user := session.User(c) user := session.User(c)
latest, _ := strconv.ParseBool(c.Query("latest")) latest, _ := strconv.ParseBool(c.Query("latest"))
@ -54,11 +54,11 @@ func GetFeed(c *gin.Context) {
config := ToConfig(c) config := ToConfig(c)
sync := shared.Syncer{ sync := forge.Syncer{
Forge: forge, Forge: _forge,
Store: _store, Store: _store,
Perms: _store, Perms: _store,
Match: shared.NamespaceFilter(config.OwnersWhitelist), Match: forge.NamespaceFilter(config.OwnersWhitelist),
} }
if err := sync.Sync(c, user, server.Config.FlatPermissions); err != nil { if err := sync.Sync(c, user, server.Config.FlatPermissions); err != nil {
log.Debug().Msgf("sync error: %s: %s", user.Login, err) log.Debug().Msgf("sync error: %s: %s", user.Login, err)
@ -87,7 +87,7 @@ func GetFeed(c *gin.Context) {
func GetRepos(c *gin.Context) { func GetRepos(c *gin.Context) {
_store := store.FromContext(c) _store := store.FromContext(c)
forge := server.Config.Services.Forge _forge := server.Config.Services.Forge
user := session.User(c) user := session.User(c)
all, _ := strconv.ParseBool(c.Query("all")) all, _ := strconv.ParseBool(c.Query("all"))
@ -103,11 +103,11 @@ func GetRepos(c *gin.Context) {
config := ToConfig(c) config := ToConfig(c)
sync := shared.Syncer{ sync := forge.Syncer{
Forge: forge, Forge: _forge,
Store: _store, Store: _store,
Perms: _store, Perms: _store,
Match: shared.NamespaceFilter(config.OwnersWhitelist), Match: forge.NamespaceFilter(config.OwnersWhitelist),
} }
if err := sync.Sync(c, user, server.Config.FlatPermissions); err != nil { if err := sync.Sync(c, user, server.Config.FlatPermissions); err != nil {

9
server/forge/bitbucket/bitbucket.go
View file

@ -27,6 +27,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/bitbucket/internal" "github.com/woodpecker-ci/woodpecker/server/forge/bitbucket/internal"
"github.com/woodpecker-ci/woodpecker/server/forge/common" "github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
@ -73,7 +74,7 @@ func (c *config) Login(ctx context.Context, w http.ResponseWriter, req *http.Req
// get the OAuth errors // get the OAuth errors
if err := req.FormValue("error"); err != "" { if err := req.FormValue("error"); err != "" {
return nil, &forge.AuthError{ return nil, &forge_types.AuthError{
Err: err, Err: err,
Description: req.FormValue("error_description"), Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"), URI: req.FormValue("error_uri"),
@ -222,8 +223,8 @@ func (c *config) File(ctx context.Context, u *model.User, r *model.Repo, p *mode
return []byte(*config), err return []byte(*config), err
} }
func (c *config) Dir(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]*forge.FileMeta, error) { func (c *config) Dir(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
return nil, fmt.Errorf("Not implemented") return nil, forge_types.ErrNotImplemented
} }
// Status creates a pipeline status for the Bitbucket commit. // Status creates a pipeline status for the Bitbucket commit.
@ -298,7 +299,7 @@ func (c *config) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]
// BranchHead returns the sha of the head (lastest commit) of the specified branch // BranchHead returns the sha of the head (lastest commit) of the specified branch
func (c *config) BranchHead(ctx context.Context, u *model.User, r *model.Repo, branch string) (string, error) { func (c *config) BranchHead(ctx context.Context, u *model.User, r *model.Repo, branch string) (string, error) {
// TODO(1138): missing implementation // TODO(1138): missing implementation
return "", fmt.Errorf("missing implementation") return "", forge_types.ErrNotImplemented
} }
// Hook parses the incoming Bitbucket hook and returns the Repository and // Hook parses the incoming Bitbucket hook and returns the Repository and

7
server/forge/bitbucketserver/bitbucketserver.go
View file

@ -33,6 +33,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/bitbucketserver/internal" "github.com/woodpecker-ci/woodpecker/server/forge/bitbucketserver/internal"
"github.com/woodpecker-ci/woodpecker/server/forge/common" "github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
@ -185,8 +186,8 @@ func (c *Config) File(ctx context.Context, u *model.User, r *model.Repo, p *mode
return client.FindFileForRepo(r.Owner, r.Name, f, p.Ref) return client.FindFileForRepo(r.Owner, r.Name, f, p.Ref)
} }
func (c *Config) Dir(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]*forge.FileMeta, error) { func (c *Config) Dir(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
return nil, fmt.Errorf("Not implemented") return nil, forge_types.ErrNotImplemented
} }
// Status is not supported by the bitbucketserver driver. // Status is not supported by the bitbucketserver driver.
@ -240,7 +241,7 @@ func (c *Config) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]
// BranchHead returns the sha of the head (lastest commit) of the specified branch // BranchHead returns the sha of the head (lastest commit) of the specified branch
func (c *Config) BranchHead(ctx context.Context, u *model.User, r *model.Repo, branch string) (string, error) { func (c *Config) BranchHead(ctx context.Context, u *model.User, r *model.Repo, branch string) (string, error) {
// TODO(1138): missing implementation // TODO(1138): missing implementation
return "", fmt.Errorf("missing implementation") return "", forge_types.ErrNotImplemented
} }
func (c *Config) Deactivate(ctx context.Context, u *model.User, r *model.Repo, link string) error { func (c *Config) Deactivate(ctx context.Context, u *model.User, r *model.Repo, link string) error {

11
server/forge/coding/coding.go
View file

@ -28,6 +28,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/coding/internal" "github.com/woodpecker-ci/woodpecker/server/forge/coding/internal"
"github.com/woodpecker-ci/woodpecker/server/forge/common" "github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
@ -87,7 +88,7 @@ func (c *Coding) Login(ctx context.Context, res http.ResponseWriter, req *http.R
// get the OAuth errors // get the OAuth errors
if err := req.FormValue("error"); err != "" { if err := req.FormValue("error"); err != "" {
return nil, &forge.AuthError{ return nil, &forge_types.AuthError{
Err: err, Err: err,
Description: req.FormValue("error_description"), Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"), URI: req.FormValue("error_uri"),
@ -151,7 +152,7 @@ func (c *Coding) Refresh(ctx context.Context, u *model.User) (bool, error) {
// Teams fetches a list of team memberships from the forge. // Teams fetches a list of team memberships from the forge.
func (c *Coding) Teams(ctx context.Context, u *model.User) ([]*model.Team, error) { func (c *Coding) Teams(ctx context.Context, u *model.User) ([]*model.Team, error) {
// EMPTY: not implemented in Coding OAuth API // EMPTY: not implemented in Coding OAuth API
return nil, fmt.Errorf("Not implemented") return nil, forge_types.ErrNotImplemented
} }
// TeamPerm fetches the named organization permissions from // TeamPerm fetches the named organization permissions from
@ -244,8 +245,8 @@ func (c *Coding) File(ctx context.Context, u *model.User, r *model.Repo, b *mode
return data, nil return data, nil
} }
func (c *Coding) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge.FileMeta, error) { func (c *Coding) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
return nil, fmt.Errorf("Not implemented") return nil, forge_types.ErrNotImplemented
} }
// Status sends the commit status to the forge. // Status sends the commit status to the forge.
@ -297,7 +298,7 @@ func (c *Coding) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]
// BranchHead returns the sha of the head (lastest commit) of the specified branch // BranchHead returns the sha of the head (lastest commit) of the specified branch
func (c *Coding) BranchHead(ctx context.Context, u *model.User, r *model.Repo, branch string) (string, error) { func (c *Coding) BranchHead(ctx context.Context, u *model.User, r *model.Repo, branch string) (string, error) {
// TODO(1138): missing implementation // TODO(1138): missing implementation
return "", fmt.Errorf("missing implementation") return "", forge_types.ErrNotImplemented
} }
// Hook parses the post-commit hook from the Request body and returns the // Hook parses the post-commit hook from the Request body and returns the

33
server/shared/configFetcher.go → server/forge/configFetcher.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package shared package forge
import ( import (
"context" "context"
@ -23,27 +23,25 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/plugins/config" "github.com/woodpecker-ci/woodpecker/server/plugins/config"
"github.com/woodpecker-ci/woodpecker/shared/constant" "github.com/woodpecker-ci/woodpecker/shared/constant"
) )
type ConfigFetcher interface { type ConfigFetcher interface {
Fetch(ctx context.Context) (files []*forge.FileMeta, err error) Fetch(ctx context.Context) (files []*types.FileMeta, err error)
} }
// TODO(974) move to new package
type configFetcher struct { type configFetcher struct {
forge forge.Forge forge Forge
user *model.User user *model.User
repo *model.Repo repo *model.Repo
pipeline *model.Pipeline pipeline *model.Pipeline
configExtension config.Extension configExtension config.Extension
} }
func NewConfigFetcher(forge forge.Forge, configExtension config.Extension, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ConfigFetcher { func NewConfigFetcher(forge Forge, configExtension config.Extension, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ConfigFetcher {
return &configFetcher{ return &configFetcher{
forge: forge, forge: forge,
user: user, user: user,
@ -57,7 +55,7 @@ func NewConfigFetcher(forge forge.Forge, configExtension config.Extension, user
var configFetchTimeout = time.Second * 3 var configFetchTimeout = time.Second * 3
// Fetch pipeline config from source forge // Fetch pipeline config from source forge
func (cf *configFetcher) Fetch(ctx context.Context) (files []*forge.FileMeta, err error) { func (cf *configFetcher) Fetch(ctx context.Context) (files []*types.FileMeta, err error) {
log.Trace().Msgf("Start Fetching config for '%s'", cf.repo.FullName) log.Trace().Msgf("Start Fetching config for '%s'", cf.repo.FullName)
// try to fetch 3 times // try to fetch 3 times
@ -92,7 +90,7 @@ func (cf *configFetcher) Fetch(ctx context.Context) (files []*forge.FileMeta, er
} }
// fetch config by timeout // fetch config by timeout
func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config string) ([]*forge.FileMeta, error) { func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config string) ([]*types.FileMeta, error) {
ctx, cancel := context.WithTimeout(c, timeout) ctx, cancel := context.WithTimeout(c, timeout)
defer cancel() defer cancel()
@ -121,12 +119,12 @@ func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config
case <-ctx.Done(): case <-ctx.Done():
return nil, ctx.Err() return nil, ctx.Err()
default: default:
return []*forge.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not find config: %s", err) return []*types.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not find config: %s", err)
} }
} }
func filterPipelineFiles(files []*forge.FileMeta) []*forge.FileMeta { func filterPipelineFiles(files []*types.FileMeta) []*types.FileMeta {
var res []*forge.FileMeta var res []*types.FileMeta
for _, file := range files { for _, file := range files {
if strings.HasSuffix(file.Name, ".yml") || strings.HasSuffix(file.Name, ".yaml") { if strings.HasSuffix(file.Name, ".yml") || strings.HasSuffix(file.Name, ".yaml") {
@ -137,13 +135,13 @@ func filterPipelineFiles(files []*forge.FileMeta) []*forge.FileMeta {
return res return res
} }
func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fileMeta []*forge.FileMeta, found bool) { func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fileMeta []*types.FileMeta, found bool) {
file, err := cf.forge.File(c, cf.user, cf.repo, cf.pipeline, config) file, err := cf.forge.File(c, cf.user, cf.repo, cf.pipeline, config)
if err == nil && len(file) != 0 { if err == nil && len(file) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config) log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config)
return []*forge.FileMeta{{ return []*types.FileMeta{{
Name: config, Name: config,
Data: file, Data: file,
}}, true }}, true
@ -152,7 +150,7 @@ func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fi
return nil, false return nil, false
} }
func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string, userDefined bool) ([]*forge.FileMeta, error) { func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string, userDefined bool) ([]*types.FileMeta, error) {
userDefinedLog := "" userDefinedLog := ""
if userDefined { if userDefined {
userDefinedLog = "user defined" userDefinedLog = "user defined"
@ -161,8 +159,11 @@ func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []st
for _, fileOrFolder := range configs { for _, fileOrFolder := range configs {
if strings.HasSuffix(fileOrFolder, "/") { if strings.HasSuffix(fileOrFolder, "/") {
// config is a folder // config is a folder
// if folder is not supported we will get a "Not implemented" error and continue
files, err := cf.forge.Dir(c, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(fileOrFolder, "/")) files, err := cf.forge.Dir(c, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(fileOrFolder, "/"))
// if folder is not supported we will get a "Not implemented" error and continue
if err != nil && !errors.Is(err, types.ErrNotImplemented) {
log.Error().Err(err).Str("repo", cf.repo.FullName).Str("user", cf.user.Login).Msg("could not get folder from forge")
}
files = filterPipelineFiles(files) files = filterPipelineFiles(files)
if err == nil && len(files) != 0 { if err == nil && len(files) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found %d %s files in '%s'", cf.repo.FullName, len(files), userDefinedLog, fileOrFolder) log.Trace().Msgf("ConfigFetch[%s]: found %d %s files in '%s'", cf.repo.FullName, len(files), userDefinedLog, fileOrFolder)

18
server/shared/configFetcher_test.go → server/forge/configFetcher_test.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package shared_test package forge_test
import ( import (
"context" "context"
@ -32,13 +32,11 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/mocks" "github.com/woodpecker-ci/woodpecker/server/forge/mocks"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/plugins/config" "github.com/woodpecker-ci/woodpecker/server/plugins/config"
"github.com/woodpecker-ci/woodpecker/server/shared"
) )
// TODO(974) move to new package
func TestFetch(t *testing.T) { func TestFetch(t *testing.T) {
t.Parallel() t.Parallel()
@ -294,12 +292,12 @@ func TestFetch(t *testing.T) {
repo := &model.Repo{Owner: "laszlocph", Name: "multipipeline", Config: tt.repoConfig} repo := &model.Repo{Owner: "laszlocph", Name: "multipipeline", Config: tt.repoConfig}
f := new(mocks.Forge) f := new(mocks.Forge)
dirs := map[string][]*forge.FileMeta{} dirs := map[string][]*forge_types.FileMeta{}
for _, file := range tt.files { for _, file := range tt.files {
f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil) f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil)
path := filepath.Dir(file.name) path := filepath.Dir(file.name)
if path != "." { if path != "." {
dirs[path] = append(dirs[path], &forge.FileMeta{ dirs[path] = append(dirs[path], &forge_types.FileMeta{
Name: file.name, Name: file.name,
Data: file.data, Data: file.data,
}) })
@ -314,7 +312,7 @@ func TestFetch(t *testing.T) {
f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("File not found")) f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("File not found"))
f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("Directory not found")) f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("Directory not found"))
configFetcher := shared.NewConfigFetcher( configFetcher := forge.NewConfigFetcher(
f, f,
config.NewHTTP("", ""), config.NewHTTP("", ""),
&model.User{Token: "xxx"}, &model.User{Token: "xxx"},
@ -499,12 +497,12 @@ func TestFetchFromConfigService(t *testing.T) {
repo := &model.Repo{Owner: "laszlocph", Name: tt.name, Config: tt.repoConfig} // Using test name as repo name to provide different responses in mock server repo := &model.Repo{Owner: "laszlocph", Name: tt.name, Config: tt.repoConfig} // Using test name as repo name to provide different responses in mock server
f := new(mocks.Forge) f := new(mocks.Forge)
dirs := map[string][]*forge.FileMeta{} dirs := map[string][]*forge_types.FileMeta{}
for _, file := range tt.files { for _, file := range tt.files {
f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil) f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil)
path := filepath.Dir(file.name) path := filepath.Dir(file.name)
if path != "." { if path != "." {
dirs[path] = append(dirs[path], &forge.FileMeta{ dirs[path] = append(dirs[path], &forge_types.FileMeta{
Name: file.name, Name: file.name,
Data: file.data, Data: file.data,
}) })
@ -519,7 +517,7 @@ func TestFetchFromConfigService(t *testing.T) {
f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("File not found")) f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("File not found"))
f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("Directory not found")) f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("Directory not found"))
configFetcher := shared.NewConfigFetcher( configFetcher := forge.NewConfigFetcher(
f, f,
configAPI, configAPI,
&model.User{Token: "xxx"}, &model.User{Token: "xxx"},

9
server/forge/gitea/gitea.go
View file

@ -38,6 +38,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/common" "github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
@ -112,7 +113,7 @@ func (c *Gitea) Login(ctx context.Context, w http.ResponseWriter, req *http.Requ
// get the OAuth errors // get the OAuth errors
if err := req.FormValue("error"); err != "" { if err := req.FormValue("error"); err != "" {
return nil, &forge.AuthError{ return nil, &forge_types.AuthError{
Err: err, Err: err,
Description: req.FormValue("error_description"), Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"), URI: req.FormValue("error_uri"),
@ -292,8 +293,8 @@ func (c *Gitea) File(ctx context.Context, u *model.User, r *model.Repo, b *model
return cfg, err return cfg, err
} }
func (c *Gitea) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge.FileMeta, error) { func (c *Gitea) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
var configs []*forge.FileMeta var configs []*forge_types.FileMeta
client, err := c.newClientToken(ctx, u.Token) client, err := c.newClientToken(ctx, u.Token)
if err != nil { if err != nil {
@ -316,7 +317,7 @@ func (c *Gitea) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.
return nil, fmt.Errorf("multi-pipeline cannot get %s: %s", e.Path, err) return nil, fmt.Errorf("multi-pipeline cannot get %s: %s", e.Path, err)
} }
configs = append(configs, &forge.FileMeta{ configs = append(configs, &forge_types.FileMeta{
Name: e.Path, Name: e.Path,
Data: data, Data: data,
}) })

11
server/forge/github/github.go
View file

@ -32,6 +32,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/common" "github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
"github.com/woodpecker-ci/woodpecker/shared/utils" "github.com/woodpecker-ci/woodpecker/shared/utils"
@ -90,7 +91,7 @@ func (c *client) Login(ctx context.Context, res http.ResponseWriter, req *http.R
// get the OAuth errors // get the OAuth errors
if err := req.FormValue("error"); err != "" { if err := req.FormValue("error"); err != "" {
return nil, &forge.AuthError{ return nil, &forge_types.AuthError{
Err: err, Err: err,
Description: req.FormValue("error_description"), Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"), URI: req.FormValue("error_uri"),
@ -235,7 +236,7 @@ func (c *client) File(ctx context.Context, u *model.User, r *model.Repo, b *mode
return []byte(data), err return []byte(data), err
} }
func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge.FileMeta, error) { func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
client := c.newClientToken(ctx, u.Token) client := c.newClientToken(ctx, u.Token)
opts := new(github.RepositoryContentGetOptions) opts := new(github.RepositoryContentGetOptions)
@ -245,7 +246,7 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model
return nil, err return nil, err
} }
fc := make(chan *forge.FileMeta) fc := make(chan *forge_types.FileMeta)
errc := make(chan error) errc := make(chan error)
for _, file := range data { for _, file := range data {
@ -254,7 +255,7 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model
if err != nil { if err != nil {
errc <- err errc <- err
} else { } else {
fc <- &forge.FileMeta{ fc <- &forge_types.FileMeta{
Name: path, Name: path,
Data: content, Data: content,
} }
@ -262,7 +263,7 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model
}(f + "/" + *file.Name) }(f + "/" + *file.Name)
} }
var files []*forge.FileMeta var files []*forge_types.FileMeta
for i := 0; i < len(data); i++ { for i := 0; i < len(data); i++ {
select { select {

9
server/forge/gitlab/gitlab.go
View file

@ -33,6 +33,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/common" "github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
"github.com/woodpecker-ci/woodpecker/shared/utils" "github.com/woodpecker-ci/woodpecker/shared/utils"
@ -102,7 +103,7 @@ func (g *Gitlab) Login(ctx context.Context, res http.ResponseWriter, req *http.R
// get the OAuth errors // get the OAuth errors
if err := req.FormValue("error"); err != "" { if err := req.FormValue("error"); err != "" {
return nil, &forge.AuthError{ return nil, &forge_types.AuthError{
Err: err, Err: err,
Description: req.FormValue("error_description"), Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"), URI: req.FormValue("error_uri"),
@ -339,13 +340,13 @@ func (g *Gitlab) File(ctx context.Context, user *model.User, repo *model.Repo, p
} }
// Dir fetches a folder from the forge repository // Dir fetches a folder from the forge repository
func (g *Gitlab) Dir(ctx context.Context, user *model.User, repo *model.Repo, pipeline *model.Pipeline, path string) ([]*forge.FileMeta, error) { func (g *Gitlab) Dir(ctx context.Context, user *model.User, repo *model.Repo, pipeline *model.Pipeline, path string) ([]*forge_types.FileMeta, error) {
client, err := newClient(g.URL, user.Token, g.SkipVerify) client, err := newClient(g.URL, user.Token, g.SkipVerify)
if err != nil { if err != nil {
return nil, err return nil, err
} }
files := make([]*forge.FileMeta, 0, perPage) files := make([]*forge_types.FileMeta, 0, perPage)
_repo, err := g.getProject(ctx, client, repo.Owner, repo.Name) _repo, err := g.getProject(ctx, client, repo.Owner, repo.Name)
if err != nil { if err != nil {
return nil, err return nil, err
@ -372,7 +373,7 @@ func (g *Gitlab) Dir(ctx context.Context, user *model.User, repo *model.Repo, pi
if err != nil { if err != nil {
return nil, err return nil, err
} }
files = append(files, &forge.FileMeta{ files = append(files, &forge_types.FileMeta{
Name: batch[i].Path, Name: batch[i].Path,
Data: data, Data: data,
}) })

5
server/forge/gogs/gogs.go
View file

@ -28,6 +28,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/common" "github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
@ -209,8 +210,8 @@ func (c *client) File(ctx context.Context, u *model.User, r *model.Repo, b *mode
return cfg, err return cfg, err
} }
func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge.FileMeta, error) { func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
return nil, fmt.Errorf("Not implemented") return nil, forge_types.ErrNotImplemented
} }
// Status is not supported by the Gogs driver. // Status is not supported by the Gogs driver.

15
server/forge/mocks/forge.go
View file

@ -1,16 +1,17 @@
// Code generated by mockery v2.14.0. DO NOT EDIT. // Code generated by mockery v2.14.1. DO NOT EDIT.
package mocks package mocks
import ( import (
context "context" context "context"
http "net/http"
forge "github.com/woodpecker-ci/woodpecker/server/forge" http "net/http"
mock "github.com/stretchr/testify/mock" mock "github.com/stretchr/testify/mock"
model "github.com/woodpecker-ci/woodpecker/server/model" model "github.com/woodpecker-ci/woodpecker/server/model"
types "github.com/woodpecker-ci/woodpecker/server/forge/types"
) )
// Forge is an autogenerated mock type for the Forge type // Forge is an autogenerated mock type for the Forge type
@ -112,15 +113,15 @@ func (_m *Forge) Deactivate(ctx context.Context, u *model.User, r *model.Repo, l
} }
// Dir provides a mock function with given fields: ctx, u, r, b, f // Dir provides a mock function with given fields: ctx, u, r, b, f
func (_m *Forge) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge.FileMeta, error) { func (_m *Forge) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*types.FileMeta, error) {
ret := _m.Called(ctx, u, r, b, f) ret := _m.Called(ctx, u, r, b, f)
var r0 []*forge.FileMeta var r0 []*types.FileMeta
if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo, *model.Pipeline, string) []*forge.FileMeta); ok { if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo, *model.Pipeline, string) []*types.FileMeta); ok {
r0 = rf(ctx, u, r, b, f) r0 = rf(ctx, u, r, b, f)
} else { } else {
if ret.Get(0) != nil { if ret.Get(0) != nil {
r0 = ret.Get(0).([]*forge.FileMeta) r0 = ret.Get(0).([]*types.FileMeta)
} }
} }

15
server/forge/remote.go
View file

@ -22,6 +22,7 @@ import (
"context" "context"
"net/http" "net/http"
"github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
@ -58,7 +59,7 @@ type Forge interface {
File(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error) File(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error)
// Dir fetches a folder from the forge repository // Dir fetches a folder from the forge repository
Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*FileMeta, error) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*types.FileMeta, error)
// Status sends the commit status to the forge. // Status sends the commit status to the forge.
// An example would be the GitHub pull request status. // An example would be the GitHub pull request status.
@ -91,18 +92,6 @@ type Forge interface {
OrgMembership(ctx context.Context, u *model.User, owner string) (*model.OrgPerm, error) OrgMembership(ctx context.Context, u *model.User, owner string) (*model.OrgPerm, error)
} }
// FileMeta represents a file in version control
type FileMeta struct {
Name string
Data []byte
}
type ByName []*FileMeta
func (a ByName) Len() int { return len(a) }
func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name }
func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
// Refresher refreshes an oauth token and expiration for the given user. It // Refresher refreshes an oauth token and expiration for the given user. It
// returns true if the token was refreshed, false if the token was not refreshed, // returns true if the token was refreshed, false if the token was not refreshed,
// and error if it failed to refersh. // and error if it failed to refersh.

7
server/forge/errors.go → server/forge/types/errors.go
View file

@ -1,3 +1,4 @@
// Copyright 2022 Woodpecker Authors
// Copyright 2018 Drone.IO Inc. // Copyright 2018 Drone.IO Inc.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
@ -12,7 +13,9 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package forge package types
import "errors"
// AuthError represents forge authentication error. // AuthError represents forge authentication error.
type AuthError struct { type AuthError struct {
@ -35,3 +38,5 @@ func (ae *AuthError) Error() string {
// check interface // check interface
var _ error = new(AuthError) var _ error = new(AuthError)
var ErrNotImplemented = errors.New("Not implemented")

35
server/forge/types/meta.go Normal file
View file

@ -0,0 +1,35 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import "sort"
// FileMeta represents a file in version control
type FileMeta struct {
Name string
Data []byte
}
type fileMetaList []*FileMeta
func (a fileMetaList) Len() int { return len(a) }
func (a fileMetaList) Less(i, j int) bool { return a[i].Name < a[j].Name }
func (a fileMetaList) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func SortByName(fm []*FileMeta) []*FileMeta {
l := fileMetaList(fm)
sort.Sort(l)
return l
}

7
server/shared/userSyncer.go → server/forge/userSyncer.go
View file

@ -13,27 +13,24 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package shared package forge
import ( import (
"context" "context"
"fmt" "fmt"
"time" "time"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
// TODO(974) move to new package
// UserSyncer syncs the user repository and permissions. // UserSyncer syncs the user repository and permissions.
type UserSyncer interface { type UserSyncer interface {
Sync(ctx context.Context, user *model.User) error Sync(ctx context.Context, user *model.User) error
} }
type Syncer struct { type Syncer struct {
Forge forge.Forge Forge Forge
Store store.Store Store store.Store
Perms model.PermStore Perms model.PermStore
Match FilterFunc Match FilterFunc

66
server/grpc/rpc.go
View file

@ -35,9 +35,9 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/logging" "github.com/woodpecker-ci/woodpecker/server/logging"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/pipeline"
"github.com/woodpecker-ci/woodpecker/server/pubsub" "github.com/woodpecker-ci/woodpecker/server/pubsub"
"github.com/woodpecker-ci/woodpecker/server/queue" "github.com/woodpecker-ci/woodpecker/server/queue"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
@ -108,13 +108,13 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error {
return err return err
} }
pipeline, err := s.store.GetPipeline(pstep.PipelineID) currentPipeline, err := s.store.GetPipeline(pstep.PipelineID)
if err != nil { if err != nil {
log.Error().Msgf("error: cannot find pipeline with id %d: %s", pstep.PipelineID, err) log.Error().Msgf("error: cannot find pipeline with id %d: %s", pstep.PipelineID, err)
return err return err
} }
step, err := s.store.StepChild(pipeline, pstep.PID, state.Step) step, err := s.store.StepChild(currentPipeline, pstep.PID, state.Step)
if err != nil { if err != nil {
log.Error().Msgf("error: cannot find step with name %s: %s", state.Step, err) log.Error().Msgf("error: cannot find step with name %s: %s", state.Step, err)
return err return err
@ -128,20 +128,20 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error {
} }
} }
repo, err := s.store.GetRepo(pipeline.RepoID) repo, err := s.store.GetRepo(currentPipeline.RepoID)
if err != nil { if err != nil {
log.Error().Msgf("error: cannot find repo with id %d: %s", pipeline.RepoID, err) log.Error().Msgf("error: cannot find repo with id %d: %s", currentPipeline.RepoID, err)
return err return err
} }
if _, err = shared.UpdateStepStatus(s.store, *step, state, pipeline.Started); err != nil { if _, err = pipeline.UpdateStepStatus(s.store, *step, state, currentPipeline.Started); err != nil {
log.Error().Err(err).Msg("rpc.update: cannot update step") log.Error().Err(err).Msg("rpc.update: cannot update step")
} }
if pipeline.Steps, err = s.store.StepList(pipeline); err != nil { if currentPipeline.Steps, err = s.store.StepList(currentPipeline); err != nil {
log.Error().Err(err).Msg("can not get step list from store") log.Error().Err(err).Msg("can not get step list from store")
} }
if pipeline.Steps, err = model.Tree(pipeline.Steps); err != nil { if currentPipeline.Steps, err = model.Tree(currentPipeline.Steps); err != nil {
log.Error().Err(err).Msg("can not build tree from step list") log.Error().Err(err).Msg("can not build tree from step list")
return err return err
} }
@ -153,7 +153,7 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error {
} }
message.Data, _ = json.Marshal(model.Event{ message.Data, _ = json.Marshal(model.Event{
Repo: *repo, Repo: *repo,
Pipeline: *pipeline, Pipeline: *currentPipeline,
}) })
if err := s.pubsub.Publish(c, "topic/events", message); err != nil { if err := s.pubsub.Publish(c, "topic/events", message); err != nil {
log.Error().Err(err).Msg("can not publish step list to") log.Error().Err(err).Msg("can not publish step list to")
@ -255,26 +255,26 @@ func (s *RPC) Init(c context.Context, id string, state rpc.State) error {
} }
} }
pipeline, err := s.store.GetPipeline(step.PipelineID) currentPipeline, err := s.store.GetPipeline(step.PipelineID)
if err != nil { if err != nil {
log.Error().Msgf("error: cannot find pipeline with id %d: %s", step.PipelineID, err) log.Error().Msgf("error: cannot find pipeline with id %d: %s", step.PipelineID, err)
return err return err
} }
repo, err := s.store.GetRepo(pipeline.RepoID) repo, err := s.store.GetRepo(currentPipeline.RepoID)
if err != nil { if err != nil {
log.Error().Msgf("error: cannot find repo with id %d: %s", pipeline.RepoID, err) log.Error().Msgf("error: cannot find repo with id %d: %s", currentPipeline.RepoID, err)
return err return err
} }
if pipeline.Status == model.StatusPending { if currentPipeline.Status == model.StatusPending {
if pipeline, err = shared.UpdateToStatusRunning(s.store, *pipeline, state.Started); err != nil { if currentPipeline, err = pipeline.UpdateToStatusRunning(s.store, *currentPipeline, state.Started); err != nil {
log.Error().Msgf("error: init: cannot update build_id %d state: %s", pipeline.ID, err) log.Error().Msgf("error: init: cannot update build_id %d state: %s", currentPipeline.ID, err)
} }
} }
defer func() { defer func() {
pipeline.Steps, _ = s.store.StepList(pipeline) currentPipeline.Steps, _ = s.store.StepList(currentPipeline)
message := pubsub.Message{ message := pubsub.Message{
Labels: map[string]string{ Labels: map[string]string{
"repo": repo.FullName, "repo": repo.FullName,
@ -283,14 +283,14 @@ func (s *RPC) Init(c context.Context, id string, state rpc.State) error {
} }
message.Data, _ = json.Marshal(model.Event{ message.Data, _ = json.Marshal(model.Event{
Repo: *repo, Repo: *repo,
Pipeline: *pipeline, Pipeline: *currentPipeline,
}) })
if err := s.pubsub.Publish(c, "topic/events", message); err != nil { if err := s.pubsub.Publish(c, "topic/events", message); err != nil {
log.Error().Err(err).Msg("can not publish step list to") log.Error().Err(err).Msg("can not publish step list to")
} }
}() }()
_, err = shared.UpdateStepToStatusStarted(s.store, *step, state) _, err = pipeline.UpdateStepToStatusStarted(s.store, *step, state)
return err return err
} }
@ -307,25 +307,25 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error {
return err return err
} }
pipeline, err := s.store.GetPipeline(step.PipelineID) currentPipeline, err := s.store.GetPipeline(step.PipelineID)
if err != nil { if err != nil {
log.Error().Msgf("error: cannot find pipeline with id %d: %s", step.PipelineID, err) log.Error().Msgf("error: cannot find pipeline with id %d: %s", step.PipelineID, err)
return err return err
} }
repo, err := s.store.GetRepo(pipeline.RepoID) repo, err := s.store.GetRepo(currentPipeline.RepoID)
if err != nil { if err != nil {
log.Error().Msgf("error: cannot find repo with id %d: %s", pipeline.RepoID, err) log.Error().Msgf("error: cannot find repo with id %d: %s", currentPipeline.RepoID, err)
return err return err
} }
log.Trace(). log.Trace().
Str("repo_id", fmt.Sprint(repo.ID)). Str("repo_id", fmt.Sprint(repo.ID)).
Str("build_id", fmt.Sprint(pipeline.ID)). Str("build_id", fmt.Sprint(currentPipeline.ID)).
Str("step_id", id). Str("step_id", id).
Msgf("gRPC Done with state: %#v", state) Msgf("gRPC Done with state: %#v", state)
if step, err = shared.UpdateStepStatusToDone(s.store, *step, state); err != nil { if step, err = pipeline.UpdateStepStatusToDone(s.store, *step, state); err != nil {
log.Error().Msgf("error: done: cannot update step_id %d state: %s", step.ID, err) log.Error().Msgf("error: done: cannot update step_id %d state: %s", step.ID, err)
} }
@ -339,34 +339,34 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error {
log.Error().Msgf("error: done: cannot ack step_id %d: %s", stepID, err) log.Error().Msgf("error: done: cannot ack step_id %d: %s", stepID, err)
} }
steps, err := s.store.StepList(pipeline) steps, err := s.store.StepList(currentPipeline)
if err != nil { if err != nil {
return err return err
} }
s.completeChildrenIfParentCompleted(steps, step) s.completeChildrenIfParentCompleted(steps, step)
if !model.IsThereRunningStage(steps) { if !model.IsThereRunningStage(steps) {
if pipeline, err = shared.UpdateStatusToDone(s.store, *pipeline, model.PipelineStatus(steps), step.Stopped); err != nil { if currentPipeline, err = pipeline.UpdateStatusToDone(s.store, *currentPipeline, model.PipelineStatus(steps), step.Stopped); err != nil {
log.Error().Err(err).Msgf("error: done: cannot update build_id %d final state", pipeline.ID) log.Error().Err(err).Msgf("error: done: cannot update build_id %d final state", currentPipeline.ID)
} }
} }
s.updateForgeStatus(c, repo, pipeline, step) s.updateForgeStatus(c, repo, currentPipeline, step)
if err := s.logger.Close(c, id); err != nil { if err := s.logger.Close(c, id); err != nil {
log.Error().Err(err).Msgf("done: cannot close build_id %d logger", step.ID) log.Error().Err(err).Msgf("done: cannot close build_id %d logger", step.ID)
} }
if err := s.notify(c, repo, pipeline, steps); err != nil { if err := s.notify(c, repo, currentPipeline, steps); err != nil {
return err return err
} }
if pipeline.Status == model.StatusSuccess || pipeline.Status == model.StatusFailure { if currentPipeline.Status == model.StatusSuccess || currentPipeline.Status == model.StatusFailure {
s.pipelineCount.WithLabelValues(repo.FullName, pipeline.Branch, string(pipeline.Status), "total").Inc() s.pipelineCount.WithLabelValues(repo.FullName, currentPipeline.Branch, string(currentPipeline.Status), "total").Inc()
s.pipelineTime.WithLabelValues(repo.FullName, pipeline.Branch, string(pipeline.Status), "total").Set(float64(pipeline.Finished - pipeline.Started)) s.pipelineTime.WithLabelValues(repo.FullName, currentPipeline.Branch, string(currentPipeline.Status), "total").Set(float64(currentPipeline.Finished - currentPipeline.Started))
} }
if model.IsMultiPipeline(steps) { if model.IsMultiPipeline(steps) {
s.pipelineTime.WithLabelValues(repo.FullName, pipeline.Branch, string(step.State), step.Name).Set(float64(step.Stopped - step.Started)) s.pipelineTime.WithLabelValues(repo.FullName, currentPipeline.Branch, string(step.State), step.Name).Set(float64(step.Stopped - step.Started))
} }
return nil return nil
@ -385,7 +385,7 @@ func (s *RPC) Log(c context.Context, id string, line *rpc.Line) error {
func (s *RPC) completeChildrenIfParentCompleted(steps []*model.Step, completedStep *model.Step) { func (s *RPC) completeChildrenIfParentCompleted(steps []*model.Step, completedStep *model.Step) {
for _, p := range steps { for _, p := range steps {
if p.Running() && p.PPID == completedStep.PID { if p.Running() && p.PPID == completedStep.PID {
if _, err := shared.UpdateStepToStatusSkipped(s.store, *p, completedStep.Stopped); err != nil { if _, err := pipeline.UpdateStepToStatusSkipped(s.store, *p, completedStep.Stopped); err != nil {
log.Error().Msgf("error: done: cannot update step_id %d child state: %s", p.ID, err) log.Error().Msgf("error: done: cannot update step_id %d child state: %s", p.ID, err)
} }
} }

23
server/pipeline/approve.go
View file

@ -20,49 +20,48 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server/forge" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
// Approve update the status to pending for blocked pipeline because of a gated repo // Approve update the status to pending for blocked pipeline because of a gated repo
// and start them afterwards // and start them afterwards
func Approve(ctx context.Context, store store.Store, pipeline *model.Pipeline, user *model.User, repo *model.Repo) (*model.Pipeline, error) { func Approve(ctx context.Context, store store.Store, currentPipeline *model.Pipeline, user *model.User, repo *model.Repo) (*model.Pipeline, error) {
if pipeline.Status != model.StatusBlocked { if currentPipeline.Status != model.StatusBlocked {
return nil, ErrBadRequest{Msg: fmt.Sprintf("cannot decline a pipeline with status %s", pipeline.Status)} return nil, ErrBadRequest{Msg: fmt.Sprintf("cannot decline a pipeline with status %s", currentPipeline.Status)}
} }
// fetch the pipeline file from the database // fetch the pipeline file from the database
configs, err := store.ConfigsForPipeline(pipeline.ID) configs, err := store.ConfigsForPipeline(currentPipeline.ID)
if err != nil { if err != nil {
msg := fmt.Sprintf("failure to get pipeline config for %s. %s", repo.FullName, err) msg := fmt.Sprintf("failure to get pipeline config for %s. %s", repo.FullName, err)
log.Error().Msg(msg) log.Error().Msg(msg)
return nil, ErrNotFound{Msg: msg} return nil, ErrNotFound{Msg: msg}
} }
if pipeline, err = shared.UpdateToStatusPending(store, *pipeline, user.Login); err != nil { if currentPipeline, err = UpdateToStatusPending(store, *currentPipeline, user.Login); err != nil {
return nil, fmt.Errorf("error updating pipeline. %s", err) return nil, fmt.Errorf("error updating pipeline. %s", err)
} }
var yamls []*forge.FileMeta var yamls []*forge_types.FileMeta
for _, y := range configs { for _, y := range configs {
yamls = append(yamls, &forge.FileMeta{Data: y.Data, Name: y.Name}) yamls = append(yamls, &forge_types.FileMeta{Data: y.Data, Name: y.Name})
} }
pipeline, pipelineItems, err := createPipelineItems(ctx, store, pipeline, user, repo, yamls, nil) currentPipeline, pipelineItems, err := createPipelineItems(ctx, store, currentPipeline, user, repo, yamls, nil)
if err != nil { if err != nil {
msg := fmt.Sprintf("failure to createBuildItems for %s", repo.FullName) msg := fmt.Sprintf("failure to createBuildItems for %s", repo.FullName)
log.Error().Err(err).Msg(msg) log.Error().Err(err).Msg(msg)
return nil, err return nil, err
} }
pipeline, err = start(ctx, store, pipeline, user, repo, pipelineItems) currentPipeline, err = start(ctx, store, currentPipeline, user, repo, pipelineItems)
if err != nil { if err != nil {
msg := fmt.Sprintf("failure to start pipeline for %s: %v", repo.FullName, err) msg := fmt.Sprintf("failure to start pipeline for %s: %v", repo.FullName, err)
log.Error().Err(err).Msg(msg) log.Error().Err(err).Msg(msg)
return nil, fmt.Errorf(msg) return nil, fmt.Errorf(msg)
} }
return pipeline, nil return currentPipeline, nil
} }

7
server/pipeline/cancel.go
View file

@ -23,7 +23,6 @@ import (
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/queue" "github.com/woodpecker-ci/woodpecker/server/queue"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
@ -74,18 +73,18 @@ func Cancel(ctx context.Context, store store.Store, repo *model.Repo, pipeline *
for _, step := range steps { for _, step := range steps {
if step.State == model.StatusPending { if step.State == model.StatusPending {
if step.PPID != 0 { if step.PPID != 0 {
if _, err = shared.UpdateStepToStatusSkipped(store, *step, 0); err != nil { if _, err = UpdateStepToStatusSkipped(store, *step, 0); err != nil {
log.Error().Msgf("error: done: cannot update step_id %d state: %s", step.ID, err) log.Error().Msgf("error: done: cannot update step_id %d state: %s", step.ID, err)
} }
} else { } else {
if _, err = shared.UpdateStepToStatusKilled(store, *step); err != nil { if _, err = UpdateStepToStatusKilled(store, *step); err != nil {
log.Error().Msgf("error: done: cannot update step_id %d state: %s", step.ID, err) log.Error().Msgf("error: done: cannot update step_id %d state: %s", step.ID, err)
} }
} }
} }
} }
killedBuild, err := shared.UpdateToStatusKilled(store, *pipeline) killedBuild, err := UpdateToStatusKilled(store, *pipeline)
if err != nil { if err != nil {
log.Error().Err(err).Msgf("UpdateToStatusKilled: %v", pipeline) log.Error().Err(err).Msgf("UpdateToStatusKilled: %v", pipeline)
return err return err

16
server/pipeline/config.go
View file

@ -18,26 +18,26 @@ import (
"crypto/sha256" "crypto/sha256"
"fmt" "fmt"
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/pipeline"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
func findOrPersistPipelineConfig(store store.Store, pipeline *model.Pipeline, forgeYamlConfig *forge.FileMeta) (*model.Config, error) { func findOrPersistPipelineConfig(store store.Store, currentPipeline *model.Pipeline, forgeYamlConfig *forge_types.FileMeta) (*model.Config, error) {
sha := fmt.Sprintf("%x", sha256.Sum256(forgeYamlConfig.Data)) sha := fmt.Sprintf("%x", sha256.Sum256(forgeYamlConfig.Data))
conf, err := store.ConfigFindIdentical(pipeline.RepoID, sha) conf, err := store.ConfigFindIdentical(currentPipeline.RepoID, sha)
if err != nil { if err != nil {
conf = &model.Config{ conf = &model.Config{
RepoID: pipeline.RepoID, RepoID: currentPipeline.RepoID,
Data: forgeYamlConfig.Data, Data: forgeYamlConfig.Data,
Hash: sha, Hash: sha,
Name: shared.SanitizePath(forgeYamlConfig.Name), Name: pipeline.SanitizePath(forgeYamlConfig.Name),
} }
err = store.ConfigCreate(conf) err = store.ConfigCreate(conf)
if err != nil { if err != nil {
// retry in case we receive two hooks at the same time // retry in case we receive two hooks at the same time
conf, err = store.ConfigFindIdentical(pipeline.RepoID, sha) conf, err = store.ConfigFindIdentical(currentPipeline.RepoID, sha)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -46,7 +46,7 @@ func findOrPersistPipelineConfig(store store.Store, pipeline *model.Pipeline, fo
pipelineConfig := &model.PipelineConfig{ pipelineConfig := &model.PipelineConfig{
ConfigID: conf.ID, ConfigID: conf.ID,
PipelineID: pipeline.ID, PipelineID: currentPipeline.ID,
} }
if err := store.PipelineConfigCreate(pipelineConfig); err != nil { if err := store.PipelineConfigCreate(pipelineConfig); err != nil {
return nil, err return nil, err

6
server/pipeline/create.go
View file

@ -23,8 +23,8 @@ import (
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge" "github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
@ -53,14 +53,14 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline
} }
var ( var (
forgeYamlConfigs []*forge.FileMeta forgeYamlConfigs []*types.FileMeta
configFetchErr error configFetchErr error
filtered bool filtered bool
parseErr error parseErr error
) )
// fetch the pipeline file from the forge // fetch the pipeline file from the forge
configFetcher := shared.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.ConfigService, repoUser, repo, pipeline) configFetcher := forge.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.ConfigService, repoUser, repo, pipeline)
forgeYamlConfigs, configFetchErr = configFetcher.Fetch(ctx) forgeYamlConfigs, configFetchErr = configFetcher.Fetch(ctx)
if configFetchErr == nil { if configFetchErr == nil {
filtered, parseErr = checkIfFiltered(pipeline, forgeYamlConfigs) filtered, parseErr = checkIfFiltered(pipeline, forgeYamlConfigs)

3
server/pipeline/decline.go
View file

@ -20,7 +20,6 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
@ -30,7 +29,7 @@ func Decline(ctx context.Context, store store.Store, pipeline *model.Pipeline, u
return nil, fmt.Errorf("cannot decline a pipeline with status %s", pipeline.Status) return nil, fmt.Errorf("cannot decline a pipeline with status %s", pipeline.Status)
} }
_, err := shared.UpdateToStatusDeclined(store, *pipeline, user.Login) _, err := UpdateToStatusDeclined(store, *pipeline, user.Login)
if err != nil { if err != nil {
return nil, fmt.Errorf("error updating pipeline. %s", err) return nil, fmt.Errorf("error updating pipeline. %s", err)
} }

12
server/pipeline/filter.go
View file

@ -19,17 +19,17 @@ package pipeline
import ( import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend" "github.com/woodpecker-ci/woodpecker/pipeline/frontend"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/server/forge" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
) )
func zeroSteps(pipeline *model.Pipeline, forgeYamlConfigs []*forge.FileMeta) bool { func zeroSteps(currentPipeline *model.Pipeline, forgeYamlConfigs []*forge_types.FileMeta) bool {
b := shared.StepBuilder{ b := pipeline.StepBuilder{
Repo: &model.Repo{}, Repo: &model.Repo{},
Curr: pipeline, Curr: currentPipeline,
Last: &model.Pipeline{}, Last: &model.Pipeline{},
Netrc: &model.Netrc{}, Netrc: &model.Netrc{},
Secs: []*model.Secret{}, Secs: []*model.Secret{},
@ -51,7 +51,7 @@ func zeroSteps(pipeline *model.Pipeline, forgeYamlConfigs []*forge.FileMeta) boo
// TODO: parse yaml once and not for each filter function // TODO: parse yaml once and not for each filter function
// Check if at least one pipeline step will be execute otherwise we will just ignore this webhook // Check if at least one pipeline step will be execute otherwise we will just ignore this webhook
func checkIfFiltered(pipeline *model.Pipeline, forgeYamlConfigs []*forge.FileMeta) (bool, error) { func checkIfFiltered(pipeline *model.Pipeline, forgeYamlConfigs []*forge_types.FileMeta) (bool, error) {
log.Trace().Msgf("hook.branchFiltered(): pipeline branch: '%s' pipeline event: '%s' config count: %d", pipeline.Branch, pipeline.Event, len(forgeYamlConfigs)) log.Trace().Msgf("hook.branchFiltered(): pipeline branch: '%s' pipeline event: '%s' config count: %d", pipeline.Branch, pipeline.Event, len(forgeYamlConfigs))
matchMetadata := frontend.Metadata{ matchMetadata := frontend.Metadata{

35
server/pipeline/items.go
View file

@ -21,33 +21,36 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
func createPipelineItems(ctx context.Context, store store.Store, pipeline *model.Pipeline, user *model.User, repo *model.Repo, yamls []*forge.FileMeta, envs map[string]string) (*model.Pipeline, []*shared.PipelineItem, error) { func createPipelineItems(ctx context.Context, store store.Store,
currentPipeline *model.Pipeline, user *model.User, repo *model.Repo,
yamls []*forge_types.FileMeta, envs map[string]string,
) (*model.Pipeline, []*pipeline.Item, error) {
netrc, err := server.Config.Services.Forge.Netrc(user, repo) netrc, err := server.Config.Services.Forge.Netrc(user, repo)
if err != nil { if err != nil {
log.Error().Err(err).Msg("Failed to generate netrc file") log.Error().Err(err).Msg("Failed to generate netrc file")
} }
// get the previous pipeline so that we can send status change notifications // get the previous pipeline so that we can send status change notifications
last, err := store.GetPipelineLastBefore(repo, pipeline.Branch, pipeline.ID) last, err := store.GetPipelineLastBefore(repo, currentPipeline.Branch, currentPipeline.ID)
if err != nil && !errors.Is(err, sql.ErrNoRows) { if err != nil && !errors.Is(err, sql.ErrNoRows) {
log.Error().Err(err).Str("repo", repo.FullName).Msgf("Error getting last pipeline before pipeline number '%d'", pipeline.Number) log.Error().Err(err).Str("repo", repo.FullName).Msgf("Error getting last pipeline before pipeline number '%d'", currentPipeline.Number)
} }
secs, err := server.Config.Services.Secrets.SecretListPipeline(repo, pipeline) secs, err := server.Config.Services.Secrets.SecretListPipeline(repo, currentPipeline)
if err != nil { if err != nil {
log.Error().Err(err).Msgf("Error getting secrets for %s#%d", repo.FullName, pipeline.Number) log.Error().Err(err).Msgf("Error getting secrets for %s#%d", repo.FullName, currentPipeline.Number)
} }
regs, err := server.Config.Services.Registries.RegistryList(repo) regs, err := server.Config.Services.Registries.RegistryList(repo)
if err != nil { if err != nil {
log.Error().Err(err).Msgf("Error getting registry credentials for %s#%d", repo.FullName, pipeline.Number) log.Error().Err(err).Msgf("Error getting registry credentials for %s#%d", repo.FullName, currentPipeline.Number)
} }
if envs == nil { if envs == nil {
@ -60,13 +63,13 @@ func createPipelineItems(ctx context.Context, store store.Store, pipeline *model
} }
} }
for k, v := range pipeline.AdditionalVariables { for k, v := range currentPipeline.AdditionalVariables {
envs[k] = v envs[k] = v
} }
b := shared.StepBuilder{ b := pipeline.StepBuilder{
Repo: repo, Repo: repo,
Curr: pipeline, Curr: currentPipeline,
Last: last, Last: last,
Netrc: netrc, Netrc: netrc,
Secs: secs, Secs: secs,
@ -77,14 +80,14 @@ func createPipelineItems(ctx context.Context, store store.Store, pipeline *model
} }
pipelineItems, err := b.Build() pipelineItems, err := b.Build()
if err != nil { if err != nil {
pipeline, uerr := shared.UpdateToStatusError(store, *pipeline, err) currentPipeline, uerr := UpdateToStatusError(store, *currentPipeline, err)
if uerr != nil { if uerr != nil {
log.Error().Err(err).Msgf("Error setting error status of pipeline for %s#%d", repo.FullName, pipeline.Number) log.Error().Err(err).Msgf("Error setting error status of pipeline for %s#%d", repo.FullName, currentPipeline.Number)
} }
return pipeline, nil, err return currentPipeline, nil, err
} }
pipeline = shared.SetPipelineStepsOnPipeline(b.Curr, pipelineItems) currentPipeline = pipeline.SetPipelineStepsOnPipeline(b.Curr, pipelineItems)
return pipeline, pipelineItems, nil return currentPipeline, pipelineItems, nil
} }

4
server/shared/pipelineStatus.go → server/pipeline/pipelineStatus.go
View file

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package shared package pipeline
import ( import (
"time" "time"
@ -21,8 +21,6 @@ import (
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
// TODO(974) move to server/pipeline/*
func UpdateToStatusRunning(store model.UpdatePipelineStore, pipeline model.Pipeline, started int64) (*model.Pipeline, error) { func UpdateToStatusRunning(store model.UpdatePipelineStore, pipeline model.Pipeline, started int64) (*model.Pipeline, error) {
pipeline.Status = model.StatusRunning pipeline.Status = model.StatusRunning
pipeline.Started = started pipeline.Started = started

4
server/shared/pipelineStatus_test.go → server/pipeline/pipelineStatus_test.go
View file

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package shared package pipeline
import ( import (
"errors" "errors"
@ -23,8 +23,6 @@ import (
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
// TODO(974) move to server/pipeline/*
type mockUpdatePipelineStore struct{} type mockUpdatePipelineStore struct{}
func (m *mockUpdatePipelineStore) UpdatePipeline(pipeline *model.Pipeline) error { func (m *mockUpdatePipelineStore) UpdatePipeline(pipeline *model.Pipeline) error {

6
server/pipeline/queue.go
View file

@ -19,14 +19,14 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/pipeline/rpc" "github.com/woodpecker-ci/woodpecker/pipeline/rpc"
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/queue" "github.com/woodpecker-ci/woodpecker/server/queue"
"github.com/woodpecker-ci/woodpecker/server/shared"
) )
func queueBuild(pipeline *model.Pipeline, repo *model.Repo, pipelineItems []*shared.PipelineItem) error { func queueBuild(pipeline *model.Pipeline, repo *model.Repo, pipelineItems []*pipeline.Item) error {
var tasks []*queue.Task var tasks []*queue.Task
for _, item := range pipelineItems { for _, item := range pipelineItems {
if item.Step.State == model.StatusSkipped { if item.Step.State == model.StatusSkipped {
@ -58,7 +58,7 @@ func queueBuild(pipeline *model.Pipeline, repo *model.Repo, pipelineItems []*sha
return server.Config.Services.Queue.PushAtOnce(context.Background(), tasks) return server.Config.Services.Queue.PushAtOnce(context.Background(), tasks)
} }
func taskIds(dependsOn []string, pipelineItems []*shared.PipelineItem) (taskIds []string) { func taskIds(dependsOn []string, pipelineItems []*pipeline.Item) (taskIds []string) {
for _, dep := range dependsOn { for _, dep := range dependsOn {
for _, pipelineItem := range pipelineItems { for _, pipelineItem := range pipelineItems {
if pipelineItem.Step.Name == dep { if pipelineItem.Step.Name == dep {

13
server/pipeline/restart.go
View file

@ -24,9 +24,8 @@ import (
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
@ -38,7 +37,7 @@ func Restart(ctx context.Context, store store.Store, lastBuild *model.Pipeline,
return nil, ErrBadRequest{Msg: fmt.Sprintf("cannot restart a pipeline with status %s", lastBuild.Status)} return nil, ErrBadRequest{Msg: fmt.Sprintf("cannot restart a pipeline with status %s", lastBuild.Status)}
} }
var pipelineFiles []*forge.FileMeta var pipelineFiles []*forge_types.FileMeta
// fetch the old pipeline config from database // fetch the old pipeline config from database
configs, err := store.ConfigsForPipeline(lastBuild.ID) configs, err := store.ConfigsForPipeline(lastBuild.ID)
@ -49,14 +48,14 @@ func Restart(ctx context.Context, store store.Store, lastBuild *model.Pipeline,
} }
for _, y := range configs { for _, y := range configs {
pipelineFiles = append(pipelineFiles, &forge.FileMeta{Data: y.Data, Name: y.Name}) pipelineFiles = append(pipelineFiles, &forge_types.FileMeta{Data: y.Data, Name: y.Name})
} }
// If config extension is active we should refetch the config in case something changed // If config extension is active we should refetch the config in case something changed
if server.Config.Services.ConfigService != nil && server.Config.Services.ConfigService.IsConfigured() { if server.Config.Services.ConfigService != nil && server.Config.Services.ConfigService.IsConfigured() {
currentFileMeta := make([]*forge.FileMeta, len(configs)) currentFileMeta := make([]*forge_types.FileMeta, len(configs))
for i, cfg := range configs { for i, cfg := range configs {
currentFileMeta[i] = &forge.FileMeta{Name: cfg.Name, Data: cfg.Data} currentFileMeta[i] = &forge_types.FileMeta{Name: cfg.Name, Data: cfg.Data}
} }
newConfig, useOld, err := server.Config.Services.ConfigService.FetchConfig(ctx, repo, lastBuild, currentFileMeta) newConfig, useOld, err := server.Config.Services.ConfigService.FetchConfig(ctx, repo, lastBuild, currentFileMeta)
@ -81,7 +80,7 @@ func Restart(ctx context.Context, store store.Store, lastBuild *model.Pipeline,
} }
if len(configs) == 0 { if len(configs) == 0 {
newBuild, uerr := shared.UpdateToStatusError(store, *newBuild, errors.New("pipeline definition not found")) newBuild, uerr := UpdateToStatusError(store, *newBuild, errors.New("pipeline definition not found"))
if uerr != nil { if uerr != nil {
log.Debug().Err(uerr).Msg("failure to update pipeline status") log.Debug().Err(uerr).Msg("failure to update pipeline status")
} }

4
server/pipeline/start.go
View file

@ -19,13 +19,13 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store"
) )
// start a pipeline, make sure it was stored persistent in the store before // start a pipeline, make sure it was stored persistent in the store before
func start(ctx context.Context, store store.Store, activePipeline *model.Pipeline, user *model.User, repo *model.Repo, pipelineItems []*shared.PipelineItem) (*model.Pipeline, error) { func start(ctx context.Context, store store.Store, activePipeline *model.Pipeline, user *model.User, repo *model.Repo, pipelineItems []*pipeline.Item) (*model.Pipeline, error) {
// call to cancel previous pipelines if needed // call to cancel previous pipelines if needed
if err := cancelPreviousPipelines(ctx, store, activePipeline, repo); err != nil { if err := cancelPreviousPipelines(ctx, store, activePipeline, repo); err != nil {
// should be not breaking // should be not breaking

4
server/shared/stepStatus.go → server/pipeline/stepStatus.go
View file

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package shared package pipeline
import ( import (
"time" "time"
@ -22,8 +22,6 @@ import (
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
// TODO(974) move to server/pipeline/*
func UpdateStepStatus(store model.UpdateStepStore, step model.Step, state rpc.State, started int64) (*model.Step, error) { func UpdateStepStatus(store model.UpdateStepStore, step model.Step, state rpc.State, started int64) (*model.Step, error) {
if state.Exited { if state.Exited {
step.Stopped = state.Finished step.Stopped = state.Finished

4
server/shared/stepStatus_test.go → server/pipeline/stepStatus_test.go
View file

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package shared package pipeline
import ( import (
"testing" "testing"
@ -23,8 +23,6 @@ import (
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
// TODO(974) move to server/pipeline/*
type mockUpdateStepStore struct{} type mockUpdateStepStore struct{}
func (m *mockUpdateStepStore) StepUpdate(build *model.Step) error { func (m *mockUpdateStepStore) StepUpdate(build *model.Step) error {

4
server/plugins/config/extension.go
View file

@ -17,11 +17,11 @@ package config
import ( import (
"context" "context"
"github.com/woodpecker-ci/woodpecker/server/forge" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
) )
type Extension interface { type Extension interface {
IsConfigured() bool IsConfigured() bool
FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge.FileMeta) (configData []*forge.FileMeta, useOld bool, err error) FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge_types.FileMeta) (configData []*forge_types.FileMeta, useOld bool, err error)
} }

12
server/plugins/config/http.go
View file

@ -19,7 +19,7 @@ import (
"crypto" "crypto"
"fmt" "fmt"
"github.com/woodpecker-ci/woodpecker/server/forge" forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/plugins/utils" "github.com/woodpecker-ci/woodpecker/server/plugins/utils"
) )
@ -53,7 +53,7 @@ func (cp *http) IsConfigured() bool {
return cp.endpoint != "" return cp.endpoint != ""
} }
func (cp *http) FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge.FileMeta) (configData []*forge.FileMeta, useOld bool, err error) { func (cp *http) FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge_types.FileMeta) (configData []*forge_types.FileMeta, useOld bool, err error) {
currentConfigs := make([]*config, len(currentFileMeta)) currentConfigs := make([]*config, len(currentFileMeta))
for i, pipe := range currentFileMeta { for i, pipe := range currentFileMeta {
currentConfigs[i] = &config{Name: pipe.Name, Data: string(pipe.Data)} currentConfigs[i] = &config{Name: pipe.Name, Data: string(pipe.Data)}
@ -66,13 +66,13 @@ func (cp *http) FetchConfig(ctx context.Context, repo *model.Repo, pipeline *mod
return nil, false, fmt.Errorf("Failed to fetch config via http (%d) %w", status, err) return nil, false, fmt.Errorf("Failed to fetch config via http (%d) %w", status, err)
} }
var newFileMeta []*forge.FileMeta var newFileMeta []*forge_types.FileMeta
if status != 200 { if status != 200 {
newFileMeta = make([]*forge.FileMeta, 0) newFileMeta = make([]*forge_types.FileMeta, 0)
} else { } else {
newFileMeta = make([]*forge.FileMeta, len(response.Configs)) newFileMeta = make([]*forge_types.FileMeta, len(response.Configs))
for i, pipe := range response.Configs { for i, pipe := range response.Configs {
newFileMeta[i] = &forge.FileMeta{Name: pipe.Name, Data: []byte(pipe.Data)} newFileMeta[i] = &forge_types.FileMeta{Name: pipe.Name, Data: []byte(pipe.Data)}
} }
} }

2
server/store/mocks/store.go
View file

@ -1,4 +1,4 @@
// Code generated by mockery v2.14.0. DO NOT EDIT. // Code generated by mockery v2.14.1. DO NOT EDIT.
package mocks package mocks