mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-11-26 03:41:01 +00:00
Code Issues Projects Releases Wiki Activity

Split and refactor (#1394)

Browse source 
Closes #974
This commit is contained in:
6543 2022-11-06 12:44:04 +01:00 committed by GitHub
parent e901f605b1
commit 18311d4360
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 258 additions and 239 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
server/shared/stepBuilder.go → pipeline/stepBuilder.go
View file

@ -13,14 +13,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
package pipeline
import (
"fmt"
"math/rand"
"net/url"
"path/filepath"
"sort"
"strings"
"github.com/drone/envsubst"
@ -33,12 +32,10 @@ import (
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/linter"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/matrix"
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
)
// TODO(974) move to pipeline/*
// StepBuilder Takes the hook data and the yaml and returns in internal data model
type StepBuilder struct {
Repo *model.Repo
@ -48,11 +45,11 @@ type StepBuilder struct {
Secs []*model.Secret
Regs []*model.Registry
Link string
Yamls []*forge.FileMeta
Yamls []*forge_types.FileMeta
Envs map[string]string
}
type PipelineItem struct {
type Item struct {
Step *model.Step
Platform string
Labels map[string]string
@ -61,10 +58,10 @@ type PipelineItem struct {
Config *backend.Config
}
func (b *StepBuilder) Build() ([]*PipelineItem, error) {
var items []*PipelineItem
func (b *StepBuilder) Build() ([]*Item, error) {
var items []*Item
sort.Sort(forge.ByName(b.Yamls))
b.Yamls = forge_types.SortByName(b.Yamls)
pidSequence := 1
@ -149,7 +146,7 @@ func (b *StepBuilder) Build() ([]*PipelineItem, error) {
continue
}
item := &PipelineItem{
item := &Item{
Step: step,
Config: ir,
Labels: parsed.Labels,
@ -176,7 +173,7 @@ func (b *StepBuilder) Build() ([]*PipelineItem, error) {
return items, nil
}
func stepListContainsItemsToRun(items []*PipelineItem) bool {
func stepListContainsItemsToRun(items []*Item) bool {
for i := range items {
if items[i].Step.State == model.StatusPending {
return true
@ -185,8 +182,8 @@ func stepListContainsItemsToRun(items []*PipelineItem) bool {
return false
}
func filterItemsWithMissingDependencies(items []*PipelineItem) []*PipelineItem {
itemsToRemove := make([]*PipelineItem, 0)
func filterItemsWithMissingDependencies(items []*Item) []*Item {
itemsToRemove := make([]*Item, 0)
for _, item := range items {
for _, dep := range item.DependsOn {
@ -197,7 +194,7 @@ func filterItemsWithMissingDependencies(items []*PipelineItem) []*PipelineItem {
}
if len(itemsToRemove) > 0 {
filtered := make([]*PipelineItem, 0)
filtered := make([]*Item, 0)
for _, item := range items {
if !containsItemWithName(item.Step.Name, itemsToRemove) {
filtered = append(filtered, item)
@ -210,7 +207,7 @@ func filterItemsWithMissingDependencies(items []*PipelineItem) []*PipelineItem {
return items
}
func containsItemWithName(name string, items []*PipelineItem) bool {
func containsItemWithName(name string, items []*Item) bool {
for _, item := range items {
if name == item.Step.Name {
return true
@ -293,7 +290,7 @@ func (b *StepBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[
).Compile(parsed)
}
func SetPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*PipelineItem) *model.Pipeline {
func SetPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*Item) *model.Pipeline {
var pidSequence int
for _, item := range pipelineItems {
pipeline.Steps = append(pipeline.Steps, item.Step)

32
server/shared/stepBuilder_test.go → pipeline/stepBuilder_test.go
View file

@ -13,18 +13,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
package pipeline
import (
"fmt"
"testing"
"github.com/woodpecker-ci/woodpecker/server/forge"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
)
// TODO(974) move to pipeline/*
func TestGlobalEnvsubst(t *testing.T) {
t.Parallel()
@ -42,7 +40,7 @@ func TestGlobalEnvsubst(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Data: []byte(`
pipeline:
build:
@ -76,7 +74,7 @@ func TestMissingGlobalEnvsubst(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Data: []byte(`
pipeline:
build:
@ -107,7 +105,7 @@ bbb`,
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Data: []byte(`
pipeline:
xxx:
@ -141,7 +139,7 @@ func TestMultiPipeline(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Data: []byte(`
pipeline:
xxx:
@ -175,7 +173,7 @@ func TestDependsOn(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Name: "lint", Data: []byte(`
pipeline:
build:
@ -221,7 +219,7 @@ func TestRunsOn(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Data: []byte(`
pipeline:
deploy:
@ -257,7 +255,7 @@ func TestPipelineName(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Name: ".woodpecker/lint.yml", Data: []byte(`
pipeline:
build:
@ -292,7 +290,7 @@ func TestBranchFilter(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Data: []byte(`
pipeline:
xxx:
@ -338,7 +336,7 @@ func TestRootWhenFilter(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Data: []byte(`
when:
event:
@ -386,7 +384,7 @@ func TestZeroSteps(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Data: []byte(`
skip_clone: true
pipeline:
@ -420,7 +418,7 @@ func TestZeroStepsAsMultiPipelineDeps(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Name: "zerostep", Data: []byte(`
skip_clone: true
pipeline:
@ -468,7 +466,7 @@ func TestZeroStepsAsMultiPipelineTransitiveDeps(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Name: "zerostep", Data: []byte(`
skip_clone: true
pipeline:
@ -524,7 +522,7 @@ func TestTree(t *testing.T) {
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: []*forge.FileMeta{
Yamls: []*forge_types.FileMeta{
{Data: []byte(`
pipeline:
build:

18
server/api/user.go
View file

@ -25,9 +25,9 @@ import (
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/router/middleware/session"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store"
"github.com/woodpecker-ci/woodpecker/shared/token"
)
@ -38,7 +38,7 @@ func GetSelf(c *gin.Context) {
func GetFeed(c *gin.Context) {
_store := store.FromContext(c)
forge := server.Config.Services.Forge
_forge := server.Config.Services.Forge
user := session.User(c)
latest, _ := strconv.ParseBool(c.Query("latest"))
@ -54,11 +54,11 @@ func GetFeed(c *gin.Context) {
config := ToConfig(c)
sync := shared.Syncer{
Forge: forge,
sync := forge.Syncer{
Forge: _forge,
Store: _store,
Perms: _store,
Match: shared.NamespaceFilter(config.OwnersWhitelist),
Match: forge.NamespaceFilter(config.OwnersWhitelist),
}
if err := sync.Sync(c, user, server.Config.FlatPermissions); err != nil {
log.Debug().Msgf("sync error: %s: %s", user.Login, err)
@ -87,7 +87,7 @@ func GetFeed(c *gin.Context) {
func GetRepos(c *gin.Context) {
_store := store.FromContext(c)
forge := server.Config.Services.Forge
_forge := server.Config.Services.Forge
user := session.User(c)
all, _ := strconv.ParseBool(c.Query("all"))
@ -103,11 +103,11 @@ func GetRepos(c *gin.Context) {
config := ToConfig(c)
sync := shared.Syncer{
Forge: forge,
sync := forge.Syncer{
Forge: _forge,
Store: _store,
Perms: _store,
Match: shared.NamespaceFilter(config.OwnersWhitelist),
Match: forge.NamespaceFilter(config.OwnersWhitelist),
}
if err := sync.Sync(c, user, server.Config.FlatPermissions); err != nil {

9
server/forge/bitbucket/bitbucket.go
View file

@ -27,6 +27,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/bitbucket/internal"
"github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
)
@ -73,7 +74,7 @@ func (c *config) Login(ctx context.Context, w http.ResponseWriter, req *http.Req
// get the OAuth errors
if err := req.FormValue("error"); err != "" {
return nil, &forge.AuthError{
return nil, &forge_types.AuthError{
Err: err,
Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"),
@ -222,8 +223,8 @@ func (c *config) File(ctx context.Context, u *model.User, r *model.Repo, p *mode
return []byte(*config), err
}
func (c *config) Dir(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]*forge.FileMeta, error) {
return nil, fmt.Errorf("Not implemented")
func (c *config) Dir(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
return nil, forge_types.ErrNotImplemented
}
// Status creates a pipeline status for the Bitbucket commit.
@ -298,7 +299,7 @@ func (c *config) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]
// BranchHead returns the sha of the head (lastest commit) of the specified branch
func (c *config) BranchHead(ctx context.Context, u *model.User, r *model.Repo, branch string) (string, error) {
// TODO(1138): missing implementation
return "", fmt.Errorf("missing implementation")
return "", forge_types.ErrNotImplemented
}
// Hook parses the incoming Bitbucket hook and returns the Repository and

7
server/forge/bitbucketserver/bitbucketserver.go
View file

@ -33,6 +33,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/bitbucketserver/internal"
"github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
)
@ -185,8 +186,8 @@ func (c *Config) File(ctx context.Context, u *model.User, r *model.Repo, p *mode
return client.FindFileForRepo(r.Owner, r.Name, f, p.Ref)
}
func (c *Config) Dir(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]*forge.FileMeta, error) {
return nil, fmt.Errorf("Not implemented")
func (c *Config) Dir(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
return nil, forge_types.ErrNotImplemented
}
// Status is not supported by the bitbucketserver driver.
@ -240,7 +241,7 @@ func (c *Config) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]
// BranchHead returns the sha of the head (lastest commit) of the specified branch
func (c *Config) BranchHead(ctx context.Context, u *model.User, r *model.Repo, branch string) (string, error) {
// TODO(1138): missing implementation
return "", fmt.Errorf("missing implementation")
return "", forge_types.ErrNotImplemented
}
func (c *Config) Deactivate(ctx context.Context, u *model.User, r *model.Repo, link string) error {

11
server/forge/coding/coding.go
View file

@ -28,6 +28,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/coding/internal"
"github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
)
@ -87,7 +88,7 @@ func (c *Coding) Login(ctx context.Context, res http.ResponseWriter, req *http.R
// get the OAuth errors
if err := req.FormValue("error"); err != "" {
return nil, &forge.AuthError{
return nil, &forge_types.AuthError{
Err: err,
Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"),
@ -151,7 +152,7 @@ func (c *Coding) Refresh(ctx context.Context, u *model.User) (bool, error) {
// Teams fetches a list of team memberships from the forge.
func (c *Coding) Teams(ctx context.Context, u *model.User) ([]*model.Team, error) {
// EMPTY: not implemented in Coding OAuth API
return nil, fmt.Errorf("Not implemented")
return nil, forge_types.ErrNotImplemented
}
// TeamPerm fetches the named organization permissions from
@ -244,8 +245,8 @@ func (c *Coding) File(ctx context.Context, u *model.User, r *model.Repo, b *mode
return data, nil
}
func (c *Coding) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge.FileMeta, error) {
return nil, fmt.Errorf("Not implemented")
func (c *Coding) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
return nil, forge_types.ErrNotImplemented
}
// Status sends the commit status to the forge.
@ -297,7 +298,7 @@ func (c *Coding) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]
// BranchHead returns the sha of the head (lastest commit) of the specified branch
func (c *Coding) BranchHead(ctx context.Context, u *model.User, r *model.Repo, branch string) (string, error) {
// TODO(1138): missing implementation
return "", fmt.Errorf("missing implementation")
return "", forge_types.ErrNotImplemented
}
// Hook parses the post-commit hook from the Request body and returns the

33
server/shared/configFetcher.go → server/forge/configFetcher.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
package forge
import (
"context"
@ -23,27 +23,25 @@ import (
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/plugins/config"
"github.com/woodpecker-ci/woodpecker/shared/constant"
)
type ConfigFetcher interface {
Fetch(ctx context.Context) (files []*forge.FileMeta, err error)
Fetch(ctx context.Context) (files []*types.FileMeta, err error)
}
// TODO(974) move to new package
type configFetcher struct {
forge forge.Forge
forge Forge
user *model.User
repo *model.Repo
pipeline *model.Pipeline
configExtension config.Extension
}
func NewConfigFetcher(forge forge.Forge, configExtension config.Extension, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ConfigFetcher {
func NewConfigFetcher(forge Forge, configExtension config.Extension, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ConfigFetcher {
return &configFetcher{
forge: forge,
user: user,
@ -57,7 +55,7 @@ func NewConfigFetcher(forge forge.Forge, configExtension config.Extension, user
var configFetchTimeout = time.Second * 3
// Fetch pipeline config from source forge
func (cf *configFetcher) Fetch(ctx context.Context) (files []*forge.FileMeta, err error) {
func (cf *configFetcher) Fetch(ctx context.Context) (files []*types.FileMeta, err error) {
log.Trace().Msgf("Start Fetching config for '%s'", cf.repo.FullName)
// try to fetch 3 times
@ -92,7 +90,7 @@ func (cf *configFetcher) Fetch(ctx context.Context) (files []*forge.FileMeta, er
}
// fetch config by timeout
func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config string) ([]*forge.FileMeta, error) {
func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config string) ([]*types.FileMeta, error) {
ctx, cancel := context.WithTimeout(c, timeout)
defer cancel()
@ -121,12 +119,12 @@ func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config
case <-ctx.Done():
return nil, ctx.Err()
default:
return []*forge.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not find config: %s", err)
return []*types.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not find config: %s", err)
}
}
func filterPipelineFiles(files []*forge.FileMeta) []*forge.FileMeta {
var res []*forge.FileMeta
func filterPipelineFiles(files []*types.FileMeta) []*types.FileMeta {
var res []*types.FileMeta
for _, file := range files {
if strings.HasSuffix(file.Name, ".yml") || strings.HasSuffix(file.Name, ".yaml") {
@ -137,13 +135,13 @@ func filterPipelineFiles(files []*forge.FileMeta) []*forge.FileMeta {
return res
}
func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fileMeta []*forge.FileMeta, found bool) {
func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fileMeta []*types.FileMeta, found bool) {
file, err := cf.forge.File(c, cf.user, cf.repo, cf.pipeline, config)
if err == nil && len(file) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config)
return []*forge.FileMeta{{
return []*types.FileMeta{{
Name: config,
Data: file,
}}, true
@ -152,7 +150,7 @@ func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fi
return nil, false
}
func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string, userDefined bool) ([]*forge.FileMeta, error) {
func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string, userDefined bool) ([]*types.FileMeta, error) {
userDefinedLog := ""
if userDefined {
userDefinedLog = "user defined"
@ -161,8 +159,11 @@ func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []st
for _, fileOrFolder := range configs {
if strings.HasSuffix(fileOrFolder, "/") {
// config is a folder
// if folder is not supported we will get a "Not implemented" error and continue
files, err := cf.forge.Dir(c, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(fileOrFolder, "/"))
// if folder is not supported we will get a "Not implemented" error and continue
if err != nil && !errors.Is(err, types.ErrNotImplemented) {
log.Error().Err(err).Str("repo", cf.repo.FullName).Str("user", cf.user.Login).Msg("could not get folder from forge")
}
files = filterPipelineFiles(files)
if err == nil && len(files) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found %d %s files in '%s'", cf.repo.FullName, len(files), userDefinedLog, fileOrFolder)

18
server/shared/configFetcher_test.go → server/forge/configFetcher_test.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package shared_test
package forge_test
import (
"context"
@ -32,13 +32,11 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/mocks"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/plugins/config"
"github.com/woodpecker-ci/woodpecker/server/shared"
)
// TODO(974) move to new package
func TestFetch(t *testing.T) {
t.Parallel()
@ -294,12 +292,12 @@ func TestFetch(t *testing.T) {
repo := &model.Repo{Owner: "laszlocph", Name: "multipipeline", Config: tt.repoConfig}
f := new(mocks.Forge)
dirs := map[string][]*forge.FileMeta{}
dirs := map[string][]*forge_types.FileMeta{}
for _, file := range tt.files {
f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil)
path := filepath.Dir(file.name)
if path != "." {
dirs[path] = append(dirs[path], &forge.FileMeta{
dirs[path] = append(dirs[path], &forge_types.FileMeta{
Name: file.name,
Data: file.data,
})
@ -314,7 +312,7 @@ func TestFetch(t *testing.T) {
f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("File not found"))
f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("Directory not found"))
configFetcher := shared.NewConfigFetcher(
configFetcher := forge.NewConfigFetcher(
f,
config.NewHTTP("", ""),
&model.User{Token: "xxx"},
@ -499,12 +497,12 @@ func TestFetchFromConfigService(t *testing.T) {
repo := &model.Repo{Owner: "laszlocph", Name: tt.name, Config: tt.repoConfig} // Using test name as repo name to provide different responses in mock server
f := new(mocks.Forge)
dirs := map[string][]*forge.FileMeta{}
dirs := map[string][]*forge_types.FileMeta{}
for _, file := range tt.files {
f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil)
path := filepath.Dir(file.name)
if path != "." {
dirs[path] = append(dirs[path], &forge.FileMeta{
dirs[path] = append(dirs[path], &forge_types.FileMeta{
Name: file.name,
Data: file.data,
})
@ -519,7 +517,7 @@ func TestFetchFromConfigService(t *testing.T) {
f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("File not found"))
f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("Directory not found"))
configFetcher := shared.NewConfigFetcher(
configFetcher := forge.NewConfigFetcher(
f,
configAPI,
&model.User{Token: "xxx"},

9
server/forge/gitea/gitea.go
View file

@ -38,6 +38,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/store"
)
@ -112,7 +113,7 @@ func (c *Gitea) Login(ctx context.Context, w http.ResponseWriter, req *http.Requ
// get the OAuth errors
if err := req.FormValue("error"); err != "" {
return nil, &forge.AuthError{
return nil, &forge_types.AuthError{
Err: err,
Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"),
@ -292,8 +293,8 @@ func (c *Gitea) File(ctx context.Context, u *model.User, r *model.Repo, b *model
return cfg, err
}
func (c *Gitea) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge.FileMeta, error) {
var configs []*forge.FileMeta
func (c *Gitea) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
var configs []*forge_types.FileMeta
client, err := c.newClientToken(ctx, u.Token)
if err != nil {
@ -316,7 +317,7 @@ func (c *Gitea) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.
return nil, fmt.Errorf("multi-pipeline cannot get %s: %s", e.Path, err)
}
configs = append(configs, &forge.FileMeta{
configs = append(configs, &forge_types.FileMeta{
Name: e.Path,
Data: data,
})

11
server/forge/github/github.go
View file

@ -32,6 +32,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/store"
"github.com/woodpecker-ci/woodpecker/shared/utils"
@ -90,7 +91,7 @@ func (c *client) Login(ctx context.Context, res http.ResponseWriter, req *http.R
// get the OAuth errors
if err := req.FormValue("error"); err != "" {
return nil, &forge.AuthError{
return nil, &forge_types.AuthError{
Err: err,
Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"),
@ -235,7 +236,7 @@ func (c *client) File(ctx context.Context, u *model.User, r *model.Repo, b *mode
return []byte(data), err
}
func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge.FileMeta, error) {
func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
client := c.newClientToken(ctx, u.Token)
opts := new(github.RepositoryContentGetOptions)
@ -245,7 +246,7 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model
return nil, err
}
fc := make(chan *forge.FileMeta)
fc := make(chan *forge_types.FileMeta)
errc := make(chan error)
for _, file := range data {
@ -254,7 +255,7 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model
if err != nil {
errc <- err
} else {
fc <- &forge.FileMeta{
fc <- &forge_types.FileMeta{
Name: path,
Data: content,
}
@ -262,7 +263,7 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model
}(f + "/" + *file.Name)
}
var files []*forge.FileMeta
var files []*forge_types.FileMeta
for i := 0; i < len(data); i++ {
select {

9
server/forge/gitlab/gitlab.go
View file

@ -33,6 +33,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/store"
"github.com/woodpecker-ci/woodpecker/shared/utils"
@ -102,7 +103,7 @@ func (g *Gitlab) Login(ctx context.Context, res http.ResponseWriter, req *http.R
// get the OAuth errors
if err := req.FormValue("error"); err != "" {
return nil, &forge.AuthError{
return nil, &forge_types.AuthError{
Err: err,
Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"),
@ -339,13 +340,13 @@ func (g *Gitlab) File(ctx context.Context, user *model.User, repo *model.Repo, p
}
// Dir fetches a folder from the forge repository
func (g *Gitlab) Dir(ctx context.Context, user *model.User, repo *model.Repo, pipeline *model.Pipeline, path string) ([]*forge.FileMeta, error) {
func (g *Gitlab) Dir(ctx context.Context, user *model.User, repo *model.Repo, pipeline *model.Pipeline, path string) ([]*forge_types.FileMeta, error) {
client, err := newClient(g.URL, user.Token, g.SkipVerify)
if err != nil {
return nil, err
}
files := make([]*forge.FileMeta, 0, perPage)
files := make([]*forge_types.FileMeta, 0, perPage)
_repo, err := g.getProject(ctx, client, repo.Owner, repo.Name)
if err != nil {
return nil, err
@ -372,7 +373,7 @@ func (g *Gitlab) Dir(ctx context.Context, user *model.User, repo *model.Repo, pi
if err != nil {
return nil, err
}
files = append(files, &forge.FileMeta{
files = append(files, &forge_types.FileMeta{
Name: batch[i].Path,
Data: data,
})

5
server/forge/gogs/gogs.go
View file

@ -28,6 +28,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/common"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
)
@ -209,8 +210,8 @@ func (c *client) File(ctx context.Context, u *model.User, r *model.Repo, b *mode
return cfg, err
}
func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge.FileMeta, error) {
return nil, fmt.Errorf("Not implemented")
func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge_types.FileMeta, error) {
return nil, forge_types.ErrNotImplemented
}
// Status is not supported by the Gogs driver.

15
server/forge/mocks/forge.go
View file

@ -1,16 +1,17 @@
// Code generated by mockery v2.14.0. DO NOT EDIT.
// Code generated by mockery v2.14.1. DO NOT EDIT.
package mocks
import (
context "context"
http "net/http"
forge "github.com/woodpecker-ci/woodpecker/server/forge"
http "net/http"
mock "github.com/stretchr/testify/mock"
model "github.com/woodpecker-ci/woodpecker/server/model"
types "github.com/woodpecker-ci/woodpecker/server/forge/types"
)
// Forge is an autogenerated mock type for the Forge type
@ -112,15 +113,15 @@ func (_m *Forge) Deactivate(ctx context.Context, u *model.User, r *model.Repo, l
}
// Dir provides a mock function with given fields: ctx, u, r, b, f
func (_m *Forge) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*forge.FileMeta, error) {
func (_m *Forge) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*types.FileMeta, error) {
ret := _m.Called(ctx, u, r, b, f)
var r0 []*forge.FileMeta
if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo, *model.Pipeline, string) []*forge.FileMeta); ok {
var r0 []*types.FileMeta
if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo, *model.Pipeline, string) []*types.FileMeta); ok {
r0 = rf(ctx, u, r, b, f)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*forge.FileMeta)
r0 = ret.Get(0).([]*types.FileMeta)
}
}

15
server/forge/remote.go
View file

@ -22,6 +22,7 @@ import (
"context"
"net/http"
"github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
)
@ -58,7 +59,7 @@ type Forge interface {
File(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error)
// Dir fetches a folder from the forge repository
Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*FileMeta, error)
Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]*types.FileMeta, error)
// Status sends the commit status to the forge.
// An example would be the GitHub pull request status.
@ -91,18 +92,6 @@ type Forge interface {
OrgMembership(ctx context.Context, u *model.User, owner string) (*model.OrgPerm, error)
}
// FileMeta represents a file in version control
type FileMeta struct {
Name string
Data []byte
}
type ByName []*FileMeta
func (a ByName) Len() int { return len(a) }
func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name }
func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
// Refresher refreshes an oauth token and expiration for the given user. It
// returns true if the token was refreshed, false if the token was not refreshed,
// and error if it failed to refersh.

7
server/forge/errors.go → server/forge/types/errors.go
View file

@ -1,3 +1,4 @@
// Copyright 2022 Woodpecker Authors
// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
@ -12,7 +13,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package forge
package types
import "errors"
// AuthError represents forge authentication error.
type AuthError struct {
@ -35,3 +38,5 @@ func (ae *AuthError) Error() string {
// check interface
var _ error = new(AuthError)
var ErrNotImplemented = errors.New("Not implemented")

35
server/forge/types/meta.go Normal file
View file

@ -0,0 +1,35 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import "sort"
// FileMeta represents a file in version control
type FileMeta struct {
Name string
Data []byte
}
type fileMetaList []*FileMeta
func (a fileMetaList) Len() int { return len(a) }
func (a fileMetaList) Less(i, j int) bool { return a[i].Name < a[j].Name }
func (a fileMetaList) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func SortByName(fm []*FileMeta) []*FileMeta {
l := fileMetaList(fm)
sort.Sort(l)
return l
}

7
server/shared/userSyncer.go → server/forge/userSyncer.go
View file

@ -13,27 +13,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
package forge
import (
"context"
"fmt"
"time"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/store"
)
// TODO(974) move to new package
// UserSyncer syncs the user repository and permissions.
type UserSyncer interface {
Sync(ctx context.Context, user *model.User) error
}
type Syncer struct {
Forge forge.Forge
Forge Forge
Store store.Store
Perms model.PermStore
Match FilterFunc

66
server/grpc/rpc.go
View file

@ -35,9 +35,9 @@ import (
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/logging"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/pipeline"
"github.com/woodpecker-ci/woodpecker/server/pubsub"
"github.com/woodpecker-ci/woodpecker/server/queue"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store"
)
@ -108,13 +108,13 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error {
return err
}
pipeline, err := s.store.GetPipeline(pstep.PipelineID)
currentPipeline, err := s.store.GetPipeline(pstep.PipelineID)
if err != nil {
log.Error().Msgf("error: cannot find pipeline with id %d: %s", pstep.PipelineID, err)
return err
}
step, err := s.store.StepChild(pipeline, pstep.PID, state.Step)
step, err := s.store.StepChild(currentPipeline, pstep.PID, state.Step)
if err != nil {
log.Error().Msgf("error: cannot find step with name %s: %s", state.Step, err)
return err
@ -128,20 +128,20 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error {
}
}
repo, err := s.store.GetRepo(pipeline.RepoID)
repo, err := s.store.GetRepo(currentPipeline.RepoID)
if err != nil {
log.Error().Msgf("error: cannot find repo with id %d: %s", pipeline.RepoID, err)
log.Error().Msgf("error: cannot find repo with id %d: %s", currentPipeline.RepoID, err)
return err
}
if _, err = shared.UpdateStepStatus(s.store, *step, state, pipeline.Started); err != nil {
if _, err = pipeline.UpdateStepStatus(s.store, *step, state, currentPipeline.Started); err != nil {
log.Error().Err(err).Msg("rpc.update: cannot update step")
}
if pipeline.Steps, err = s.store.StepList(pipeline); err != nil {
if currentPipeline.Steps, err = s.store.StepList(currentPipeline); err != nil {
log.Error().Err(err).Msg("can not get step list from store")
}
if pipeline.Steps, err = model.Tree(pipeline.Steps); err != nil {
if currentPipeline.Steps, err = model.Tree(currentPipeline.Steps); err != nil {
log.Error().Err(err).Msg("can not build tree from step list")
return err
}
@ -153,7 +153,7 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error {
}
message.Data, _ = json.Marshal(model.Event{
Repo: *repo,
Pipeline: *pipeline,
Pipeline: *currentPipeline,
})
if err := s.pubsub.Publish(c, "topic/events", message); err != nil {
log.Error().Err(err).Msg("can not publish step list to")
@ -255,26 +255,26 @@ func (s *RPC) Init(c context.Context, id string, state rpc.State) error {
}
}
pipeline, err := s.store.GetPipeline(step.PipelineID)
currentPipeline, err := s.store.GetPipeline(step.PipelineID)
if err != nil {
log.Error().Msgf("error: cannot find pipeline with id %d: %s", step.PipelineID, err)
return err
}
repo, err := s.store.GetRepo(pipeline.RepoID)
repo, err := s.store.GetRepo(currentPipeline.RepoID)
if err != nil {
log.Error().Msgf("error: cannot find repo with id %d: %s", pipeline.RepoID, err)
log.Error().Msgf("error: cannot find repo with id %d: %s", currentPipeline.RepoID, err)
return err
}
if pipeline.Status == model.StatusPending {
if pipeline, err = shared.UpdateToStatusRunning(s.store, *pipeline, state.Started); err != nil {
log.Error().Msgf("error: init: cannot update build_id %d state: %s", pipeline.ID, err)
if currentPipeline.Status == model.StatusPending {
if currentPipeline, err = pipeline.UpdateToStatusRunning(s.store, *currentPipeline, state.Started); err != nil {
log.Error().Msgf("error: init: cannot update build_id %d state: %s", currentPipeline.ID, err)
}
}
defer func() {
pipeline.Steps, _ = s.store.StepList(pipeline)
currentPipeline.Steps, _ = s.store.StepList(currentPipeline)
message := pubsub.Message{
Labels: map[string]string{
"repo": repo.FullName,
@ -283,14 +283,14 @@ func (s *RPC) Init(c context.Context, id string, state rpc.State) error {
}
message.Data, _ = json.Marshal(model.Event{
Repo: *repo,
Pipeline: *pipeline,
Pipeline: *currentPipeline,
})
if err := s.pubsub.Publish(c, "topic/events", message); err != nil {
log.Error().Err(err).Msg("can not publish step list to")
}
}()
_, err = shared.UpdateStepToStatusStarted(s.store, *step, state)
_, err = pipeline.UpdateStepToStatusStarted(s.store, *step, state)
return err
}
@ -307,25 +307,25 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error {
return err
}
pipeline, err := s.store.GetPipeline(step.PipelineID)
currentPipeline, err := s.store.GetPipeline(step.PipelineID)
if err != nil {
log.Error().Msgf("error: cannot find pipeline with id %d: %s", step.PipelineID, err)
return err
}
repo, err := s.store.GetRepo(pipeline.RepoID)
repo, err := s.store.GetRepo(currentPipeline.RepoID)
if err != nil {
log.Error().Msgf("error: cannot find repo with id %d: %s", pipeline.RepoID, err)
log.Error().Msgf("error: cannot find repo with id %d: %s", currentPipeline.RepoID, err)
return err
}
log.Trace().
Str("repo_id", fmt.Sprint(repo.ID)).
Str("build_id", fmt.Sprint(pipeline.ID)).
Str("build_id", fmt.Sprint(currentPipeline.ID)).
Str("step_id", id).
Msgf("gRPC Done with state: %#v", state)
if step, err = shared.UpdateStepStatusToDone(s.store, *step, state); err != nil {
if step, err = pipeline.UpdateStepStatusToDone(s.store, *step, state); err != nil {
log.Error().Msgf("error: done: cannot update step_id %d state: %s", step.ID, err)
}
@ -339,34 +339,34 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error {
log.Error().Msgf("error: done: cannot ack step_id %d: %s", stepID, err)
}
steps, err := s.store.StepList(pipeline)
steps, err := s.store.StepList(currentPipeline)
if err != nil {
return err
}
s.completeChildrenIfParentCompleted(steps, step)
if !model.IsThereRunningStage(steps) {
if pipeline, err = shared.UpdateStatusToDone(s.store, *pipeline, model.PipelineStatus(steps), step.Stopped); err != nil {
log.Error().Err(err).Msgf("error: done: cannot update build_id %d final state", pipeline.ID)
if currentPipeline, err = pipeline.UpdateStatusToDone(s.store, *currentPipeline, model.PipelineStatus(steps), step.Stopped); err != nil {
log.Error().Err(err).Msgf("error: done: cannot update build_id %d final state", currentPipeline.ID)
}
}
s.updateForgeStatus(c, repo, pipeline, step)
s.updateForgeStatus(c, repo, currentPipeline, step)
if err := s.logger.Close(c, id); err != nil {
log.Error().Err(err).Msgf("done: cannot close build_id %d logger", step.ID)
}
if err := s.notify(c, repo, pipeline, steps); err != nil {
if err := s.notify(c, repo, currentPipeline, steps); err != nil {
return err
}
if pipeline.Status == model.StatusSuccess || pipeline.Status == model.StatusFailure {
s.pipelineCount.WithLabelValues(repo.FullName, pipeline.Branch, string(pipeline.Status), "total").Inc()
s.pipelineTime.WithLabelValues(repo.FullName, pipeline.Branch, string(pipeline.Status), "total").Set(float64(pipeline.Finished - pipeline.Started))
if currentPipeline.Status == model.StatusSuccess || currentPipeline.Status == model.StatusFailure {
s.pipelineCount.WithLabelValues(repo.FullName, currentPipeline.Branch, string(currentPipeline.Status), "total").Inc()
s.pipelineTime.WithLabelValues(repo.FullName, currentPipeline.Branch, string(currentPipeline.Status), "total").Set(float64(currentPipeline.Finished - currentPipeline.Started))
}
if model.IsMultiPipeline(steps) {
s.pipelineTime.WithLabelValues(repo.FullName, pipeline.Branch, string(step.State), step.Name).Set(float64(step.Stopped - step.Started))
s.pipelineTime.WithLabelValues(repo.FullName, currentPipeline.Branch, string(step.State), step.Name).Set(float64(step.Stopped - step.Started))
}
return nil
@ -385,7 +385,7 @@ func (s *RPC) Log(c context.Context, id string, line *rpc.Line) error {
func (s *RPC) completeChildrenIfParentCompleted(steps []*model.Step, completedStep *model.Step) {
for _, p := range steps {
if p.Running() && p.PPID == completedStep.PID {
if _, err := shared.UpdateStepToStatusSkipped(s.store, *p, completedStep.Stopped); err != nil {
if _, err := pipeline.UpdateStepToStatusSkipped(s.store, *p, completedStep.Stopped); err != nil {
log.Error().Msgf("error: done: cannot update step_id %d child state: %s", p.ID, err)
}
}

23
server/pipeline/approve.go
View file

@ -20,49 +20,48 @@ import (
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server/forge"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store"
)
// Approve update the status to pending for blocked pipeline because of a gated repo
// and start them afterwards
func Approve(ctx context.Context, store store.Store, pipeline *model.Pipeline, user *model.User, repo *model.Repo) (*model.Pipeline, error) {
if pipeline.Status != model.StatusBlocked {
return nil, ErrBadRequest{Msg: fmt.Sprintf("cannot decline a pipeline with status %s", pipeline.Status)}
func Approve(ctx context.Context, store store.Store, currentPipeline *model.Pipeline, user *model.User, repo *model.Repo) (*model.Pipeline, error) {
if currentPipeline.Status != model.StatusBlocked {
return nil, ErrBadRequest{Msg: fmt.Sprintf("cannot decline a pipeline with status %s", currentPipeline.Status)}
}
// fetch the pipeline file from the database
configs, err := store.ConfigsForPipeline(pipeline.ID)
configs, err := store.ConfigsForPipeline(currentPipeline.ID)
if err != nil {
msg := fmt.Sprintf("failure to get pipeline config for %s. %s", repo.FullName, err)
log.Error().Msg(msg)
return nil, ErrNotFound{Msg: msg}
}
if pipeline, err = shared.UpdateToStatusPending(store, *pipeline, user.Login); err != nil {
if currentPipeline, err = UpdateToStatusPending(store, *currentPipeline, user.Login); err != nil {
return nil, fmt.Errorf("error updating pipeline. %s", err)
}
var yamls []*forge.FileMeta
var yamls []*forge_types.FileMeta
for _, y := range configs {
yamls = append(yamls, &forge.FileMeta{Data: y.Data, Name: y.Name})
yamls = append(yamls, &forge_types.FileMeta{Data: y.Data, Name: y.Name})
}
pipeline, pipelineItems, err := createPipelineItems(ctx, store, pipeline, user, repo, yamls, nil)
currentPipeline, pipelineItems, err := createPipelineItems(ctx, store, currentPipeline, user, repo, yamls, nil)
if err != nil {
msg := fmt.Sprintf("failure to createBuildItems for %s", repo.FullName)
log.Error().Err(err).Msg(msg)
return nil, err
}
pipeline, err = start(ctx, store, pipeline, user, repo, pipelineItems)
currentPipeline, err = start(ctx, store, currentPipeline, user, repo, pipelineItems)
if err != nil {
msg := fmt.Sprintf("failure to start pipeline for %s: %v", repo.FullName, err)
log.Error().Err(err).Msg(msg)
return nil, fmt.Errorf(msg)
}
return pipeline, nil
return currentPipeline, nil
}

7
server/pipeline/cancel.go
View file

@ -23,7 +23,6 @@ import (
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/queue"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store"
)
@ -74,18 +73,18 @@ func Cancel(ctx context.Context, store store.Store, repo *model.Repo, pipeline *
for _, step := range steps {
if step.State == model.StatusPending {
if step.PPID != 0 {
if _, err = shared.UpdateStepToStatusSkipped(store, *step, 0); err != nil {
if _, err = UpdateStepToStatusSkipped(store, *step, 0); err != nil {
log.Error().Msgf("error: done: cannot update step_id %d state: %s", step.ID, err)
}
} else {
if _, err = shared.UpdateStepToStatusKilled(store, *step); err != nil {
if _, err = UpdateStepToStatusKilled(store, *step); err != nil {
log.Error().Msgf("error: done: cannot update step_id %d state: %s", step.ID, err)
}
}
}
}
killedBuild, err := shared.UpdateToStatusKilled(store, *pipeline)
killedBuild, err := UpdateToStatusKilled(store, *pipeline)
if err != nil {
log.Error().Err(err).Msgf("UpdateToStatusKilled: %v", pipeline)
return err

16
server/pipeline/config.go
View file

@ -18,26 +18,26 @@ import (
"crypto/sha256"
"fmt"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/pipeline"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store"
)
func findOrPersistPipelineConfig(store store.Store, pipeline *model.Pipeline, forgeYamlConfig *forge.FileMeta) (*model.Config, error) {
func findOrPersistPipelineConfig(store store.Store, currentPipeline *model.Pipeline, forgeYamlConfig *forge_types.FileMeta) (*model.Config, error) {
sha := fmt.Sprintf("%x", sha256.Sum256(forgeYamlConfig.Data))
conf, err := store.ConfigFindIdentical(pipeline.RepoID, sha)
conf, err := store.ConfigFindIdentical(currentPipeline.RepoID, sha)
if err != nil {
conf = &model.Config{
RepoID: pipeline.RepoID,
RepoID: currentPipeline.RepoID,
Data: forgeYamlConfig.Data,
Hash: sha,
Name: shared.SanitizePath(forgeYamlConfig.Name),
Name: pipeline.SanitizePath(forgeYamlConfig.Name),
}
err = store.ConfigCreate(conf)
if err != nil {
// retry in case we receive two hooks at the same time
conf, err = store.ConfigFindIdentical(pipeline.RepoID, sha)
conf, err = store.ConfigFindIdentical(currentPipeline.RepoID, sha)
if err != nil {
return nil, err
}
@ -46,7 +46,7 @@ func findOrPersistPipelineConfig(store store.Store, pipeline *model.Pipeline, fo
pipelineConfig := &model.PipelineConfig{
ConfigID: conf.ID,
PipelineID: pipeline.ID,
PipelineID: currentPipeline.ID,
}
if err := store.PipelineConfigCreate(pipelineConfig); err != nil {
return nil, err

6
server/pipeline/create.go
View file

@ -23,8 +23,8 @@ import (
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store"
)
@ -53,14 +53,14 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline
}
var (
forgeYamlConfigs []*forge.FileMeta
forgeYamlConfigs []*types.FileMeta
configFetchErr error
filtered bool
parseErr error
)
// fetch the pipeline file from the forge
configFetcher := shared.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.ConfigService, repoUser, repo, pipeline)
configFetcher := forge.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.ConfigService, repoUser, repo, pipeline)
forgeYamlConfigs, configFetchErr = configFetcher.Fetch(ctx)
if configFetchErr == nil {
filtered, parseErr = checkIfFiltered(pipeline, forgeYamlConfigs)

3
server/pipeline/decline.go
View file

@ -20,7 +20,6 @@ import (
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store"
)
@ -30,7 +29,7 @@ func Decline(ctx context.Context, store store.Store, pipeline *model.Pipeline, u
return nil, fmt.Errorf("cannot decline a pipeline with status %s", pipeline.Status)
}
_, err := shared.UpdateToStatusDeclined(store, *pipeline, user.Login)
_, err := UpdateToStatusDeclined(store, *pipeline, user.Login)
if err != nil {
return nil, fmt.Errorf("error updating pipeline. %s", err)
}

12
server/pipeline/filter.go
View file

@ -19,17 +19,17 @@ package pipeline
import (
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/server/forge"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
)
func zeroSteps(pipeline *model.Pipeline, forgeYamlConfigs []*forge.FileMeta) bool {
b := shared.StepBuilder{
func zeroSteps(currentPipeline *model.Pipeline, forgeYamlConfigs []*forge_types.FileMeta) bool {
b := pipeline.StepBuilder{
Repo: &model.Repo{},
Curr: pipeline,
Curr: currentPipeline,
Last: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
@ -51,7 +51,7 @@ func zeroSteps(pipeline *model.Pipeline, forgeYamlConfigs []*forge.FileMeta) boo
// TODO: parse yaml once and not for each filter function
// Check if at least one pipeline step will be execute otherwise we will just ignore this webhook
func checkIfFiltered(pipeline *model.Pipeline, forgeYamlConfigs []*forge.FileMeta) (bool, error) {
func checkIfFiltered(pipeline *model.Pipeline, forgeYamlConfigs []*forge_types.FileMeta) (bool, error) {
log.Trace().Msgf("hook.branchFiltered(): pipeline branch: '%s' pipeline event: '%s' config count: %d", pipeline.Branch, pipeline.Event, len(forgeYamlConfigs))
matchMetadata := frontend.Metadata{

35
server/pipeline/items.go
View file

@ -21,33 +21,36 @@ import (
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store"
)
func createPipelineItems(ctx context.Context, store store.Store, pipeline *model.Pipeline, user *model.User, repo *model.Repo, yamls []*forge.FileMeta, envs map[string]string) (*model.Pipeline, []*shared.PipelineItem, error) {
func createPipelineItems(ctx context.Context, store store.Store,
currentPipeline *model.Pipeline, user *model.User, repo *model.Repo,
yamls []*forge_types.FileMeta, envs map[string]string,
) (*model.Pipeline, []*pipeline.Item, error) {
netrc, err := server.Config.Services.Forge.Netrc(user, repo)
if err != nil {
log.Error().Err(err).Msg("Failed to generate netrc file")
}
// get the previous pipeline so that we can send status change notifications
last, err := store.GetPipelineLastBefore(repo, pipeline.Branch, pipeline.ID)
last, err := store.GetPipelineLastBefore(repo, currentPipeline.Branch, currentPipeline.ID)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
log.Error().Err(err).Str("repo", repo.FullName).Msgf("Error getting last pipeline before pipeline number '%d'", pipeline.Number)
log.Error().Err(err).Str("repo", repo.FullName).Msgf("Error getting last pipeline before pipeline number '%d'", currentPipeline.Number)
}
secs, err := server.Config.Services.Secrets.SecretListPipeline(repo, pipeline)
secs, err := server.Config.Services.Secrets.SecretListPipeline(repo, currentPipeline)
if err != nil {
log.Error().Err(err).Msgf("Error getting secrets for %s#%d", repo.FullName, pipeline.Number)
log.Error().Err(err).Msgf("Error getting secrets for %s#%d", repo.FullName, currentPipeline.Number)
}
regs, err := server.Config.Services.Registries.RegistryList(repo)
if err != nil {
log.Error().Err(err).Msgf("Error getting registry credentials for %s#%d", repo.FullName, pipeline.Number)
log.Error().Err(err).Msgf("Error getting registry credentials for %s#%d", repo.FullName, currentPipeline.Number)
}
if envs == nil {
@ -60,13 +63,13 @@ func createPipelineItems(ctx context.Context, store store.Store, pipeline *model
}
}
for k, v := range pipeline.AdditionalVariables {
for k, v := range currentPipeline.AdditionalVariables {
envs[k] = v
}
b := shared.StepBuilder{
b := pipeline.StepBuilder{
Repo: repo,
Curr: pipeline,
Curr: currentPipeline,
Last: last,
Netrc: netrc,
Secs: secs,
@ -77,14 +80,14 @@ func createPipelineItems(ctx context.Context, store store.Store, pipeline *model
}
pipelineItems, err := b.Build()
if err != nil {
pipeline, uerr := shared.UpdateToStatusError(store, *pipeline, err)
currentPipeline, uerr := UpdateToStatusError(store, *currentPipeline, err)
if uerr != nil {
log.Error().Err(err).Msgf("Error setting error status of pipeline for %s#%d", repo.FullName, pipeline.Number)
log.Error().Err(err).Msgf("Error setting error status of pipeline for %s#%d", repo.FullName, currentPipeline.Number)
}
return pipeline, nil, err
return currentPipeline, nil, err
}
pipeline = shared.SetPipelineStepsOnPipeline(b.Curr, pipelineItems)
currentPipeline = pipeline.SetPipelineStepsOnPipeline(b.Curr, pipelineItems)
return pipeline, pipelineItems, nil
return currentPipeline, pipelineItems, nil
}

4
server/shared/pipelineStatus.go → server/pipeline/pipelineStatus.go
View file

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
package pipeline
import (
"time"
@ -21,8 +21,6 @@ import (
"github.com/woodpecker-ci/woodpecker/server/model"
)
// TODO(974) move to server/pipeline/*
func UpdateToStatusRunning(store model.UpdatePipelineStore, pipeline model.Pipeline, started int64) (*model.Pipeline, error) {
pipeline.Status = model.StatusRunning
pipeline.Started = started

4
server/shared/pipelineStatus_test.go → server/pipeline/pipelineStatus_test.go
View file

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
package pipeline
import (
"errors"
@ -23,8 +23,6 @@ import (
"github.com/woodpecker-ci/woodpecker/server/model"
)
// TODO(974) move to server/pipeline/*
type mockUpdatePipelineStore struct{}
func (m *mockUpdatePipelineStore) UpdatePipeline(pipeline *model.Pipeline) error {

6
server/pipeline/queue.go
View file

@ -19,14 +19,14 @@ import (
"encoding/json"
"fmt"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/pipeline/rpc"
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/queue"
"github.com/woodpecker-ci/woodpecker/server/shared"
)
func queueBuild(pipeline *model.Pipeline, repo *model.Repo, pipelineItems []*shared.PipelineItem) error {
func queueBuild(pipeline *model.Pipeline, repo *model.Repo, pipelineItems []*pipeline.Item) error {
var tasks []*queue.Task
for _, item := range pipelineItems {
if item.Step.State == model.StatusSkipped {
@ -58,7 +58,7 @@ func queueBuild(pipeline *model.Pipeline, repo *model.Repo, pipelineItems []*sha
return server.Config.Services.Queue.PushAtOnce(context.Background(), tasks)
}
func taskIds(dependsOn []string, pipelineItems []*shared.PipelineItem) (taskIds []string) {
func taskIds(dependsOn []string, pipelineItems []*pipeline.Item) (taskIds []string) {
for _, dep := range dependsOn {
for _, pipelineItem := range pipelineItems {
if pipelineItem.Step.Name == dep {

13
server/pipeline/restart.go
View file

@ -24,9 +24,8 @@ import (
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store"
)
@ -38,7 +37,7 @@ func Restart(ctx context.Context, store store.Store, lastBuild *model.Pipeline,
return nil, ErrBadRequest{Msg: fmt.Sprintf("cannot restart a pipeline with status %s", lastBuild.Status)}
}
var pipelineFiles []*forge.FileMeta
var pipelineFiles []*forge_types.FileMeta
// fetch the old pipeline config from database
configs, err := store.ConfigsForPipeline(lastBuild.ID)
@ -49,14 +48,14 @@ func Restart(ctx context.Context, store store.Store, lastBuild *model.Pipeline,
}
for _, y := range configs {
pipelineFiles = append(pipelineFiles, &forge.FileMeta{Data: y.Data, Name: y.Name})
pipelineFiles = append(pipelineFiles, &forge_types.FileMeta{Data: y.Data, Name: y.Name})
}
// If config extension is active we should refetch the config in case something changed
if server.Config.Services.ConfigService != nil && server.Config.Services.ConfigService.IsConfigured() {
currentFileMeta := make([]*forge.FileMeta, len(configs))
currentFileMeta := make([]*forge_types.FileMeta, len(configs))
for i, cfg := range configs {
currentFileMeta[i] = &forge.FileMeta{Name: cfg.Name, Data: cfg.Data}
currentFileMeta[i] = &forge_types.FileMeta{Name: cfg.Name, Data: cfg.Data}
}
newConfig, useOld, err := server.Config.Services.ConfigService.FetchConfig(ctx, repo, lastBuild, currentFileMeta)
@ -81,7 +80,7 @@ func Restart(ctx context.Context, store store.Store, lastBuild *model.Pipeline,
}
if len(configs) == 0 {
newBuild, uerr := shared.UpdateToStatusError(store, *newBuild, errors.New("pipeline definition not found"))
newBuild, uerr := UpdateToStatusError(store, *newBuild, errors.New("pipeline definition not found"))
if uerr != nil {
log.Debug().Err(uerr).Msg("failure to update pipeline status")
}

4
server/pipeline/start.go
View file

@ -19,13 +19,13 @@ import (
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/store"
)
// start a pipeline, make sure it was stored persistent in the store before
func start(ctx context.Context, store store.Store, activePipeline *model.Pipeline, user *model.User, repo *model.Repo, pipelineItems []*shared.PipelineItem) (*model.Pipeline, error) {
func start(ctx context.Context, store store.Store, activePipeline *model.Pipeline, user *model.User, repo *model.Repo, pipelineItems []*pipeline.Item) (*model.Pipeline, error) {
// call to cancel previous pipelines if needed
if err := cancelPreviousPipelines(ctx, store, activePipeline, repo); err != nil {
// should be not breaking

4
server/shared/stepStatus.go → server/pipeline/stepStatus.go
View file

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
package pipeline
import (
"time"
@ -22,8 +22,6 @@ import (
"github.com/woodpecker-ci/woodpecker/server/model"
)
// TODO(974) move to server/pipeline/*
func UpdateStepStatus(store model.UpdateStepStore, step model.Step, state rpc.State, started int64) (*model.Step, error) {
if state.Exited {
step.Stopped = state.Finished

4
server/shared/stepStatus_test.go → server/pipeline/stepStatus_test.go
View file

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
package pipeline
import (
"testing"
@ -23,8 +23,6 @@ import (
"github.com/woodpecker-ci/woodpecker/server/model"
)
// TODO(974) move to server/pipeline/*
type mockUpdateStepStore struct{}
func (m *mockUpdateStepStore) StepUpdate(build *model.Step) error {

4
server/plugins/config/extension.go
View file

@ -17,11 +17,11 @@ package config
import (
"context"
"github.com/woodpecker-ci/woodpecker/server/forge"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
)
type Extension interface {
IsConfigured() bool
FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge.FileMeta) (configData []*forge.FileMeta, useOld bool, err error)
FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge_types.FileMeta) (configData []*forge_types.FileMeta, useOld bool, err error)
}

12
server/plugins/config/http.go
View file

@ -19,7 +19,7 @@ import (
"crypto"
"fmt"
"github.com/woodpecker-ci/woodpecker/server/forge"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/plugins/utils"
)
@ -53,7 +53,7 @@ func (cp *http) IsConfigured() bool {
return cp.endpoint != ""
}
func (cp *http) FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge.FileMeta) (configData []*forge.FileMeta, useOld bool, err error) {
func (cp *http) FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge_types.FileMeta) (configData []*forge_types.FileMeta, useOld bool, err error) {
currentConfigs := make([]*config, len(currentFileMeta))
for i, pipe := range currentFileMeta {
currentConfigs[i] = &config{Name: pipe.Name, Data: string(pipe.Data)}
@ -66,13 +66,13 @@ func (cp *http) FetchConfig(ctx context.Context, repo *model.Repo, pipeline *mod
return nil, false, fmt.Errorf("Failed to fetch config via http (%d) %w", status, err)
}
var newFileMeta []*forge.FileMeta
var newFileMeta []*forge_types.FileMeta
if status != 200 {
newFileMeta = make([]*forge.FileMeta, 0)
newFileMeta = make([]*forge_types.FileMeta, 0)
} else {
newFileMeta = make([]*forge.FileMeta, len(response.Configs))
newFileMeta = make([]*forge_types.FileMeta, len(response.Configs))
for i, pipe := range response.Configs {
newFileMeta[i] = &forge.FileMeta{Name: pipe.Name, Data: []byte(pipe.Data)}
newFileMeta[i] = &forge_types.FileMeta{Name: pipe.Name, Data: []byte(pipe.Data)}
}
}

2
server/store/mocks/store.go
View file

@ -1,4 +1,4 @@
// Code generated by mockery v2.14.0. DO NOT EDIT.
// Code generated by mockery v2.14.1. DO NOT EDIT.
package mocks