Support .yaml as file-ending for workflow config too (#1388)

This implements #1073, adds .yaml to the accepted endings for woodpecker configs.

This currently adds some more lines to the duplication (tried to compensate by fixing the other duplication in the configFetcher) as the CLI and Server are still separate.

cli/common/pipeline.go

@@ -3,26 +3,23 @@ package common
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/urfave/cli/v2"
+
+	"github.com/woodpecker-ci/woodpecker/shared/constant"
 )
 
-func DetectPipelineConfig() (multiplies bool, config string, _ error) {
-	config = ".woodpecker"
-	if fi, err := os.Stat(config); err == nil && fi.IsDir() {
-		return true, config, nil
+func DetectPipelineConfig() (isDir bool, config string, _ error) {
+	for _, config := range constant.DefaultConfigOrder {
+		shouldBeDir := strings.HasSuffix(config, "/")
+		config = strings.TrimSuffix(config, "/")
+
+		if fi, err := os.Stat(config); err == nil && shouldBeDir == fi.IsDir() {
+			return fi.IsDir(), config, nil
+		}
 	}
 
-	config = ".woodpecker.yml"
-	if fi, err := os.Stat(config); err == nil && !fi.IsDir() {
-		return true, config, nil
-	}
-
-	config = ".drone.yml"
-	fi, err := os.Stat(config)
-	if err == nil && !fi.IsDir() {
-		return false, config, nil
-	}
 	return false, "", fmt.Errorf("could not detect pipeline config")
 }
 

docs/docs/20-usage/25-multi-pipeline.md

@@ -6,7 +6,7 @@ This Feature is only available for GitHub, Gitea & GitLab repositories. Follow [
 
 By default, Woodpecker looks for the pipeline definition in `.woodpecker.yml` in the project root.
 
-The Multi-Pipeline feature allows the pipeline to be split into several files and placed in the `.woodpecker/` folder. Only `.yml` files will be used and files in any subfolders like `.woodpecker/sub-folder/test.yml` will be ignored. You can set some custom path like `.my-ci/pipelines/` instead of `.woodpecker/` in the [project settings](./71-project-settings.md).
+The Multi-Pipeline feature allows the pipeline to be split into several files and placed in the `.woodpecker/` folder. Only `.yml` and `.yaml` files will be used and files in any subfolders like `.woodpecker/sub-folder/test.yml` will be ignored. You can set some custom path like `.my-ci/pipelines/` instead of `.woodpecker/` in the [project settings](./71-project-settings.md).
 
 ## Rational
 
@@ -90,7 +90,7 @@ The pipelines run in parallel on separate agents and share nothing.
 
 Dependencies between pipelines can be set with the `depends_on` element. A pipeline doesn't execute until all of its dependencies finished successfully.
 
-The name for a `depends_on` entry is the filename without the path, leading dots and without the file extension `.yml`. If the project config for example uses `.woodpecker/` as path for CI files with a file named `.woodpecker/.lint.yml` the corresponding `depends_on` entry would be `lint`.
+The name for a `depends_on` entry is the filename without the path, leading dots and without the file extension `.yml` or `.yaml`. If the project config for example uses `.woodpecker/` as path for CI files with a file named `.woodpecker/.lint.yml` the corresponding `depends_on` entry would be `lint`.
 
 ```diff
 pipeline:

docs/docs/20-usage/71-project-settings.md

@@ -6,7 +6,7 @@ As the owner of a project in Woodpecker you can change project related settings
 
 ## Pipeline path
 
-The path to the pipeline config file or folder. By default it is left empty which will use the following configuration resolution `.woodpecker/*.yml` -> `.woodpecker.yml` -> `.drone.yml`. If you set a custom path Woodpecker tries to load your configuration or fails if no configuration could be found at the specified location. To use a [multi pipeline](./25-multi-pipeline.md) you have to change it to a folder path ending with a `/` like `.woodpecker/`.
+The path to the pipeline config file or folder. By default it is left empty which will use the following configuration resolution `.woodpecker/*.yml` and `.woodpecker/*.yaml` (without any preference in handling them) -> `.woodpecker.yml` -> `.woodpecker.yaml` -> `.drone.yml`. If you set a custom path Woodpecker tries to load your configuration or fails if no configuration could be found at the specified location. To use a [multi pipeline](./25-multi-pipeline.md) you have to change it to a folder path ending with a `/` like `.woodpecker/`.
 
 ## Repository hooks
 

docs/docs/91-migrations.md

@@ -13,6 +13,7 @@ Some versions need some changes to the server configuration or the pipeline conf
 - Updated Prometheus gauge `build_*` to `pipeline_*`
 - Updated Prometheus gauge `*_job_*` to `*_step_*`
 - Renamed config env `WOODPECKER_MAX_PROCS` to `WOODPECKER_MAX_WORKFLOWS` (still available as fallback)
+- The pipelines are now also read from `.yaml` files, the new default order is `.woodpecker/*.yml` and `.woodpecker/*.yaml` (without any prioritization) -> `.woodpecker.yml` ->  `.woodpecker.yaml` -> `.drone.yml`
 
 ## 0.15.0
 

server/shared/configFetcher.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/woodpecker-ci/woodpecker/server/model"
 	"github.com/woodpecker-ci/woodpecker/server/remote"
+	"github.com/woodpecker-ci/woodpecker/shared/constant"
 )
 
 type ConfigFetcher interface {
@@ -91,73 +92,36 @@ func (cf *configFetcher) Fetch(ctx context.Context) (files []*remote.FileMeta, e
 }
 
 // fetch config by timeout
-// TODO: deduplicate code
 func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config string) ([]*remote.FileMeta, error) {
 	ctx, cancel := context.WithTimeout(c, timeout)
 	defer cancel()
 
 	if len(config) > 0 {
 		log.Trace().Msgf("ConfigFetch[%s]: use user config '%s'", cf.repo.FullName, config)
-		// either a file
-		if !strings.HasSuffix(config, "/") {
-			file, err := cf.remote.File(ctx, cf.user, cf.repo, cf.pipeline, config)
-			if err == nil && len(file) != 0 {
-				log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config)
-				return []*remote.FileMeta{{
-					Name: config,
-					Data: file,
-				}}, nil
-			}
+
+		// could be adapted to allow the user to supply a list like we do in the defaults
+		configs := []string{config}
+
+		fileMeta, err := cf.getFirstAvailableConfig(ctx, configs, true)
+		if err == nil {
+			return fileMeta, err
 		}
 
-		// or a folder
-		files, err := cf.remote.Dir(ctx, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(config, "/"))
-		if err == nil && len(files) != 0 {
-			log.Trace().Msgf("ConfigFetch[%s]: found %d files in '%s'", cf.repo.FullName, len(files), config)
-			return filterPipelineFiles(files), nil
-		}
-
-		return nil, fmt.Errorf("config '%s' not found: %s", config, err)
+		return nil, fmt.Errorf("user defined config '%s' not found: %s", config, err)
 	}
 
-	log.Trace().Msgf("ConfigFetch[%s]: user did not defined own config follow default procedure", cf.repo.FullName)
-	// no user defined config so try .woodpecker/*.yml -> .woodpecker.yml -> .drone.yml
-
-	// test .woodpecker/ folder
-	// if folder is not supported we will get a "Not implemented" error and continue
-	config = ".woodpecker"
-	files, err := cf.remote.Dir(ctx, cf.user, cf.repo, cf.pipeline, config)
-	files = filterPipelineFiles(files)
-	if err == nil && len(files) != 0 {
-		log.Trace().Msgf("ConfigFetch[%s]: found %d files in '%s'", cf.repo.FullName, len(files), config)
-		return files, nil
-	}
-
-	config = ".woodpecker.yml"
-	file, err := cf.remote.File(ctx, cf.user, cf.repo, cf.pipeline, config)
-	if err == nil && len(file) != 0 {
-		log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config)
-		return []*remote.FileMeta{{
-			Name: config,
-			Data: file,
-		}}, nil
-	}
-
-	config = ".drone.yml"
-	file, err = cf.remote.File(ctx, cf.user, cf.repo, cf.pipeline, config)
-	if err == nil && len(file) != 0 {
-		log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config)
-		return []*remote.FileMeta{{
-			Name: config,
-			Data: file,
-		}}, nil
+	log.Trace().Msgf("ConfigFetch[%s]: user did not defined own config, following default procedure", cf.repo.FullName)
+	// for the order see shared/constants/constants.go
+	fileMeta, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:], false)
+	if err == nil {
+		return fileMeta, err
 	}
 
 	select {
 	case <-ctx.Done():
 		return nil, ctx.Err()
 	default:
-		return []*remote.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not found config: %s", err)
+		return []*remote.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not find config: %s", err)
 	}
 }
 
@@ -165,10 +129,54 @@ func filterPipelineFiles(files []*remote.FileMeta) []*remote.FileMeta {
 	var res []*remote.FileMeta
 
 	for _, file := range files {
-		if strings.HasSuffix(file.Name, ".yml") {
+		if strings.HasSuffix(file.Name, ".yml") || strings.HasSuffix(file.Name, ".yaml") {
 			res = append(res, file)
 		}
 	}
 
 	return res
 }
+
+func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fileMeta []*remote.FileMeta, found bool) {
+	file, err := cf.remote.File(c, cf.user, cf.repo, cf.pipeline, config)
+
+	if err == nil && len(file) != 0 {
+		log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config)
+
+		return []*remote.FileMeta{{
+			Name: config,
+			Data: file,
+		}}, true
+	}
+
+	return nil, false
+}
+
+func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string, userDefined bool) ([]*remote.FileMeta, error) {
+	userDefinedLog := ""
+	if userDefined {
+		userDefinedLog = "user defined"
+	}
+
+	for _, fileOrFolder := range configs {
+		if strings.HasSuffix(fileOrFolder, "/") {
+			// config is a folder
+			// if folder is not supported we will get a "Not implemented" error and continue
+			files, err := cf.remote.Dir(c, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(fileOrFolder, "/"))
+			files = filterPipelineFiles(files)
+			if err == nil && len(files) != 0 {
+				log.Trace().Msgf("ConfigFetch[%s]: found %d %s files in '%s'", cf.repo.FullName, len(files), userDefinedLog, fileOrFolder)
+				return files, nil
+			}
+		}
+
+		// config is a file
+		if fileMeta, found := cf.checkPipelineFile(c, fileOrFolder); found {
+			log.Trace().Msgf("ConfigFetch[%s]: found %s file: '%s'", cf.repo.FullName, userDefinedLog, fileOrFolder)
+			return fileMeta, nil
+		}
+	}
+
+	// nothing found
+	return nil, fmt.Errorf("%s configs not found searched: %s", userDefinedLog, strings.Join(configs, ", "))
+}

server/shared/configFetcher_test.go

@@ -74,6 +74,61 @@ func TestFetch(t *testing.T) {
 			},
 			expectedError: false,
 		},
+		{
+			name:       "Default config with .yaml - .woodpecker/",
+			repoConfig: "",
+			files: []file{{
+				name: ".woodpecker/text.txt",
+				data: dummyData,
+			}, {
+				name: ".woodpecker/release.yaml",
+				data: dummyData,
+			}, {
+				name: ".woodpecker/image.png",
+				data: dummyData,
+			}},
+			expectedFileNames: []string{
+				".woodpecker/release.yaml",
+			},
+			expectedError: false,
+		},
+		{
+			name:       "Default config with .yaml, .yml mix - .woodpecker/",
+			repoConfig: "",
+			files: []file{{
+				name: ".woodpecker/text.txt",
+				data: dummyData,
+			}, {
+				name: ".woodpecker/release.yaml",
+				data: dummyData,
+			}, {
+				name: ".woodpecker/other.yml",
+				data: dummyData,
+			}, {
+				name: ".woodpecker/image.png",
+				data: dummyData,
+			}},
+			expectedFileNames: []string{
+				".woodpecker/release.yaml",
+				".woodpecker/other.yml",
+			},
+			expectedError: false,
+		},
+		{
+			name:       "Default config check .woodpecker.yml before .woodpecker.yaml",
+			repoConfig: "",
+			files: []file{{
+				name: ".woodpecker.yaml",
+				data: dummyData,
+			}, {
+				name: ".woodpecker.yml",
+				data: dummyData,
+			}},
+			expectedFileNames: []string{
+				".woodpecker.yml",
+			},
+			expectedError: false,
+		},
 		{
 			name:       "Override via API with custom config",
 			repoConfig: "",

server/shared/stepBuilder.go

@@ -400,6 +400,7 @@ func metadataPipelineFromModelPipeline(pipeline *model.Pipeline, includeParent b
 func SanitizePath(path string) string {
 	path = filepath.Base(path)
 	path = strings.TrimSuffix(path, ".yml")
+	path = strings.TrimSuffix(path, ".yaml")
 	path = strings.TrimPrefix(path, ".")
 	return path
 }

server/shared/stepBuilder_test.go

@@ -568,6 +568,18 @@ func TestSanitizePath(t *testing.T) {
 			path:          "folder/sub-folder/test.yml",
 			sanitizedPath: "test",
 		},
+		{
+			path:          ".woodpecker/test.yaml",
+			sanitizedPath: "test",
+		},
+		{
+			path:          ".woodpecker.yaml",
+			sanitizedPath: "woodpecker",
+		},
+		{
+			path:          "folder/sub-folder/test.yaml",
+			sanitizedPath: "test",
+		},
 	}
 
 	for _, test := range testTable {

shared/constant/constant.go

@@ -22,6 +22,15 @@ var PrivilegedPlugins = []string{
 	"woodpeckerci/plugin-docker-buildx",
 }
 
+// DefaultConfigOrder represent the priority in witch woodpecker serarch for a pipeline config by default
+// folders are indicated by supplying a trailing /
+var DefaultConfigOrder = [...]string{
+	".woodpecker/",
+	".woodpecker.yml",
+	".woodpecker.yaml",
+	".drone.yml",
+}
+
 const (
 	DefaultCloneImage = "docker.io/woodpeckerci/plugin-git:v1.6.0"
 )