Allow to store logs in files (#3568)

Co-authored-by: Anbraten <6918444+anbraten@users.noreply.github.com>

cmd/server/flags.go

@@ -241,6 +241,17 @@ var flags = append([]cli.Flag{
 		Usage:   "Disable version check in admin web ui.",
 		Name:    "skip-version-check",
 	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_LOG_STORE"},
+		Name:    "log-store",
+		Usage:   "log store to use ('database' or 'file')",
+		Value:   "database",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_LOG_STORE_FILE_PATH"},
+		Name:    "log-store-file-path",
+		Usage:   "directory used for file based log storage",
+	},
 	//
 	// backend options for pipeline compiler
 	//

cmd/server/server.go

@@ -276,6 +276,11 @@ func setupEvilGlobals(c *cli.Context, s store.Store) error {
 	}
 	server.Config.Services.Manager = serviceManager
 
+	server.Config.Services.LogStore, err = setupLogStore(c, s)
+	if err != nil {
+		return fmt.Errorf("could not setup log store: %w", err)
+	}
+
 	// authentication
 	server.Config.Pipeline.AuthenticatePublicRepos = c.Bool("authenticate-public-repos")
 

cmd/server/setup.go

@@ -30,6 +30,8 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/server"
 	"go.woodpecker-ci.org/woodpecker/v2/server/cache"
 	"go.woodpecker-ci.org/woodpecker/v2/server/queue"
+	logService "go.woodpecker-ci.org/woodpecker/v2/server/services/log"
+	"go.woodpecker-ci.org/woodpecker/v2/server/services/log/file"
 	"go.woodpecker-ci.org/woodpecker/v2/server/store"
 	"go.woodpecker-ci.org/woodpecker/v2/server/store/datastore"
 )
@@ -154,3 +156,12 @@ func setupMetrics(g *errgroup.Group, _store store.Store) {
 		}
 	})
 }
+
+func setupLogStore(c *cli.Context, s store.Store) (logService.Service, error) {
+	switch c.String("log-store") {
+	case "file":
+		return file.NewLogStore(c.String("log-store-file-path"))
+	default:
+		return s, nil
+	}
+}

docs/docs/30-administration/10-server-config.md

@@ -543,6 +543,18 @@ Enable the Swagger UI for API documentation.
 
 Disable version check in admin web UI.
 
+### `WOODPECKER_LOG_STORE`
+
+> Default: `database`
+
+Where to store logs. Possible values: `database` or `file`.
+
+### `WOODPECKER_LOG_STORE_FILE_PATH`
+
+> Default empty
+
+Directory to store logs in if [`WOODPECKER_LOG_STORE`](#woodpecker_log_store) is `file`.
+
 ---
 
 ### `WOODPECKER_GITHUB_...`

server/api/pipeline.go

@@ -285,7 +285,7 @@ func GetStepLogs(c *gin.Context) {
 		return
 	}
 
-	logs, err := _store.LogFind(step)
+	logs, err := server.Config.Services.LogStore.LogFind(step)
 	if err != nil {
 		handleDBError(c, err)
 		return
@@ -622,7 +622,7 @@ func DeletePipelineLogs(c *gin.Context) {
 	}
 
 	for _, step := range steps {
-		if lErr := _store.LogDelete(step); err != nil {
+		if lErr := server.Config.Services.LogStore.LogDelete(step); err != nil {
 			err = errors.Join(err, lErr)
 		}
 	}

server/config.go

@@ -24,6 +24,7 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/server/pubsub"
 	"go.woodpecker-ci.org/woodpecker/v2/server/queue"
 	"go.woodpecker-ci.org/woodpecker/v2/server/services"
+	"go.woodpecker-ci.org/woodpecker/v2/server/services/log"
 	"go.woodpecker-ci.org/woodpecker/v2/server/services/permissions"
 )
 
@@ -34,6 +35,7 @@ var Config = struct {
 		Logs       logging.Log
 		Membership cache.MembershipService
 		Manager    services.Manager
+		LogStore   log.Service
 	}
 	Server struct {
 		Key                 string

server/grpc/rpc.go

@@ -338,8 +338,7 @@ func (s *RPC) Log(c context.Context, _logEntry *rpc.LogEntry) error {
 			log.Error().Err(err).Msgf("rpc server could not write to logger")
 		}
 	}()
-	// make line persistent in database
+	return server.Config.Services.LogStore.LogAppend(logEntry)
-	return s.store.LogAppend(logEntry)
 }
 
 func (s *RPC) RegisterAgent(ctx context.Context, platform, backend, version string, capacity int32) (int64, error) {

server/services/log/file/file.go

@@ -0,0 +1,82 @@
+package file
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go.woodpecker-ci.org/woodpecker/v2/server/model"
+	"go.woodpecker-ci.org/woodpecker/v2/server/services/log"
+)
+
+type logStore struct {
+	base string
+}
+
+func NewLogStore(base string) (log.Service, error) {
+	if base == "" {
+		return nil, fmt.Errorf("file storage base path is required")
+	}
+	if _, err := os.Stat(base); err != nil && os.IsNotExist(err) {
+		err = os.MkdirAll(base, 0o600)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return logStore{base: base}, nil
+}
+
+func (l logStore) filePath(id int64) string {
+	return filepath.Join(l.base, fmt.Sprintf("%d.json", id))
+}
+
+func (l logStore) LogFind(step *model.Step) ([]*model.LogEntry, error) {
+	filename := l.filePath(step.ID)
+	file, err := os.Open(filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	s := bufio.NewScanner(file)
+	var entries []*model.LogEntry
+	for s.Scan() {
+		j := s.Text()
+		if len(strings.TrimSpace(j)) == 0 {
+			continue
+		}
+		entry := &model.LogEntry{}
+		err = json.Unmarshal([]byte(j), entry)
+		if err != nil {
+			return nil, err
+		}
+		entries = append(entries, entry)
+	}
+
+	return entries, nil
+}
+
+func (l logStore) LogAppend(logEntry *model.LogEntry) error {
+	file, err := os.OpenFile(l.filePath(logEntry.StepID), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
+	if err != nil {
+		return err
+	}
+	jsonData, err := json.Marshal(logEntry)
+	if err != nil {
+		return err
+	}
+	_, err = file.Write(append(jsonData, byte('\n')))
+	if err != nil {
+		return err
+	}
+	return file.Close()
+}
+
+func (l logStore) LogDelete(step *model.Step) error {
+	return os.Remove(l.filePath(step.ID))
+}

server/services/log/service.go

@@ -0,0 +1,9 @@
+package log
+
+import "go.woodpecker-ci.org/woodpecker/v2/server/model"
+
+type Service interface {
+	LogFind(step *model.Step) ([]*model.LogEntry, error)
+	LogAppend(logEntry *model.LogEntry) error
+	LogDelete(step *model.Step) error
+}