From 37d1ca8bc1d990bac4b265463aedaaa4995a1ab8 Mon Sep 17 00:00:00 2001
From: hg <k@isakov.net>
Date: Mon, 26 Aug 2024 01:53:04 +0500
Subject: [PATCH] Read long log lines from file storage correctly (#4048)

---
 agent/logger.go                  | 8 +-------
 cli/exec/exec.go                 | 3 +--
 pipeline/const.go                | 8 +++++++-
 server/services/log/file/file.go | 9 +++++++++
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/agent/logger.go b/agent/logger.go
index 1bb99a1f1..7e3469fd6 100644
--- a/agent/logger.go
+++ b/agent/logger.go
@@ -26,12 +26,6 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
 )
 
-const (
-	// Store not more than 1mb in a log-line as 4mb is the limit of a grpc message
-	// and log-lines needs to be parsed by the browsers later on.
-	maxLogLineLength = 1024 * 1024 // 1mb
-)
-
 func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, workflow *rpc.Workflow) pipeline.Logger {
 	return func(step *backend.Step, rc io.ReadCloser) error {
 		defer rc.Close()
@@ -50,7 +44,7 @@ func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, w
 		logger.Debug().Msg("log stream opened")
 
 		logStream := log.NewLineWriter(r.client, step.UUID, secrets...)
-		if err := log.CopyLineByLine(logStream, rc, maxLogLineLength); err != nil {
+		if err := log.CopyLineByLine(logStream, rc, pipeline.MaxLogLineLength); err != nil {
 			logger.Error().Err(err).Msg("copy limited logStream part")
 		}
 
diff --git a/cli/exec/exec.go b/cli/exec/exec.go
index df5885ef5..79994e6b9 100644
--- a/cli/exec/exec.go
+++ b/cli/exec/exec.go
@@ -281,8 +281,7 @@ func convertPathForWindows(path string) string {
 	return filepath.ToSlash(path)
 }
 
-const maxLogLineLength = 1024 * 1024 // 1mb
 var defaultLogger = pipeline.Logger(func(step *backend_types.Step, rc io.ReadCloser) error {
 	logWriter := NewLineWriter(step.Name, step.UUID)
-	return pipelineLog.CopyLineByLine(logWriter, rc, maxLogLineLength)
+	return pipelineLog.CopyLineByLine(logWriter, rc, pipeline.MaxLogLineLength)
 })
diff --git a/pipeline/const.go b/pipeline/const.go
index a4bec9789..473e961ab 100644
--- a/pipeline/const.go
+++ b/pipeline/const.go
@@ -14,4 +14,10 @@
 
 package pipeline
 
-const ExitCodeKilled int = 137
+const (
+	ExitCodeKilled int = 137
+
+	// Store no more than 1mb in a log-line as 4mb is the limit of a grpc message
+	// and log-lines needs to be parsed by the browsers later on.
+	MaxLogLineLength int = 1 * 1024 * 1024 // 1mb
+)
diff --git a/server/services/log/file/file.go b/server/services/log/file/file.go
index 276c31798..ce2c6a17f 100644
--- a/server/services/log/file/file.go
+++ b/server/services/log/file/file.go
@@ -8,10 +8,16 @@ import (
 	"path/filepath"
 	"strings"
 
+	"go.woodpecker-ci.org/woodpecker/v2/pipeline"
 	"go.woodpecker-ci.org/woodpecker/v2/server/model"
 	"go.woodpecker-ci.org/woodpecker/v2/server/services/log"
 )
 
+const (
+	// base64 overhead + space for other JSON fields (just to be safe)
+	maxLineLength int = (pipeline.MaxLogLineLength/3)*4 + (64 * 1024)
+)
+
 type logStore struct {
 	base string
 }
@@ -43,7 +49,10 @@ func (l logStore) LogFind(step *model.Step) ([]*model.LogEntry, error) {
 		return nil, err
 	}
 
+	buf := make([]byte, 0, bufio.MaxScanTokenSize)
 	s := bufio.NewScanner(file)
+	s.Buffer(buf, maxLineLength)
+
 	var entries []*model.LogEntry
 	for s.Scan() {
 		j := s.Text()