mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-11-12 12:15:00 +00:00
Code Issues Projects Releases Wiki Activity

Read long log lines from file storage correctly (#4048)

Browse source
This commit is contained in:
hg 2024-08-26 01:53:04 +05:00 committed by GitHub
parent 3fc138d5d2
commit 37d1ca8bc1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 18 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
agent/logger.go
View file

@ -26,12 +26,6 @@ import (
"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc" "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
) )
const (
// Store not more than 1mb in a log-line as 4mb is the limit of a grpc message
// and log-lines needs to be parsed by the browsers later on.
maxLogLineLength = 1024 * 1024 // 1mb
)
func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, workflow *rpc.Workflow) pipeline.Logger { func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, workflow *rpc.Workflow) pipeline.Logger {
return func(step *backend.Step, rc io.ReadCloser) error { return func(step *backend.Step, rc io.ReadCloser) error {
defer rc.Close() defer rc.Close()
@ -50,7 +44,7 @@ func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, w
logger.Debug().Msg("log stream opened") logger.Debug().Msg("log stream opened")
logStream := log.NewLineWriter(r.client, step.UUID, secrets...) logStream := log.NewLineWriter(r.client, step.UUID, secrets...)
if err := log.CopyLineByLine(logStream, rc, maxLogLineLength); err != nil { if err := log.CopyLineByLine(logStream, rc, pipeline.MaxLogLineLength); err != nil {
logger.Error().Err(err).Msg("copy limited logStream part") logger.Error().Err(err).Msg("copy limited logStream part")
} }

3
cli/exec/exec.go
View file

@ -281,8 +281,7 @@ func convertPathForWindows(path string) string {
return filepath.ToSlash(path) return filepath.ToSlash(path)
} }
const maxLogLineLength = 1024 * 1024 // 1mb
var defaultLogger = pipeline.Logger(func(step *backend_types.Step, rc io.ReadCloser) error { var defaultLogger = pipeline.Logger(func(step *backend_types.Step, rc io.ReadCloser) error {
logWriter := NewLineWriter(step.Name, step.UUID) logWriter := NewLineWriter(step.Name, step.UUID)
return pipelineLog.CopyLineByLine(logWriter, rc, maxLogLineLength) return pipelineLog.CopyLineByLine(logWriter, rc, pipeline.MaxLogLineLength)
}) })

8
pipeline/const.go
View file

@ -14,4 +14,10 @@
package pipeline package pipeline
const ExitCodeKilled int = 137 const (
ExitCodeKilled int = 137
// Store no more than 1mb in a log-line as 4mb is the limit of a grpc message
// and log-lines needs to be parsed by the browsers later on.
MaxLogLineLength int = 1 * 1024 * 1024 // 1mb
)

9
server/services/log/file/file.go
View file

@ -8,10 +8,16 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"go.woodpecker-ci.org/woodpecker/v2/pipeline"
"go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/model"
"go.woodpecker-ci.org/woodpecker/v2/server/services/log" "go.woodpecker-ci.org/woodpecker/v2/server/services/log"
) )
const (
// base64 overhead + space for other JSON fields (just to be safe)
maxLineLength int = (pipeline.MaxLogLineLength/3)*4 + (64 * 1024)
)
type logStore struct { type logStore struct {
base string base string
} }
@ -43,7 +49,10 @@ func (l logStore) LogFind(step *model.Step) ([]*model.LogEntry, error) {
return nil, err return nil, err
} }
buf := make([]byte, 0, bufio.MaxScanTokenSize)
s := bufio.NewScanner(file) s := bufio.NewScanner(file)
s.Buffer(buf, maxLineLength)
var entries []*model.LogEntry var entries []*model.LogEntry
for s.Scan() { for s.Scan() {
j := s.Text() j := s.Text()