mirror of
https://github.com/woodpecker-ci/woodpecker.git
synced 2024-11-28 12:51:04 +00:00
Read long log lines from file storage correctly (#4048)
This commit is contained in:
parent
3fc138d5d2
commit
37d1ca8bc1
4 changed files with 18 additions and 10 deletions
|
@ -26,12 +26,6 @@ import (
|
||||||
"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
|
"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
// Store not more than 1mb in a log-line as 4mb is the limit of a grpc message
|
|
||||||
// and log-lines needs to be parsed by the browsers later on.
|
|
||||||
maxLogLineLength = 1024 * 1024 // 1mb
|
|
||||||
)
|
|
||||||
|
|
||||||
func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, workflow *rpc.Workflow) pipeline.Logger {
|
func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, workflow *rpc.Workflow) pipeline.Logger {
|
||||||
return func(step *backend.Step, rc io.ReadCloser) error {
|
return func(step *backend.Step, rc io.ReadCloser) error {
|
||||||
defer rc.Close()
|
defer rc.Close()
|
||||||
|
@ -50,7 +44,7 @@ func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, w
|
||||||
logger.Debug().Msg("log stream opened")
|
logger.Debug().Msg("log stream opened")
|
||||||
|
|
||||||
logStream := log.NewLineWriter(r.client, step.UUID, secrets...)
|
logStream := log.NewLineWriter(r.client, step.UUID, secrets...)
|
||||||
if err := log.CopyLineByLine(logStream, rc, maxLogLineLength); err != nil {
|
if err := log.CopyLineByLine(logStream, rc, pipeline.MaxLogLineLength); err != nil {
|
||||||
logger.Error().Err(err).Msg("copy limited logStream part")
|
logger.Error().Err(err).Msg("copy limited logStream part")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -281,8 +281,7 @@ func convertPathForWindows(path string) string {
|
||||||
return filepath.ToSlash(path)
|
return filepath.ToSlash(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
const maxLogLineLength = 1024 * 1024 // 1mb
|
|
||||||
var defaultLogger = pipeline.Logger(func(step *backend_types.Step, rc io.ReadCloser) error {
|
var defaultLogger = pipeline.Logger(func(step *backend_types.Step, rc io.ReadCloser) error {
|
||||||
logWriter := NewLineWriter(step.Name, step.UUID)
|
logWriter := NewLineWriter(step.Name, step.UUID)
|
||||||
return pipelineLog.CopyLineByLine(logWriter, rc, maxLogLineLength)
|
return pipelineLog.CopyLineByLine(logWriter, rc, pipeline.MaxLogLineLength)
|
||||||
})
|
})
|
||||||
|
|
|
@ -14,4 +14,10 @@
|
||||||
|
|
||||||
package pipeline
|
package pipeline
|
||||||
|
|
||||||
const ExitCodeKilled int = 137
|
const (
|
||||||
|
ExitCodeKilled int = 137
|
||||||
|
|
||||||
|
// Store no more than 1mb in a log-line as 4mb is the limit of a grpc message
|
||||||
|
// and log-lines needs to be parsed by the browsers later on.
|
||||||
|
MaxLogLineLength int = 1 * 1024 * 1024 // 1mb
|
||||||
|
)
|
||||||
|
|
|
@ -8,10 +8,16 @@ import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"go.woodpecker-ci.org/woodpecker/v2/pipeline"
|
||||||
"go.woodpecker-ci.org/woodpecker/v2/server/model"
|
"go.woodpecker-ci.org/woodpecker/v2/server/model"
|
||||||
"go.woodpecker-ci.org/woodpecker/v2/server/services/log"
|
"go.woodpecker-ci.org/woodpecker/v2/server/services/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// base64 overhead + space for other JSON fields (just to be safe)
|
||||||
|
maxLineLength int = (pipeline.MaxLogLineLength/3)*4 + (64 * 1024)
|
||||||
|
)
|
||||||
|
|
||||||
type logStore struct {
|
type logStore struct {
|
||||||
base string
|
base string
|
||||||
}
|
}
|
||||||
|
@ -43,7 +49,10 @@ func (l logStore) LogFind(step *model.Step) ([]*model.LogEntry, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
buf := make([]byte, 0, bufio.MaxScanTokenSize)
|
||||||
s := bufio.NewScanner(file)
|
s := bufio.NewScanner(file)
|
||||||
|
s.Buffer(buf, maxLineLength)
|
||||||
|
|
||||||
var entries []*model.LogEntry
|
var entries []*model.LogEntry
|
||||||
for s.Scan() {
|
for s.Scan() {
|
||||||
j := s.Text()
|
j := s.Text()
|
||||||
|
|
Loading…
Reference in a new issue