woodpecker/pipeline/backend/docker/convert.go
Anbraten 556607b525
Rework log streaming and related functions (#1802)
closes #1801
closes #1815 
closes #1144
closes  #983
closes  #557
closes #1827
regression of #1791

# TODO
- [x] adjust log model
- [x] add migration for logs
- [x] send log line via grpc using step-id
- [x] save log-line to db
- [x] stream log-lines to UI
- [x] use less structs for log-data
- [x] make web UI work
  - [x] display logs loaded from db
  - [x] display streaming logs
- [ ] ~~make migration work~~ -> dedicated pull (#1828)

# TESTED
- [x] new logs are stored in database
- [x] log retrieval via cli (of new logs) works
- [x] log streaming works (tested via curl & webui)
- [x] log retrieval via web (of new logs) works

---------

Co-authored-by: 6543 <6543@obermui.de>
2023-06-06 09:52:08 +02:00

194 lines
4.8 KiB
Go

// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (
"encoding/base64"
"encoding/json"
"regexp"
"strings"
"github.com/docker/docker/api/types/container"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
)
// returns a container configuration.
func toConfig(step *types.Step) *container.Config {
config := &container.Config{
Image: step.Image,
Labels: map[string]string{"wp_uuid": step.UUID},
WorkingDir: step.WorkingDir,
AttachStdout: true,
AttachStderr: true,
}
if len(step.Commands) != 0 {
env, entry, cmd := common.GenerateContainerConf(step.Commands)
for k, v := range env {
step.Environment[k] = v
}
config.Entrypoint = entry
config.Cmd = cmd
}
if len(step.Environment) != 0 {
config.Env = toEnv(step.Environment)
}
if len(step.Volumes) != 0 {
config.Volumes = toVol(step.Volumes)
}
return config
}
// returns a container host configuration.
func toHostConfig(step *types.Step) *container.HostConfig {
config := &container.HostConfig{
Resources: container.Resources{
CPUQuota: step.CPUQuota,
CPUShares: step.CPUShares,
CpusetCpus: step.CPUSet,
Memory: step.MemLimit,
MemorySwap: step.MemSwapLimit,
},
LogConfig: container.LogConfig{
Type: "json-file",
},
Privileged: step.Privileged,
ShmSize: step.ShmSize,
Sysctls: step.Sysctls,
}
// if len(step.VolumesFrom) != 0 {
// config.VolumesFrom = step.VolumesFrom
// }
if len(step.NetworkMode) != 0 {
config.NetworkMode = container.NetworkMode(step.NetworkMode)
}
if len(step.IpcMode) != 0 {
config.IpcMode = container.IpcMode(step.IpcMode)
}
if len(step.DNS) != 0 {
config.DNS = step.DNS
}
if len(step.DNSSearch) != 0 {
config.DNSSearch = step.DNSSearch
}
if len(step.ExtraHosts) != 0 {
config.ExtraHosts = step.ExtraHosts
}
if len(step.Devices) != 0 {
config.Devices = toDev(step.Devices)
}
if len(step.Volumes) != 0 {
config.Binds = step.Volumes
}
config.Tmpfs = map[string]string{}
for _, path := range step.Tmpfs {
if !strings.Contains(path, ":") {
config.Tmpfs[path] = ""
continue
}
parts, err := splitVolumeParts(path)
if err != nil {
continue
}
config.Tmpfs[parts[0]] = parts[1]
}
return config
}
// helper function that converts a slice of volume paths to a set of
// unique volume names.
func toVol(paths []string) map[string]struct{} {
set := map[string]struct{}{}
for _, path := range paths {
parts, err := splitVolumeParts(path)
if err != nil {
continue
}
if len(parts) < 2 {
continue
}
set[parts[1]] = struct{}{}
}
return set
}
// helper function that converts a key value map of environment variables to a
// string slice in key=value format.
func toEnv(env map[string]string) []string {
var envs []string
for k, v := range env {
envs = append(envs, k+"="+v)
}
return envs
}
// helper function that converts a slice of device paths to a slice of
// container.DeviceMapping.
func toDev(paths []string) []container.DeviceMapping {
var devices []container.DeviceMapping
for _, path := range paths {
parts, err := splitVolumeParts(path)
if err != nil {
continue
}
if len(parts) < 2 {
continue
}
if strings.HasSuffix(parts[1], ":ro") || strings.HasSuffix(parts[1], ":rw") {
parts[1] = parts[1][:len(parts[1])-1]
}
devices = append(devices, container.DeviceMapping{
PathOnHost: parts[0],
PathInContainer: parts[1],
CgroupPermissions: "rwm",
})
}
return devices
}
// helper function that serializes the auth configuration as JSON
// base64 payload.
func encodeAuthToBase64(authConfig types.Auth) (string, error) {
buf, err := json.Marshal(authConfig)
if err != nil {
return "", err
}
return base64.URLEncoding.EncodeToString(buf), nil
}
// helper function that split volume path
func splitVolumeParts(volumeParts string) ([]string, error) {
pattern := `^((?:[\w]\:)?[^\:]*)\:((?:[\w]\:)?[^\:]*)(?:\:([rwom]*))?`
r, err := regexp.Compile(pattern)
if err != nil {
return []string{}, err
}
if r.MatchString(volumeParts) {
results := r.FindStringSubmatch(volumeParts)[1:]
var cleanResults []string
for _, item := range results {
if item != "" {
cleanResults = append(cleanResults, item)
}
}
return cleanResults, nil
}
return strings.Split(volumeParts, ":"), nil
}