woodpecker/cmd/drone-agent/agent.go

493 lines
12 KiB
Go
Raw Normal View History

2018-02-19 22:24:10 +00:00
// Copyright 2018 Drone.IO Inc.
2018-03-21 13:02:17 +00:00
//
2018-02-19 22:24:10 +00:00
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
2018-03-21 13:02:17 +00:00
//
2018-02-19 22:24:10 +00:00
// http://www.apache.org/licenses/LICENSE-2.0
2018-03-21 13:02:17 +00:00
//
2018-02-19 22:24:10 +00:00
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2017-06-29 22:51:22 +00:00
package main
2016-04-20 01:37:53 +00:00
import (
2017-03-16 10:14:02 +00:00
"context"
"crypto/tls"
2017-04-01 11:17:04 +00:00
"encoding/json"
grpccredentials "google.golang.org/grpc/credentials"
2017-03-16 10:14:02 +00:00
"io"
2017-04-01 11:17:04 +00:00
"io/ioutil"
2017-09-12 18:25:55 +00:00
"net/http"
"os"
2017-04-01 11:17:04 +00:00
"strconv"
2016-04-20 01:37:53 +00:00
"sync"
"time"
2017-06-28 17:21:22 +00:00
"google.golang.org/grpc"
2018-01-08 15:28:38 +00:00
"google.golang.org/grpc/keepalive"
"google.golang.org/grpc/metadata"
2017-06-28 17:21:22 +00:00
"github.com/woodpecker-ci/woodpecker/cncd/pipeline/pipeline"
"github.com/woodpecker-ci/woodpecker/cncd/pipeline/pipeline/backend"
"github.com/woodpecker-ci/woodpecker/cncd/pipeline/pipeline/backend/docker"
"github.com/woodpecker-ci/woodpecker/cncd/pipeline/pipeline/multipart"
"github.com/woodpecker-ci/woodpecker/cncd/pipeline/pipeline/rpc"
2016-09-26 08:29:05 +00:00
2017-09-15 01:42:00 +00:00
"github.com/drone/signal"
2017-08-03 19:36:22 +00:00
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
2017-03-16 10:14:02 +00:00
"github.com/tevino/abool"
"github.com/urfave/cli"
2017-06-29 23:35:38 +00:00
oldcontext "golang.org/x/net/context"
2016-04-20 01:37:53 +00:00
)
2017-03-16 10:14:02 +00:00
func loop(c *cli.Context) error {
filter := rpc.Filter{
Labels: map[string]string{
"platform": c.String("platform"),
},
Expr: c.String("filter"),
2017-03-16 10:14:02 +00:00
}
2016-04-20 01:37:53 +00:00
hostname := c.String("hostname")
if len(hostname) == 0 {
hostname, _ = os.Hostname()
}
2017-08-03 19:36:22 +00:00
if c.BoolT("debug") {
zerolog.SetGlobalLevel(zerolog.DebugLevel)
} else {
zerolog.SetGlobalLevel(zerolog.WarnLevel)
}
2017-09-12 18:25:55 +00:00
if c.Bool("pretty") {
log.Logger = log.Output(
zerolog.ConsoleWriter{
Out: os.Stderr,
NoColor: c.BoolT("nocolor"),
},
)
}
2017-09-12 20:40:24 +00:00
counter.Polling = c.Int("max-procs")
counter.Running = 0
2017-09-12 18:25:55 +00:00
if c.BoolT("healthcheck") {
go http.ListenAndServe(":3000", nil)
}
2017-06-28 17:21:22 +00:00
// TODO pass version information to grpc server
// TODO authenticate to grpc server
2017-06-29 23:35:38 +00:00
// grpc.Dial(target, ))
var transport = grpc.WithInsecure()
if c.Bool("secure-grpc") {
transport = grpc.WithTransportCredentials(grpccredentials.NewTLS(&tls.Config{InsecureSkipVerify: c.Bool("skip-insecure-grpc")}))
}
2017-06-28 17:21:22 +00:00
conn, err := grpc.Dial(
c.String("server"),
transport,
2017-06-29 23:35:38 +00:00
grpc.WithPerRPCCredentials(&credentials{
username: c.String("username"),
password: c.String("password"),
}),
grpc.WithKeepaliveParams(keepalive.ClientParameters{
2018-03-21 13:02:17 +00:00
Time: c.Duration("keepalive-time"),
Timeout: c.Duration("keepalive-timeout"),
}),
2017-03-16 10:14:02 +00:00
)
2017-03-16 10:14:02 +00:00
if err != nil {
return err
2017-03-05 07:56:08 +00:00
}
2017-06-28 17:21:22 +00:00
defer conn.Close()
client := rpc.NewGrpcClient(conn)
2017-03-05 07:56:08 +00:00
2017-03-16 10:14:02 +00:00
sigterm := abool.New()
ctx := metadata.NewOutgoingContext(
context.Background(),
metadata.Pairs("hostname", hostname),
)
2017-09-15 01:42:00 +00:00
ctx = signal.WithContextFunc(ctx, func() {
2017-03-16 10:14:02 +00:00
println("ctrl+c received, terminating process")
sigterm.Set()
})
2016-09-29 21:45:13 +00:00
2017-03-16 10:14:02 +00:00
var wg sync.WaitGroup
parallel := c.Int("max-procs")
wg.Add(parallel)
2016-09-29 21:45:13 +00:00
2017-03-16 10:14:02 +00:00
for i := 0; i < parallel; i++ {
go func() {
defer wg.Done()
for {
if sigterm.IsSet() {
return
}
2017-09-14 16:39:52 +00:00
r := runner{
client: client,
filter: filter,
hostname: hostname,
}
if err := r.run(ctx); err != nil {
2017-08-03 19:36:22 +00:00
log.Error().Err(err).Msg("pipeline done with error")
2017-03-16 10:14:02 +00:00
return
}
}
}()
2016-04-20 01:37:53 +00:00
}
2017-03-16 10:14:02 +00:00
wg.Wait()
return nil
}
2017-09-12 16:03:32 +00:00
// NOTE we need to limit the size of the logs and files that we upload.
// The maximum grpc payload size is 4194304. So until we implement streaming
// for uploads, we need to set these limits below the maximum.
2017-03-16 10:14:02 +00:00
const (
2017-09-12 16:03:32 +00:00
maxLogsUpload = 2000000 // this is per step
maxFileUpload = 1000000
2017-03-16 10:14:02 +00:00
)
2016-04-20 01:37:53 +00:00
2017-09-14 16:39:52 +00:00
type runner struct {
client rpc.Peer
filter rpc.Filter
hostname string
}
func (r *runner) run(ctx context.Context) error {
2017-08-03 19:36:22 +00:00
log.Debug().
Msg("request next execution")
2016-04-20 01:37:53 +00:00
2017-07-20 16:21:15 +00:00
meta, _ := metadata.FromOutgoingContext(ctx)
ctxmeta := metadata.NewOutgoingContext(context.Background(), meta)
2017-03-16 10:14:02 +00:00
// get the next job from the queue
2017-09-14 16:39:52 +00:00
work, err := r.client.Next(ctx, r.filter)
2017-03-16 10:14:02 +00:00
if err != nil {
return err
}
if work == nil {
return nil
2016-04-20 01:37:53 +00:00
}
2017-08-03 19:36:22 +00:00
2017-09-12 20:40:24 +00:00
timeout := time.Hour
if minutes := work.Timeout; minutes != 0 {
timeout = time.Duration(minutes) * time.Minute
}
counter.Add(
work.ID,
timeout,
extractRepositoryName(work.Config), // hack
extractBuildNumber(work.Config), // hack
)
defer counter.Done(work.ID)
2017-08-03 19:36:22 +00:00
logger := log.With().
2017-09-12 20:40:24 +00:00
Str("repo", extractRepositoryName(work.Config)). // hack
Str("build", extractBuildNumber(work.Config)). // hack
2017-08-03 19:36:22 +00:00
Str("id", work.ID).
Logger()
logger.Debug().
Msg("received execution")
2017-03-16 10:14:02 +00:00
// new docker engine
engine, err := docker.NewEnv()
2016-04-20 01:37:53 +00:00
if err != nil {
2017-08-03 19:36:22 +00:00
logger.Error().
Err(err).
Msg("cannot create docker client")
2017-03-16 10:14:02 +00:00
return err
2016-04-20 01:37:53 +00:00
}
2017-07-20 16:21:15 +00:00
ctx, cancel := context.WithTimeout(ctxmeta, timeout)
2017-03-16 10:14:02 +00:00
defer cancel()
2016-09-26 08:29:05 +00:00
2017-03-16 10:14:02 +00:00
cancelled := abool.New()
go func() {
2017-08-03 19:36:22 +00:00
logger.Debug().
Msg("listen for cancel signal")
2017-09-14 16:39:52 +00:00
if werr := r.client.Wait(ctx, work.ID); werr != nil {
2017-03-16 10:14:02 +00:00
cancelled.SetTo(true)
2017-08-03 19:36:22 +00:00
logger.Warn().
Err(werr).
Msg("cancel signal received")
2017-03-16 10:14:02 +00:00
cancel()
} else {
2017-08-03 19:36:22 +00:00
logger.Debug().
Msg("stop listening for cancel signal")
}
2017-03-16 10:14:02 +00:00
}()
2017-03-16 10:14:02 +00:00
go func() {
for {
select {
case <-ctx.Done():
2017-08-03 19:36:22 +00:00
logger.Debug().
Msg("pipeline done")
2017-03-16 10:14:02 +00:00
return
case <-time.After(time.Minute):
2017-08-03 19:36:22 +00:00
logger.Debug().
Msg("pipeline lease renewed")
2017-09-14 16:39:52 +00:00
r.client.Extend(ctx, work.ID)
2017-03-16 10:14:02 +00:00
}
}
}()
state := rpc.State{}
state.Started = time.Now().Unix()
2017-08-03 19:36:22 +00:00
2017-09-14 16:39:52 +00:00
err = r.client.Init(ctxmeta, work.ID, state)
2017-03-16 10:14:02 +00:00
if err != nil {
2017-08-03 19:36:22 +00:00
logger.Error().
Err(err).
Msg("pipeline initialization failed")
2016-09-26 08:29:05 +00:00
}
2017-03-16 10:14:02 +00:00
var uploads sync.WaitGroup
defaultLogger := pipeline.LogFunc(func(proc *backend.Step, rc multipart.Reader) error {
2017-08-03 19:36:22 +00:00
loglogger := logger.With().
Str("image", proc.Image).
Str("stage", proc.Alias).
Logger()
2017-03-16 10:14:02 +00:00
part, rerr := rc.NextPart()
if rerr != nil {
return rerr
}
uploads.Add(1)
2017-04-01 11:17:04 +00:00
var secrets []string
for _, secret := range work.Config.Secrets {
if secret.Mask {
secrets = append(secrets, secret.Value)
}
}
2017-08-03 19:36:22 +00:00
loglogger.Debug().Msg("log stream opened")
2017-04-01 11:17:04 +00:00
limitedPart := io.LimitReader(part, maxLogsUpload)
2017-09-14 16:39:52 +00:00
logstream := rpc.NewLineWriter(r.client, work.ID, proc.Alias, secrets...)
2017-04-01 11:17:04 +00:00
io.Copy(logstream, limitedPart)
2017-08-03 19:36:22 +00:00
loglogger.Debug().Msg("log stream copied")
2017-04-01 11:17:04 +00:00
file := &rpc.File{}
file.Mime = "application/json+logs"
file.Proc = proc.Alias
file.Name = "logs.json"
file.Data, _ = json.Marshal(logstream.Lines())
file.Size = len(file.Data)
file.Time = time.Now().Unix()
2017-08-03 19:36:22 +00:00
loglogger.Debug().
Msg("log stream uploading")
2017-09-14 16:39:52 +00:00
if serr := r.client.Upload(ctxmeta, work.ID, file); serr != nil {
2017-08-03 19:36:22 +00:00
loglogger.Error().
Err(serr).
Msg("log stream upload error")
2017-04-01 11:17:04 +00:00
}
2016-09-26 08:29:05 +00:00
2017-08-03 19:36:22 +00:00
loglogger.Debug().
Msg("log stream upload complete")
2017-03-16 10:14:02 +00:00
defer func() {
2017-08-03 19:36:22 +00:00
loglogger.Debug().
Msg("log stream closed")
2017-03-16 10:14:02 +00:00
uploads.Done()
}()
2016-09-28 00:33:13 +00:00
2017-03-16 10:14:02 +00:00
part, rerr = rc.NextPart()
if rerr != nil {
return nil
2016-09-28 00:33:13 +00:00
}
2017-04-01 11:17:04 +00:00
// TODO should be configurable
limitedPart = io.LimitReader(part, maxFileUpload)
file = &rpc.File{}
file.Mime = part.Header().Get("Content-Type")
file.Proc = proc.Alias
file.Name = part.FileName()
file.Data, _ = ioutil.ReadAll(limitedPart)
file.Size = len(file.Data)
file.Time = time.Now().Unix()
2017-08-02 20:04:00 +00:00
file.Meta = map[string]string{}
for key, value := range part.Header() {
file.Meta[key] = value[0]
}
2017-04-01 11:17:04 +00:00
2017-08-03 19:36:22 +00:00
loglogger.Debug().
Str("file", file.Name).
Str("mime", file.Mime).
Msg("file stream uploading")
2017-09-14 16:39:52 +00:00
if serr := r.client.Upload(ctxmeta, work.ID, file); serr != nil {
2017-08-03 19:36:22 +00:00
loglogger.Error().
Err(serr).
Str("file", file.Name).
Str("mime", file.Mime).
Msg("file stream upload error")
2017-04-01 11:17:04 +00:00
}
2017-08-03 19:36:22 +00:00
loglogger.Debug().
Str("file", file.Name).
Str("mime", file.Mime).
Msg("file stream upload complete")
2017-04-01 11:17:04 +00:00
return nil
})
defaultTracer := pipeline.TraceFunc(func(state *pipeline.State) error {
2017-08-03 19:36:22 +00:00
proclogger := logger.With().
Str("image", state.Pipeline.Step.Image).
Str("stage", state.Pipeline.Step.Alias).
Int("exit_code", state.Process.ExitCode).
Bool("exited", state.Process.Exited).
Logger()
2017-04-01 11:17:04 +00:00
procState := rpc.State{
Proc: state.Pipeline.Step.Alias,
Exited: state.Process.Exited,
ExitCode: state.Process.ExitCode,
Started: time.Now().Unix(), // TODO do not do this
Finished: time.Now().Unix(),
}
defer func() {
2017-08-03 19:36:22 +00:00
proclogger.Debug().
Msg("update step status")
2017-09-14 16:39:52 +00:00
if uerr := r.client.Update(ctxmeta, work.ID, procState); uerr != nil {
2017-08-03 19:36:22 +00:00
proclogger.Debug().
Err(uerr).
Msg("update step status error")
2017-04-01 11:17:04 +00:00
}
2017-08-03 19:36:22 +00:00
proclogger.Debug().
Msg("update step status complete")
2017-04-01 11:17:04 +00:00
}()
if state.Process.Exited {
return nil
}
if state.Pipeline.Step.Environment == nil {
state.Pipeline.Step.Environment = map[string]string{}
}
2017-09-14 16:39:52 +00:00
state.Pipeline.Step.Environment["DRONE_MACHINE"] = r.hostname
2017-04-01 11:17:04 +00:00
state.Pipeline.Step.Environment["CI_BUILD_STATUS"] = "success"
state.Pipeline.Step.Environment["CI_BUILD_STARTED"] = strconv.FormatInt(state.Pipeline.Time, 10)
state.Pipeline.Step.Environment["CI_BUILD_FINISHED"] = strconv.FormatInt(time.Now().Unix(), 10)
2017-05-05 16:19:20 +00:00
state.Pipeline.Step.Environment["DRONE_BUILD_STATUS"] = "success"
state.Pipeline.Step.Environment["DRONE_BUILD_STARTED"] = strconv.FormatInt(state.Pipeline.Time, 10)
state.Pipeline.Step.Environment["DRONE_BUILD_FINISHED"] = strconv.FormatInt(time.Now().Unix(), 10)
2017-04-01 11:17:04 +00:00
state.Pipeline.Step.Environment["CI_JOB_STATUS"] = "success"
state.Pipeline.Step.Environment["CI_JOB_STARTED"] = strconv.FormatInt(state.Pipeline.Time, 10)
state.Pipeline.Step.Environment["CI_JOB_FINISHED"] = strconv.FormatInt(time.Now().Unix(), 10)
2017-05-05 16:19:20 +00:00
state.Pipeline.Step.Environment["DRONE_JOB_STATUS"] = "success"
state.Pipeline.Step.Environment["DRONE_JOB_STARTED"] = strconv.FormatInt(state.Pipeline.Time, 10)
state.Pipeline.Step.Environment["DRONE_JOB_FINISHED"] = strconv.FormatInt(time.Now().Unix(), 10)
2017-04-01 11:17:04 +00:00
if state.Pipeline.Error != nil {
state.Pipeline.Step.Environment["CI_BUILD_STATUS"] = "failure"
state.Pipeline.Step.Environment["CI_JOB_STATUS"] = "failure"
2017-05-05 16:19:20 +00:00
state.Pipeline.Step.Environment["DRONE_BUILD_STATUS"] = "failure"
state.Pipeline.Step.Environment["DRONE_JOB_STATUS"] = "failure"
2016-09-28 00:33:13 +00:00
}
2017-03-16 10:14:02 +00:00
return nil
})
2016-09-28 00:33:13 +00:00
2017-03-16 10:14:02 +00:00
err = pipeline.New(work.Config,
pipeline.WithContext(ctx),
pipeline.WithLogger(defaultLogger),
2017-04-01 11:17:04 +00:00
pipeline.WithTracer(defaultTracer),
2017-03-16 10:14:02 +00:00
pipeline.WithEngine(engine),
).Run()
2016-09-28 00:33:13 +00:00
2017-03-16 10:14:02 +00:00
state.Finished = time.Now().Unix()
state.Exited = true
if err != nil {
2017-03-17 08:57:32 +00:00
switch xerr := err.(type) {
case *pipeline.ExitError:
2017-03-16 10:14:02 +00:00
state.ExitCode = xerr.Code
2017-03-17 08:57:32 +00:00
default:
state.ExitCode = 1
state.Error = err.Error()
2017-03-16 10:14:02 +00:00
}
if cancelled.IsSet() {
state.ExitCode = 137
}
2016-09-28 00:33:13 +00:00
}
2017-08-03 19:36:22 +00:00
logger.Debug().
Str("error", state.Error).
Int("exit_code", state.ExitCode).
Msg("pipeline complete")
logger.Debug().
Msg("uploading logs")
2017-03-16 10:14:02 +00:00
uploads.Wait()
2017-04-01 11:17:04 +00:00
2017-08-03 19:36:22 +00:00
logger.Debug().
Msg("uploading logs complete")
logger.Debug().
Str("error", state.Error).
Int("exit_code", state.ExitCode).
Msg("updating pipeline status")
2017-06-29 21:30:08 +00:00
2017-09-14 16:39:52 +00:00
err = r.client.Done(ctxmeta, work.ID, state)
2017-03-16 10:14:02 +00:00
if err != nil {
2017-08-03 19:36:22 +00:00
logger.Error().Err(err).
Msg("updating pipeline status failed")
2017-06-29 21:30:08 +00:00
} else {
2017-08-03 19:36:22 +00:00
logger.Debug().
Msg("updating pipeline status complete")
2017-03-16 10:14:02 +00:00
}
2017-03-16 10:14:02 +00:00
return nil
}
2017-06-29 23:35:38 +00:00
type credentials struct {
username string
password string
}
func (c *credentials) GetRequestMetadata(oldcontext.Context, ...string) (map[string]string, error) {
return map[string]string{
"username": c.username,
"password": c.password,
}, nil
}
func (c *credentials) RequireTransportSecurity() bool {
return false
}
2017-08-03 19:36:22 +00:00
// extract repository name from the configuration
func extractRepositoryName(config *backend.Config) string {
return config.Stages[0].Steps[0].Environment["DRONE_REPO"]
2017-08-03 19:36:22 +00:00
}
// extract build number from the configuration
func extractBuildNumber(config *backend.Config) string {
return config.Stages[0].Steps[0].Environment["DRONE_BUILD_NUMBER"]
}