mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2025-01-03 06:08:42 +00:00
Code Issues Projects Releases Wiki Activity

Remove unused file system api (#1791)

Browse source 
Co-authored-by: 6543 <6543@obermui.de>
This commit is contained in:
Anbraten 2023-05-31 18:03:03 +02:00 committed by GitHub
parent f5a85d21be
commit c464f857ae
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 291 additions and 1262 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
agent/logger.go
View file

@ -16,10 +16,8 @@ package agent
import (
"context"
"encoding/json"
"io"
"sync"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
@ -30,7 +28,7 @@ import (
"github.com/woodpecker-ci/woodpecker/pipeline/rpc"
)
func (r *Runner) createLogger(ctxmeta context.Context, logger zerolog.Logger, uploads *sync.WaitGroup, work *rpc.Pipeline) pipeline.LogFunc {
func (r *Runner) createLogger(_ context.Context, logger zerolog.Logger, uploads *sync.WaitGroup, work *rpc.Pipeline) pipeline.LogFunc {
return func(step *backend.Step, rc multipart.Reader) error {
loglogger := logger.With().
Str("image", step.Image).
@ -60,73 +58,11 @@ func (r *Runner) createLogger(ctxmeta context.Context, logger zerolog.Logger, up
loglogger.Debug().Msg("log stream copied")
data, err := json.Marshal(logStream.Lines())
if err != nil {
loglogger.Err(err).Msg("could not marshal logstream")
}
file := &rpc.File{
Mime: "application/json+logs",
Step: step.Alias,
Name: "logs.json",
Data: data,
Size: len(data),
Time: time.Now().Unix(),
}
loglogger.Debug().Msg("log stream uploading")
if serr := r.client.Upload(ctxmeta, work.ID, file); serr != nil {
loglogger.Error().Err(serr).Msg("log stream upload error")
} else {
loglogger.Debug().Msg("log stream upload complete")
}
defer func() {
loglogger.Debug().Msg("log stream closed")
uploads.Done()
}()
part, rerr = rc.NextPart()
if rerr != nil {
return nil
}
// TODO should be configurable
limitedPart = io.LimitReader(part, maxFileUpload)
data, err = io.ReadAll(limitedPart)
if err != nil {
loglogger.Err(err).Msg("could not read limited part")
}
file = &rpc.File{
Mime: part.Header().Get("Content-Type"),
Step: step.Alias,
Name: part.FileName(),
Data: data,
Size: len(data),
Time: time.Now().Unix(),
Meta: make(map[string]string),
}
for key, value := range part.Header() {
file.Meta[key] = value[0]
}
loglogger.Debug().
Str("file", file.Name).
Str("mime", file.Mime).
Msg("file stream uploading")
if serr := r.client.Upload(ctxmeta, work.ID, file); serr != nil {
loglogger.Error().
Err(serr).
Str("file", file.Name).
Str("mime", file.Mime).
Msg("file stream upload error")
}
loglogger.Debug().
Str("file", file.Name).
Str("mime", file.Mime).
Msg("file stream upload complete")
return nil
}
}

36
agent/rpc/client_grpc.go
View file

@ -277,42 +277,6 @@ func (c *client) Update(ctx context.Context, id string, state rpc.State) (err er
return nil
}
// Upload uploads the pipeline artifact.
func (c *client) Upload(ctx context.Context, id string, file *rpc.File) (err error) {
req := new(proto.UploadRequest)
req.Id = id
req.File = new(proto.File)
req.File.Name = file.Name
req.File.Mime = file.Mime
req.File.Step = file.Step
req.File.Size = int32(file.Size)
req.File.Time = file.Time
req.File.Data = file.Data
req.File.Meta = file.Meta
for {
_, err = c.client.Upload(ctx, req)
if err == nil {
break
}
log.Err(err).Msgf("grpc error: upload(): code: %v: %s", status.Code(err), err)
switch status.Code(err) {
case
codes.Aborted,
codes.DataLoss,
codes.DeadlineExceeded,
codes.Internal,
codes.Unavailable:
// non-fatal errors
default:
return err
}
<-time.After(backoff)
}
return nil
}
// Log writes the pipeline log entry.
func (c *client) Log(ctx context.Context, id string, line *rpc.Line) (err error) {
req := new(proto.LogRequest)

3
cmd/server/server.go
View file

@ -279,9 +279,6 @@ func run(c *cli.Context) error {
}
func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) {
// storage
server.Config.Storage.Files = v
// forge
server.Config.Services.Forge = f
server.Config.Services.Timeout = c.Duration("forge-timeout")

14
pipeline/rpc/peer.go
View file

@ -44,17 +44,6 @@ type (
Timeout int64 `json:"timeout"`
}
// File defines a pipeline artifact.
File struct {
Name string `json:"name"`
Step string `json:"step"`
Mime string `json:"mime"`
Time int64 `json:"time"`
Size int `json:"size"`
Data []byte `json:"data"`
Meta map[string]string `json:"meta"`
}
Version struct {
GrpcVersion int32 `json:"grpc_version,omitempty"`
ServerVersion string `json:"server_version,omitempty"`
@ -84,9 +73,6 @@ type Peer interface {
// Update updates the pipeline state.
Update(c context.Context, id string, state State) error
// Upload uploads the pipeline artifact.
Upload(c context.Context, id string, file *File) error
// Log writes the pipeline log entry.
Log(c context.Context, id string, line *Line) error

2
pipeline/rpc/proto/version.go
View file

@ -16,4 +16,4 @@ package proto
// Version is the version of the woodpecker.proto file,
// !IMPORTANT! increased by 1 each time it get changed !IMPORTANT!
const Version int32 = 1
const Version int32 = 2

722
pipeline/rpc/proto/woodpecker.pb.go
View file

File diff suppressed because it is too large Load diff

16
pipeline/rpc/proto/woodpecker.proto
View file

@ -31,7 +31,6 @@ service Woodpecker {
rpc Done (DoneRequest) returns (Empty) {}
rpc Extend (ExtendRequest) returns (Empty) {}
rpc Update (UpdateRequest) returns (Empty) {}
rpc Upload (UploadRequest) returns (Empty) {}
rpc Log (LogRequest) returns (Empty) {}
rpc RegisterAgent (RegisterAgentRequest) returns (RegisterAgentResponse) {}
rpc ReportHealth (ReportHealthRequest) returns (Empty) {}
@ -41,16 +40,6 @@ service Woodpecker {
// Basic Types
//
message File {
string name = 1;
string step = 2;
string mime = 3;
int64 time = 4;
int32 size = 5;
bytes data = 6;
map<string, string> meta = 7;
}
message State {
string name = 1;
bool exited = 2;
@ -103,11 +92,6 @@ message ExtendRequest {
string id = 1;
}
message UploadRequest {
string id = 1;
File file = 2;
}
message UpdateRequest {
string id = 1;
State state = 2;

37
pipeline/rpc/proto/woodpecker_grpc.pb.go
View file

@ -41,7 +41,6 @@ const (
Woodpecker_Done_FullMethodName = "/proto.Woodpecker/Done"
Woodpecker_Extend_FullMethodName = "/proto.Woodpecker/Extend"
Woodpecker_Update_FullMethodName = "/proto.Woodpecker/Update"
Woodpecker_Upload_FullMethodName = "/proto.Woodpecker/Upload"
Woodpecker_Log_FullMethodName = "/proto.Woodpecker/Log"
Woodpecker_RegisterAgent_FullMethodName = "/proto.Woodpecker/RegisterAgent"
Woodpecker_ReportHealth_FullMethodName = "/proto.Woodpecker/ReportHealth"
@ -58,7 +57,6 @@ type WoodpeckerClient interface {
Done(ctx context.Context, in *DoneRequest, opts ...grpc.CallOption) (*Empty, error)
Extend(ctx context.Context, in *ExtendRequest, opts ...grpc.CallOption) (*Empty, error)
Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*Empty, error)
Upload(ctx context.Context, in *UploadRequest, opts ...grpc.CallOption) (*Empty, error)
Log(ctx context.Context, in *LogRequest, opts ...grpc.CallOption) (*Empty, error)
RegisterAgent(ctx context.Context, in *RegisterAgentRequest, opts ...grpc.CallOption) (*RegisterAgentResponse, error)
ReportHealth(ctx context.Context, in *ReportHealthRequest, opts ...grpc.CallOption) (*Empty, error)
@ -135,15 +133,6 @@ func (c *woodpeckerClient) Update(ctx context.Context, in *UpdateRequest, opts .
return out, nil
}
func (c *woodpeckerClient) Upload(ctx context.Context, in *UploadRequest, opts ...grpc.CallOption) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, Woodpecker_Upload_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *woodpeckerClient) Log(ctx context.Context, in *LogRequest, opts ...grpc.CallOption) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, Woodpecker_Log_FullMethodName, in, out, opts...)
@ -182,7 +171,6 @@ type WoodpeckerServer interface {
Done(context.Context, *DoneRequest) (*Empty, error)
Extend(context.Context, *ExtendRequest) (*Empty, error)
Update(context.Context, *UpdateRequest) (*Empty, error)
Upload(context.Context, *UploadRequest) (*Empty, error)
Log(context.Context, *LogRequest) (*Empty, error)
RegisterAgent(context.Context, *RegisterAgentRequest) (*RegisterAgentResponse, error)
ReportHealth(context.Context, *ReportHealthRequest) (*Empty, error)
@ -214,9 +202,6 @@ func (UnimplementedWoodpeckerServer) Extend(context.Context, *ExtendRequest) (*E
func (UnimplementedWoodpeckerServer) Update(context.Context, *UpdateRequest) (*Empty, error) {
return nil, status.Errorf(codes.Unimplemented, "method Update not implemented")
}
func (UnimplementedWoodpeckerServer) Upload(context.Context, *UploadRequest) (*Empty, error) {
return nil, status.Errorf(codes.Unimplemented, "method Upload not implemented")
}
func (UnimplementedWoodpeckerServer) Log(context.Context, *LogRequest) (*Empty, error) {
return nil, status.Errorf(codes.Unimplemented, "method Log not implemented")
}
@ -365,24 +350,6 @@ func _Woodpecker_Update_Handler(srv interface{}, ctx context.Context, dec func(i
return interceptor(ctx, in, info, handler)
}
func _Woodpecker_Upload_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(UploadRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(WoodpeckerServer).Upload(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: Woodpecker_Upload_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).Upload(ctx, req.(*UploadRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Woodpecker_Log_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(LogRequest)
if err := dec(in); err != nil {
@ -472,10 +439,6 @@ var Woodpecker_ServiceDesc = grpc.ServiceDesc{
MethodName: "Update",
Handler: _Woodpecker_Update_Handler,
},
{
MethodName: "Upload",
Handler: _Woodpecker_Upload_Handler,
},
{
MethodName: "Log",
Handler: _Woodpecker_Log_Handler,

118
server/api/file.go
View file

@ -1,118 +0,0 @@
// Copyright 2022 Woodpecker Authors
// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"io"
"net/http"
"strconv"
"strings"
"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server/router/middleware/session"
"github.com/woodpecker-ci/woodpecker/server/store"
)
// FileList gets a list file by pipeline.
func FileList(c *gin.Context) {
_store := store.FromContext(c)
num, err := strconv.ParseInt(c.Param("number"), 10, 64)
if err != nil {
_ = c.AbortWithError(http.StatusBadRequest, err)
return
}
repo := session.Repo(c)
pipeline, err := _store.GetPipelineNumber(repo, num)
if err != nil {
handleDbGetError(c, err)
return
}
files, err := _store.FileList(pipeline, session.Pagination(c))
if err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
c.JSON(200, files)
}
// FileGet gets a file by process and name
func FileGet(c *gin.Context) {
var (
_store = store.FromContext(c)
repo = session.Repo(c)
name = strings.TrimPrefix(c.Param("file"), "/")
raw = func() bool {
return c.DefaultQuery("raw", "false") == "true"
}()
)
num, err := strconv.ParseInt(c.Param("number"), 10, 64)
if err != nil {
_ = c.AbortWithError(http.StatusBadRequest, err)
return
}
pid, err := strconv.Atoi(c.Param("step"))
if err != nil {
_ = c.AbortWithError(http.StatusBadRequest, err)
return
}
pipeline, err := _store.GetPipelineNumber(repo, num)
if err != nil {
handleDbGetError(c, err)
return
}
step, err := _store.StepFind(pipeline, pid)
if err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
file, err := _store.FileFind(step, name)
if err != nil {
c.String(http.StatusNotFound, "Error getting file %q. %s", name, err)
return
}
if !raw {
c.JSON(http.StatusOK, file)
return
}
rc, err := _store.FileRead(step, file.Name)
if err != nil {
c.String(http.StatusNotFound, "Error getting file stream %q. %s", name, err)
return
}
defer rc.Close()
switch file.Mime {
case "application/vnd.test+json":
c.Header("Content-Type", "application/json")
}
if _, err := io.Copy(c.Writer, rc); err != nil {
log.Error().Err(err).Msg("could not copy file to http response")
}
}

2
server/api/pipeline.go
View file

@ -124,13 +124,11 @@ func GetPipeline(c *gin.Context) {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
files, _ := _store.FileList(pl, &model.ListOptions{All: true})
steps, _ := _store.StepList(pl)
if pl.Steps, err = model.Tree(steps); err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
pl.Files = files
c.JSON(http.StatusOK, pl)
}

1
server/config.go
View file

@ -50,7 +50,6 @@ var Config = struct {
// Repos model.RepoStore
// Builds model.BuildStore
// Logs model.LogStore
Files model.FileStore
Steps model.StepStore
// Registries model.RegistryStore
// Secrets model.SecretStore

2
server/forge/mocks/forge.go
View file

@ -1,4 +1,4 @@
// Code generated by mockery v2.26.1. DO NOT EDIT.
// Code generated by mockery v2.28.1. DO NOT EDIT.
package mocks

75
server/grpc/rpc.go
View file

@ -19,7 +19,6 @@
package grpc
import (
"bytes"
"context"
"encoding/json"
"errors"
@ -33,7 +32,6 @@ import (
grpcMetadata "google.golang.org/grpc/metadata"
"github.com/woodpecker-ci/woodpecker/pipeline/rpc"
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/logging"
"github.com/woodpecker-ci/woodpecker/server/model"
@ -167,79 +165,6 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error {
return nil
}
// Upload implements the rpc.Upload function
func (s *RPC) Upload(_ context.Context, id string, file *rpc.File) error {
stepID, err := strconv.ParseInt(id, 10, 64)
if err != nil {
return err
}
pstep, err := s.store.StepLoad(stepID)
if err != nil {
log.Error().Msgf("error: cannot find parent step with id %d: %s", stepID, err)
return err
}
pipeline, err := s.store.GetPipeline(pstep.PipelineID)
if err != nil {
log.Error().Msgf("error: cannot find pipeline with id %d: %s", pstep.PipelineID, err)
return err
}
step, err := s.store.StepChild(pipeline, pstep.PID, file.Step)
if err != nil {
log.Error().Msgf("error: cannot find child step with name %s: %s", file.Step, err)
return err
}
if file.Mime == "application/json+logs" {
return s.store.LogSave(
step,
bytes.NewBuffer(file.Data),
)
}
report := &model.File{
PipelineID: step.PipelineID,
StepID: step.ID,
PID: step.PID,
Mime: file.Mime,
Name: file.Name,
Size: file.Size,
Time: file.Time,
}
if d, ok := file.Meta["X-Tests-Passed"]; ok {
report.Passed, _ = strconv.Atoi(d)
}
if d, ok := file.Meta["X-Tests-Failed"]; ok {
report.Failed, _ = strconv.Atoi(d)
}
if d, ok := file.Meta["X-Tests-Skipped"]; ok {
report.Skipped, _ = strconv.Atoi(d)
}
if d, ok := file.Meta["X-Checks-Passed"]; ok {
report.Passed, _ = strconv.Atoi(d)
}
if d, ok := file.Meta["X-Checks-Failed"]; ok {
report.Failed, _ = strconv.Atoi(d)
}
if d, ok := file.Meta["X-Coverage-Lines"]; ok {
report.Passed, _ = strconv.Atoi(d)
}
if d, ok := file.Meta["X-Coverage-Total"]; ok {
if total, _ := strconv.Atoi(d); total != 0 {
report.Failed = total - report.Passed
}
}
return server.Config.Storage.Files.FileCreate(
report,
bytes.NewBuffer(file.Data),
)
}
// Init implements the rpc.Init function
func (s *RPC) Init(c context.Context, id string, state rpc.State) error {
stepID, err := strconv.ParseInt(id, 10, 64)

16
server/grpc/server.go
View file

@ -118,22 +118,6 @@ func (s *WoodpeckerServer) Update(c context.Context, req *proto.UpdateRequest) (
return res, err
}
func (s *WoodpeckerServer) Upload(c context.Context, req *proto.UploadRequest) (*proto.Empty, error) {
file := &rpc.File{
Data: req.GetFile().GetData(),
Mime: req.GetFile().GetMime(),
Name: req.GetFile().GetName(),
Step: req.GetFile().GetStep(),
Size: int(req.GetFile().GetSize()),
Time: req.GetFile().GetTime(),
Meta: req.GetFile().GetMeta(),
}
res := new(proto.Empty)
err := s.peer.Upload(c, req.GetId(), file)
return res, err
}
func (s *WoodpeckerServer) Done(c context.Context, req *proto.DoneRequest) (*proto.Empty, error) {
state := rpc.State{
Error: req.GetState().GetError(),

47
server/model/file.go
View file

@ -1,47 +0,0 @@
// Copyright 2021 Woodpecker Authors
// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import "io"
// FileStore persists pipeline artifacts to storage.
type FileStore interface {
FileList(*Pipeline, *ListOptions) ([]*File, error)
FileFind(*Step, string) (*File, error)
FileRead(*Step, string) (io.ReadCloser, error)
FileCreate(*File, io.Reader) error
}
// File represents a pipeline artifact.
type File struct {
ID int64 `json:"id" xorm:"pk autoincr 'file_id'"`
PipelineID int64 `json:"-" xorm:"INDEX 'file_pipeline_id'"`
StepID int64 `json:"step_id" xorm:"UNIQUE(s) INDEX 'file_step_id'"`
PID int `json:"pid" xorm:"file_pid"`
Name string `json:"name" xorm:"UNIQUE(s) file_name"`
Size int `json:"size" xorm:"file_size"`
Mime string `json:"mime" xorm:"file_mime"`
Time int64 `json:"time" xorm:"file_time"`
Passed int `json:"passed" xorm:"file_meta_passed"`
Failed int `json:"failed" xorm:"file_meta_failed"`
Skipped int `json:"skipped" xorm:"file_meta_skipped"`
Data []byte `json:"-" xorm:"file_data"` // TODO: don't store in db but object storage?
}
// TableName return database table name for xorm
func (File) TableName() string {
return "files"
}

1
server/model/pipeline.go
View file

@ -49,7 +49,6 @@ type Pipeline struct {
Reviewer string `json:"reviewed_by" xorm:"pipeline_reviewer"`
Reviewed int64 `json:"reviewed_at" xorm:"pipeline_reviewed"`
Steps []*Step `json:"steps,omitempty" xorm:"-"`
Files []*File `json:"files,omitempty" xorm:"-"`
ChangedFiles []string `json:"changed_files,omitempty" xorm:"json 'changed_files'"`
AdditionalVariables map[string]string `json:"variables,omitempty" xorm:"json 'additional_variables'"`
PullRequestLabels []string `json:"pr_labels,omitempty" xorm:"json 'pr_labels'"`

3
server/router/api.go
View file

@ -95,9 +95,6 @@ func apiRoutes(e *gin.Engine) {
// requires push permissions
repo.DELETE("/logs/:number", session.MustPush, api.DeletePipelineLogs)
repo.GET("/files/:number", api.FileList)
repo.GET("/files/:number/:step/*file", api.FileGet)
// requires push permissions
repo.GET("/secrets", session.MustPush, api.GetSecretList)
repo.POST("/secrets", session.MustPush, api.PostSecret)

56
server/store/datastore/file.go
View file

@ -1,56 +0,0 @@
// Copyright 2021 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datastore
import (
"bytes"
"io"
"github.com/woodpecker-ci/woodpecker/server/model"
)
func (s storage) FileList(pipeline *model.Pipeline, p *model.ListOptions) ([]*model.File, error) {
var files []*model.File
return files, s.paginate(p).Where("file_pipeline_id = ?", pipeline.ID).
Find(&files)
}
func (s storage) FileFind(step *model.Step, name string) (*model.File, error) {
file := &model.File{
StepID: step.ID,
Name: name,
}
return file, wrapGet(s.engine.Get(file))
}
func (s storage) FileRead(step *model.Step, name string) (io.ReadCloser, error) {
file, err := s.FileFind(step, name)
if err != nil {
return nil, err
}
buf := bytes.NewBuffer(file.Data)
return io.NopCloser(buf), err
}
func (s storage) FileCreate(file *model.File, reader io.Reader) error {
data, err := io.ReadAll(reader)
if err != nil {
return err
}
file.Data = data
// only Insert set auto created ID back to object
_, err = s.engine.Insert(file)
return err
}

192
server/store/datastore/file_test.go
View file

@ -1,192 +0,0 @@
// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datastore
import (
"bytes"
"io"
"testing"
"github.com/stretchr/testify/assert"
"github.com/woodpecker-ci/woodpecker/server/model"
)
func TestFileFind(t *testing.T) {
store, closer := newTestStore(t, new(model.File), new(model.Step))
defer closer()
if err := store.FileCreate(
&model.File{
PipelineID: 2,
StepID: 1,
Name: "hello.txt",
Mime: "text/plain",
Size: 11,
},
bytes.NewBufferString("hello world"),
); err != nil {
t.Errorf("Unexpected error: insert file: %s", err)
return
}
file, err := store.FileFind(&model.Step{ID: 1}, "hello.txt")
if err != nil {
t.Error(err)
return
}
if got, want := file.ID, int64(1); got != want {
t.Errorf("Want file id %d, got %d", want, got)
}
if got, want := file.PipelineID, int64(2); got != want {
t.Errorf("Want file pipeline id %d, got %d", want, got)
}
if got, want := file.StepID, int64(1); got != want {
t.Errorf("Want file step id %d, got %d", want, got)
}
if got, want := file.Name, "hello.txt"; got != want {
t.Errorf("Want file name %s, got %s", want, got)
}
if got, want := file.Mime, "text/plain"; got != want {
t.Errorf("Want file mime %s, got %s", want, got)
}
if got, want := file.Size, 11; got != want {
t.Errorf("Want file size %d, got %d", want, got)
}
rc, err := store.FileRead(&model.Step{ID: 1}, "hello.txt")
if err != nil {
t.Error(err)
return
}
out, _ := io.ReadAll(rc)
if got, want := string(out), "hello world"; got != want {
t.Errorf("Want file data %s, got %s", want, got)
}
}
func TestFileList(t *testing.T) {
store, closer := newTestStore(t, new(model.File), new(model.Pipeline))
defer closer()
assert.NoError(t, store.FileCreate(
&model.File{
PipelineID: 1,
StepID: 1,
Name: "hello.txt",
Mime: "text/plain",
Size: 11,
},
bytes.NewBufferString("hello world"),
))
assert.NoError(t, store.FileCreate(
&model.File{
PipelineID: 1,
StepID: 1,
Name: "hola.txt",
Mime: "text/plain",
Size: 11,
},
bytes.NewBufferString("hola mundo"),
))
files, err := store.FileList(&model.Pipeline{ID: 1}, &model.ListOptions{Page: 1, PerPage: 50})
if err != nil {
t.Errorf("Unexpected error: select files: %s", err)
return
}
if got, want := len(files), 2; got != want {
t.Errorf("Wanted %d files, got %d", want, got)
}
}
func TestFileIndexes(t *testing.T) {
store, closer := newTestStore(t, new(model.File), new(model.Pipeline))
defer closer()
if err := store.FileCreate(
&model.File{
PipelineID: 1,
StepID: 1,
Name: "hello.txt",
Size: 11,
Mime: "text/plain",
},
bytes.NewBufferString("hello world"),
); err != nil {
t.Errorf("Unexpected error: insert file: %s", err)
return
}
// fail due to duplicate file name
if err := store.FileCreate(
&model.File{
PipelineID: 1,
StepID: 1,
Name: "hello.txt",
Mime: "text/plain",
Size: 11,
},
bytes.NewBufferString("hello world"),
); err == nil {
t.Errorf("Unexpected error: duplicate pid")
}
}
func TestFileCascade(t *testing.T) {
store, closer := newTestStore(t, new(model.File), new(model.Step), new(model.Pipeline))
defer closer()
stepOne := &model.Step{
PipelineID: 1,
PID: 1,
PGID: 1,
Name: "build",
State: "success",
}
err1 := store.StepCreate([]*model.Step{stepOne})
assert.EqualValues(t, int64(1), stepOne.ID)
err2 := store.FileCreate(
&model.File{
PipelineID: 1,
StepID: 1,
Name: "hello.txt",
Mime: "text/plain",
Size: 11,
},
bytes.NewBufferString("hello world"),
)
if err1 != nil {
t.Errorf("Unexpected error: cannot insert step: %s", err1)
} else if err2 != nil {
t.Errorf("Unexpected error: cannot insert file: %s", err2)
}
if _, err3 := store.StepFind(&model.Pipeline{ID: 1}, 1); err3 != nil {
t.Errorf("Unexpected error: cannot get inserted step: %s", err3)
}
err := store.StepClear(&model.Pipeline{ID: 1, Steps: []*model.Step{stepOne}})
assert.NoError(t, err)
file, err4 := store.FileFind(&model.Step{ID: 1}, "hello.txt")
if err4 == nil {
t.Errorf("Expected no rows in result set error")
t.Log(file)
}
}

26
server/store/datastore/migration/016_remove_files_table.go Normal file
View file

@ -0,0 +1,26 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package migration
import (
"xorm.io/xorm"
)
var dropFiles = task{
name: "drop-files",
fn: func(sess *xorm.Session) error {
return sess.DropTable("files")
},
}

2
server/store/datastore/migration/migration.go
View file

@ -44,6 +44,7 @@ var migrationTasks = []*task{
&renameForgeIDToForgeRemoteID,
&removeActiveFromUsers,
&removeInactiveRepos,
&dropFiles,
}
var allBeans = []interface{}{
@ -51,7 +52,6 @@ var allBeans = []interface{}{
new(model.Pipeline),
new(model.PipelineConfig),
new(model.Config),
new(model.File),
new(model.Logs),
new(model.Perm),
new(model.Step),

1
server/store/datastore/repo_test.go
View file

@ -301,7 +301,6 @@ func TestRepoCrud(t *testing.T) {
new(model.PipelineConfig),
new(model.Logs),
new(model.Step),
new(model.File),
new(model.Secret),
new(model.Registry),
new(model.Config),

7
server/store/datastore/step.go
View file

@ -79,10 +79,6 @@ func (s storage) StepClear(pipeline *model.Pipeline) error {
return err
}
if _, err := sess.Where("file_pipeline_id = ?", pipeline.ID).Delete(new(model.File)); err != nil {
return err
}
if _, err := sess.Where("step_pipeline_id = ?", pipeline.ID).Delete(new(model.Step)); err != nil {
return err
}
@ -94,9 +90,6 @@ func deleteStep(sess *xorm.Session, stepID int64) error {
if _, err := sess.Where("log_step_id = ?", stepID).Delete(new(model.Logs)); err != nil {
return err
}
if _, err := sess.Where("file_step_id = ?", stepID).Delete(new(model.File)); err != nil {
return err
}
_, err := sess.ID(stepID).Delete(new(model.Step))
return err
}

94
server/store/mocks/store.go
View file

@ -1,4 +1,4 @@
// Code generated by mockery v2.26.1. DO NOT EDIT.
// Code generated by mockery v2.28.1. DO NOT EDIT.
package mocks
@ -473,98 +473,6 @@ func (_m *Store) DeleteUser(_a0 *model.User) error {
return r0
}
// FileCreate provides a mock function with given fields: _a0, _a1
func (_m *Store) FileCreate(_a0 *model.File, _a1 io.Reader) error {
ret := _m.Called(_a0, _a1)
var r0 error
if rf, ok := ret.Get(0).(func(*model.File, io.Reader) error); ok {
r0 = rf(_a0, _a1)
} else {
r0 = ret.Error(0)
}
return r0
}
// FileFind provides a mock function with given fields: _a0, _a1
func (_m *Store) FileFind(_a0 *model.Step, _a1 string) (*model.File, error) {
ret := _m.Called(_a0, _a1)
var r0 *model.File
var r1 error
if rf, ok := ret.Get(0).(func(*model.Step, string) (*model.File, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(*model.Step, string) *model.File); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.File)
}
}
if rf, ok := ret.Get(1).(func(*model.Step, string) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FileList provides a mock function with given fields: _a0, _a1
func (_m *Store) FileList(_a0 *model.Pipeline, _a1 *model.ListOptions) ([]*model.File, error) {
ret := _m.Called(_a0, _a1)
var r0 []*model.File
var r1 error
if rf, ok := ret.Get(0).(func(*model.Pipeline, *model.ListOptions) ([]*model.File, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(*model.Pipeline, *model.ListOptions) []*model.File); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.File)
}
}
if rf, ok := ret.Get(1).(func(*model.Pipeline, *model.ListOptions) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FileRead provides a mock function with given fields: _a0, _a1
func (_m *Store) FileRead(_a0 *model.Step, _a1 string) (io.ReadCloser, error) {
ret := _m.Called(_a0, _a1)
var r0 io.ReadCloser
var r1 error
if rf, ok := ret.Get(0).(func(*model.Step, string) (io.ReadCloser, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(*model.Step, string) io.ReadCloser); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(io.ReadCloser)
}
}
if rf, ok := ret.Get(1).(func(*model.Step, string) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetActivePipelineList provides a mock function with given fields: repo
func (_m *Store) GetActivePipelineList(repo *model.Repo) ([]*model.Pipeline, error) {
ret := _m.Called(repo)

6
server/store/store.go
View file

@ -150,12 +150,6 @@ type Store interface {
// so either find a way to write log in chunks by xorm ...
LogSave(*model.Step, io.Reader) error
// Files
FileList(*model.Pipeline, *model.ListOptions) ([]*model.File, error)
FileFind(*model.Step, string) (*model.File, error)
FileRead(*model.Step, string) (io.ReadCloser, error)
FileCreate(*model.File, io.Reader) error
// Tasks
// TaskList TODO: paginate & opt filter
TaskList() ([]*model.Task, error)

8
web/src/lib/api/index.ts
View file

@ -133,14 +133,6 @@ export default class WoodpeckerClient extends ApiClient {
return this._get(`/api/repos/${owner}/${repo}/logs/${pipeline}/${step}`) as Promise<PipelineLog[]>;
}
getArtifact(owner: string, repo: string, pipeline: string, step: string, file: string): Promise<unknown> {
return this._get(`/api/repos/${owner}/${repo}/files/${pipeline}/${step}/${file}?raw=true`);
}
getArtifactList(owner: string, repo: string, pipeline: string): Promise<unknown> {
return this._get(`/api/repos/${owner}/${repo}/files/${pipeline}`);
}
getSecretList(owner: string, repo: string, page: number): Promise<Secret[] | null> {
return this._get(`/api/repos/${owner}/${repo}/secrets?page=${page}`) as Promise<Secret[] | null>;
}