add proc and file structs

This commit is contained in:
Brad Rydzewski 2017-03-28 17:53:06 +09:00
parent a51dfa4208
commit 3118c07329
16 changed files with 899 additions and 27 deletions

3
.gitignore vendored
View file

@ -1,6 +1,9 @@
drone/drone
*.sqlite
*_gen.go
!store/datastore/sql/sqlite/sql_gen.go
!store/datastore/sql/mysql/sql_gen.go
!store/datastore/sql/postgres/sql_gen.go
#*.css
*.txt
*.zip

23
model/file.go Normal file
View file

@ -0,0 +1,23 @@
package model
import "io"
// FileStore persists pipeline artifacts to storage.
type FileStore interface {
FileList(*Build) ([]*File, error)
FileFind(*Proc, string) (*File, error)
FileRead(*Proc, string) (io.ReadCloser, error)
FileCreate(*File, io.Reader) error
}
// File represents a pipeline artifact.
type File struct {
ID int64 `json:"id" meddler:"file_id,pk"`
BuildID int64 `json:"build_id" meddler:"file_build_id"`
ProcID int64 `json:"proc_id" meddler:"file_proc_id"`
Name string `json:"name" meddler:"file_name"`
Size int `json:"size" meddler:"file_size"`
Mime string `json:"mime" meddler:"file_mime"`
Time int64 `json:"time" meddler:"file_time"`
// Data []byte `json:"data" meddler:"file_data"`
}

View file

@ -1,8 +0,0 @@
package model
type Key struct {
ID int64 `json:"-" meddler:"key_id,pk"`
RepoID int64 `json:"-" meddler:"key_repo_id"`
Public string `json:"public" meddler:"key_public"`
Private string `json:"private" meddler:"key_private"`
}

57
model/proc.go Normal file
View file

@ -0,0 +1,57 @@
package model
// ProcStore persists process information to storage.
type ProcStore interface {
ProcFind(*Build, int) (*Proc, error)
ProcChild(*Build, int, string) (*Proc, error)
ProcList(*Build) ([]*Proc, error)
ProcCreate([]*Proc) error
ProcUpdate(*Proc) error
}
// Proc represents a process in the build pipeline.
type Proc struct {
ID int64 `json:"id" meddler:"proc_id,pk"`
BuildID int64 `json:"build_id" meddler:"proc_build_id"`
PID int `json:"pid" meddler:"proc_pid"`
PPID int `json:"ppid" meddler:"proc_ppid"`
PGID int `json:"pgid" meddler:"proc_pgid"`
Name string `json:"name" meddler:"proc_name"`
State string `json:"state" meddler:"proc_state"`
Error string `json:"error,omitempty" meddler:"proc_error"`
ExitCode int `json:"exit_code" meddler:"proc_exit_code"`
Started int64 `json:"start_time,omitempty" meddler:"proc_started"`
Stopped int64 `json:"end_time,omitempty" meddler:"proc_stopped"`
Machine string `json:"machine,omitempty" meddler:"proc_machine"`
Platform string `json:"platform,omitempty" meddler:"proc_platform"`
Environ map[string]string `json:"environ,omitempty" meddler:"proc_environ,json"`
Children []*Proc `json:"children,omitempty" meddler:"-"`
}
// Running returns true if the process state is pending or running.
func (p *Proc) Running() bool {
return p.State == StatusPending || p.State == StatusRunning
}
// Failing returns true if the process state is failed, killed or error.
func (p *Proc) Failing() bool {
return p.State == StatusError || p.State == StatusKilled || p.State == StatusFailure
}
// Tree creates a process tree from a flat process list.
func Tree(procs []*Proc) []*Proc {
var (
nodes []*Proc
parent *Proc
)
for _, proc := range procs {
if proc.PPID == 0 {
nodes = append(nodes, proc)
parent = proc
continue
} else {
parent.Children = append(parent.Children, proc)
}
}
return nodes
}

View file

@ -1,19 +0,0 @@
package model
// Work represents an item for work to be
// processed by a worker.
type Work struct {
Signed bool `json:"signed"`
Verified bool `json:"verified"`
Yaml string `json:"config"`
YamlEnc string `json:"secret"`
Repo *Repo `json:"repo"`
Build *Build `json:"build"`
BuildLast *Build `json:"build_last"`
Job *Job `json:"job"`
Netrc *Netrc `json:"netrc"`
Keys *Key `json:"keys"`
System *System `json:"system"`
Secrets []*Secret `json:"secrets"`
User *User `json:"user"`
}

View file

@ -0,0 +1,47 @@
-- +migrate Up
CREATE TABLE procs (
proc_id INTEGER PRIMARY KEY AUTOINCREMENT
,proc_build_id INTEGER
,proc_pid INTEGER
,proc_ppid INTEGER
,proc_pgid INTEGER
,proc_name TEXT
,proc_state TEXT
,proc_error TEXT
,proc_exit_code INTEGER
,proc_started INTEGER
,proc_stopped INTEGER
,proc_machine TEXT
,proc_platform TEXT
,proc_environ TEXT
,UNIQUE(proc_build_id, proc_pid)
,UNIQUE(proc_build_id, proc_name)
);
CREATE INDEX proc_build_ix ON procs (proc_build_id);
CREATE TABLE files (
file_id INTEGER PRIMARY KEY AUTOINCREMENT
,file_build_id INTEGER
,file_proc_id INTEGER
,file_name TEXT
,file_mime TEXT
,file_size INTEGER
,file_time INTEGER
,file_data BLOB
,UNIQUE(file_proc_id,file_name)
,FOREIGN KEY(file_proc_id) REFERENCES procs (proc_id) ON DELETE CASCADE
);
CREATE INDEX file_build_ix ON files (file_build_id);
CREATE INDEX file_proc_ix ON files (file_proc_id);
-- +migrate Down
DROP INDEX file_build_ix;
DROP INDEX file_proc_ix;
DROP TABLE files;
DROP INDEX proc_build_ix;
DROP TABLE procs;

63
store/datastore/files.go Normal file
View file

@ -0,0 +1,63 @@
package datastore
import (
"bytes"
"io"
"io/ioutil"
"github.com/drone/drone/model"
"github.com/drone/drone/store/datastore/sql"
"github.com/russross/meddler"
)
func (db *datastore) FileList(build *model.Build) ([]*model.File, error) {
stmt := sql.Lookup(sql.DriverMysql, "files-find-build")
list := []*model.File{}
err := meddler.QueryAll(db, &list, stmt, build.ID)
return list, err
}
func (db *datastore) FileFind(proc *model.Proc, name string) (*model.File, error) {
stmt := sql.Lookup(sql.DriverMysql, "files-find-proc-name")
file := new(model.File)
err := meddler.QueryRow(db, file, stmt, proc.ID, name)
return file, err
}
func (db *datastore) FileRead(proc *model.Proc, name string) (io.ReadCloser, error) {
stmt := sql.Lookup(sql.DriverMysql, "files-find-proc-name-data")
file := new(fileData)
err := meddler.QueryRow(db, file, stmt, proc.ID, name)
buf := bytes.NewBuffer(file.Data)
return ioutil.NopCloser(buf), err
}
func (db *datastore) FileCreate(file *model.File, r io.Reader) error {
d, err := ioutil.ReadAll(r)
if err != nil {
return err
}
f := fileData{
ID: file.ID,
BuildID: file.BuildID,
ProcID: file.ProcID,
Name: file.Name,
Size: file.Size,
Mime: file.Mime,
Time: file.Time,
Data: d,
}
return meddler.Insert(db, "files", &f)
}
type fileData struct {
ID int64 `meddler:"file_id,pk"`
BuildID int64 `meddler:"file_build_id"`
ProcID int64 `meddler:"file_proc_id"`
Name string `meddler:"file_name"`
Size int `meddler:"file_size"`
Mime string `meddler:"file_mime"`
Time int64 `meddler:"file_time"`
Data []byte `meddler:"file_data"`
}

View file

@ -0,0 +1,178 @@
package datastore
import (
"bytes"
"io/ioutil"
"testing"
"github.com/drone/drone/model"
)
func TestFileFind(t *testing.T) {
db := openTest()
defer db.Close()
s := From(db)
if err := s.FileCreate(
&model.File{
BuildID: 2,
ProcID: 1,
Name: "hello.txt",
Mime: "text/plain",
Size: 11,
},
bytes.NewBufferString("hello world"),
); err != nil {
t.Errorf("Unexpected error: insert file: %s", err)
return
}
file, err := s.FileFind(&model.Proc{ID: 1}, "hello.txt")
if err != nil {
t.Error(err)
return
}
if got, want := file.ID, int64(1); got != want {
t.Errorf("Want file id %d, got %d", want, got)
}
if got, want := file.BuildID, int64(2); got != want {
t.Errorf("Want file build id %d, got %d", want, got)
}
if got, want := file.ProcID, int64(1); got != want {
t.Errorf("Want file proc id %d, got %d", want, got)
}
if got, want := file.Name, "hello.txt"; got != want {
t.Errorf("Want file name %s, got %s", want, got)
}
if got, want := file.Mime, "text/plain"; got != want {
t.Errorf("Want file mime %s, got %s", want, got)
}
if got, want := file.Size, 11; got != want {
t.Errorf("Want file size %d, got %d", want, got)
}
rc, err := s.FileRead(&model.Proc{ID: 1}, "hello.txt")
if err != nil {
t.Error(err)
return
}
out, _ := ioutil.ReadAll(rc)
if got, want := string(out), "hello world"; got != want {
t.Errorf("Want file data %s, got %s", want, got)
}
}
func TestFileList(t *testing.T) {
db := openTest()
defer db.Close()
s := From(db)
s.FileCreate(
&model.File{
BuildID: 1,
ProcID: 1,
Name: "hello.txt",
Mime: "text/plain",
Size: 11,
},
bytes.NewBufferString("hello world"),
)
s.FileCreate(
&model.File{
BuildID: 1,
ProcID: 1,
Name: "hola.txt",
Mime: "text/plain",
Size: 11,
},
bytes.NewBufferString("hola mundo"),
)
files, err := s.FileList(&model.Build{ID: 1})
if err != nil {
t.Errorf("Unexpected error: select files: %s", err)
return
}
if got, want := len(files), 2; got != want {
t.Errorf("Wanted %d files, got %d", want, got)
}
}
func TestFileIndexes(t *testing.T) {
db := openTest()
defer db.Close()
s := From(db)
if err := s.FileCreate(
&model.File{
BuildID: 1,
ProcID: 1,
Name: "hello.txt",
Size: 11,
Mime: "text/plain",
},
bytes.NewBufferString("hello world"),
); err != nil {
t.Errorf("Unexpected error: insert file: %s", err)
return
}
// fail due to duplicate file name
if err := s.FileCreate(
&model.File{
BuildID: 1,
ProcID: 1,
Name: "hello.txt",
Mime: "text/plain",
Size: 11,
},
bytes.NewBufferString("hello world"),
); err == nil {
t.Errorf("Unexpected error: dupliate pid")
}
}
// func TestFileCascade(t *testing.T) {
// db := openTest()
// defer db.Close()
//
// s := From(db)
// err1 := s.ProcCreate([]*model.Proc{
// {
// BuildID: 1,
// PID: 1,
// PGID: 1,
// Name: "build",
// State: "success",
// },
// })
// err2 := s.FileCreate(
// &model.File{
// BuildID: 1,
// ProcID: 1,
// Name: "hello.txt",
// Mime: "text/plain",
// Size: 11,
// },
// bytes.NewBufferString("hello world"),
// )
//
// if err1 != nil {
// t.Errorf("Unexpected error: cannot insert proc: %s", err1)
// } else if err2 != nil {
// t.Errorf("Unexpected error: cannot insert file: %s", err2)
// }
//
// if _, err3 := s.ProcFind(&model.Build{ID: 1}, 1); err3 != nil {
// t.Errorf("Unexpected error: cannot get inserted proc: %s", err3)
// }
//
// db.Exec("delete from procs where proc_id = 1")
//
// file, err4 := s.FileFind(&model.Proc{ID: 1}, "hello.txt")
// if err4 == nil {
// t.Errorf("Expected no rows in result set error")
// t.Log(file)
// }
// }

41
store/datastore/procs.go Normal file
View file

@ -0,0 +1,41 @@
package datastore
import (
"github.com/drone/drone/model"
"github.com/drone/drone/store/datastore/sql"
"github.com/russross/meddler"
)
func (db *datastore) ProcFind(build *model.Build, pid int) (*model.Proc, error) {
stmt := sql.Lookup(sql.DriverSqlite, "procs-find-build-pid")
proc := new(model.Proc)
err := meddler.QueryRow(db, proc, stmt, build.ID, pid)
return proc, err
}
func (db *datastore) ProcChild(build *model.Build, pid int, child string) (*model.Proc, error) {
stmt := sql.Lookup(sql.DriverSqlite, "procs-find-build-ppid")
proc := new(model.Proc)
err := meddler.QueryRow(db, proc, stmt, build.ID, pid, child)
return proc, err
}
func (db *datastore) ProcList(build *model.Build) ([]*model.Proc, error) {
stmt := sql.Lookup(sql.DriverSqlite, "procs-find-build")
list := []*model.Proc{}
err := meddler.QueryAll(db, &list, stmt, build.ID)
return list, err
}
func (db *datastore) ProcCreate(procs []*model.Proc) error {
for _, proc := range procs {
if err := meddler.Insert(db, "procs", proc); err != nil {
return err
}
}
return nil
}
func (db *datastore) ProcUpdate(proc *model.Proc) error {
return meddler.Update(db, "procs", proc)
}

View file

@ -0,0 +1,229 @@
package datastore
import (
"testing"
"github.com/drone/drone/model"
)
func TestProcFind(t *testing.T) {
db := openTest()
defer db.Close()
s := From(db)
err := s.ProcCreate([]*model.Proc{
{
BuildID: 1000,
PID: 1,
PPID: 2,
PGID: 3,
Name: "build",
State: model.StatusSuccess,
Error: "pc load letter",
ExitCode: 255,
Machine: "localhost",
Platform: "linux/amd64",
Environ: map[string]string{"GOLANG": "tip"},
},
})
if err != nil {
t.Errorf("Unexpected error: insert procs: %s", err)
return
}
proc, err := s.ProcFind(&model.Build{ID: 1000}, 1)
if err != nil {
t.Error(err)
return
}
if got, want := proc.BuildID, int64(1000); got != want {
t.Errorf("Want proc fk %d, got %d", want, got)
}
if got, want := proc.ID, int64(1); got != want {
t.Errorf("Want proc pk %d, got %d", want, got)
}
if got, want := proc.PID, 1; got != want {
t.Errorf("Want proc ppid %d, got %d", want, got)
}
if got, want := proc.PPID, 2; got != want {
t.Errorf("Want proc ppid %d, got %d", want, got)
}
if got, want := proc.PGID, 3; got != want {
t.Errorf("Want proc pgid %d, got %d", want, got)
}
if got, want := proc.Name, "build"; got != want {
t.Errorf("Want proc name %s, got %s", want, got)
}
}
func TestProcChild(t *testing.T) {
db := openTest()
defer db.Close()
s := From(db)
err := s.ProcCreate([]*model.Proc{
{
BuildID: 1,
PID: 1,
PPID: 1,
PGID: 1,
State: "success",
},
{
BuildID: 1,
PID: 2,
PGID: 2,
PPID: 1,
Name: "build",
State: "success",
},
})
if err != nil {
t.Errorf("Unexpected error: insert procs: %s", err)
return
}
proc, err := s.ProcChild(&model.Build{ID: 1}, 1, "build")
if err != nil {
t.Error(err)
return
}
if got, want := proc.PID, 2; got != want {
t.Errorf("Want proc pid %d, got %d", want, got)
}
if got, want := proc.Name, "build"; got != want {
t.Errorf("Want proc name %s, got %s", want, got)
}
}
func TestProcList(t *testing.T) {
db := openTest()
defer db.Close()
s := From(db)
err := s.ProcCreate([]*model.Proc{
{
BuildID: 2,
PID: 1,
PPID: 1,
PGID: 1,
State: "success",
},
{
BuildID: 1,
PID: 1,
PPID: 1,
PGID: 1,
State: "success",
},
{
BuildID: 1,
PID: 2,
PGID: 2,
PPID: 1,
Name: "build",
State: "success",
},
})
if err != nil {
t.Errorf("Unexpected error: insert procs: %s", err)
return
}
procs, err := s.ProcList(&model.Build{ID: 1})
if err != nil {
t.Error(err)
return
}
if got, want := len(procs), 2; got != want {
t.Errorf("Want %d procs, got %d", want, got)
}
}
func TestProcUpdate(t *testing.T) {
db := openTest()
defer db.Close()
s := From(db)
proc := &model.Proc{
BuildID: 1,
PID: 1,
PPID: 2,
PGID: 3,
Name: "build",
State: "pending",
Error: "pc load letter",
ExitCode: 255,
Machine: "localhost",
Platform: "linux/amd64",
Environ: map[string]string{"GOLANG": "tip"},
}
if err := s.ProcCreate([]*model.Proc{proc}); err != nil {
t.Errorf("Unexpected error: insert proc: %s", err)
return
}
proc.State = "running"
if err := s.ProcUpdate(proc); err != nil {
t.Errorf("Unexpected error: update proc: %s", err)
return
}
updated, err := s.ProcFind(&model.Build{ID: 1}, 1)
if err != nil {
t.Error(err)
return
}
if got, want := updated.State, "running"; got != want {
t.Errorf("Want proc name %s, got %s", want, got)
}
}
func TestProcIndexes(t *testing.T) {
db := openTest()
defer db.Close()
s := From(db)
if err := s.ProcCreate([]*model.Proc{
{
BuildID: 1,
PID: 1,
PPID: 1,
PGID: 1,
State: "running",
Name: "build",
},
}); err != nil {
t.Errorf("Unexpected error: insert procs: %s", err)
return
}
// fail due to duplicate pid
if err := s.ProcCreate([]*model.Proc{
{
BuildID: 1,
PID: 1,
PPID: 1,
PGID: 1,
State: "success",
Name: "clone",
},
}); err == nil {
t.Errorf("Unexpected error: dupliate pid")
}
// fail due to duplicate process name
if err := s.ProcCreate([]*model.Proc{
{
BuildID: 1,
PID: 2,
PPID: 1,
PGID: 1,
State: "success",
Name: "build",
},
}); err == nil {
t.Errorf("Unexpected error: dupliate name")
}
}
// func TestProcCascade(t *testing.T) {
//
// }

View file

@ -0,0 +1,21 @@
package sql
import (
"github.com/drone/drone/store/datastore/sql/sqlite"
)
// Supported database drivers
const (
DriverSqlite = "sqlite"
DriverMysql = "mysql"
DriverPostgres = "postgres"
)
// Lookup returns the named sql statement compatible with
// the specified database driver.
func Lookup(driver string, name string) string {
switch driver {
default:
return sqlite.Lookup(name)
}
}

View file

@ -0,0 +1,41 @@
-- name: files-find-build
SELECT
file_id
,file_build_id
,file_proc_id
,file_name
,file_mime
,file_size
,file_time
FROM files
WHERE file_build_id = ?
-- name: files-find-proc-name
SELECT
file_id
,file_build_id
,file_proc_id
,file_name
,file_mime
,file_size
,file_time
FROM files
WHERE file_proc_id = ?
AND file_name = ?
-- name: files-find-proc-name-data
SELECT
file_id
,file_build_id
,file_proc_id
,file_name
,file_mime
,file_size
,file_time
,file_data
FROM files
WHERE file_proc_id = ?
AND file_name = ?

View file

@ -0,0 +1,62 @@
-- name: procs-find-build
SELECT
proc_id
,proc_build_id
,proc_pid
,proc_ppid
,proc_pgid
,proc_name
,proc_state
,proc_error
,proc_exit_code
,proc_started
,proc_stopped
,proc_machine
,proc_platform
,proc_environ
FROM procs
WHERE proc_build_id = ?
-- name: procs-find-build-pid
SELECT
proc_id
,proc_build_id
,proc_pid
,proc_ppid
,proc_pgid
,proc_name
,proc_state
,proc_error
,proc_exit_code
,proc_started
,proc_stopped
,proc_machine
,proc_platform
,proc_environ
FROM procs
WHERE proc_build_id = ?
AND proc_pid = ?
-- name: procs-find-build-ppid
SELECT
proc_id
,proc_build_id
,proc_pid
,proc_ppid
,proc_pgid
,proc_name
,proc_state
,proc_error
,proc_exit_code
,proc_started
,proc_stopped
,proc_machine
,proc_platform
,proc_environ
FROM procs
WHERE proc_build_id = ?
AND proc_ppid = ?
AND proc_name = ?

View file

@ -0,0 +1,3 @@
package sqlite
//go:generate sqlbin sql --package=sqlite

View file

@ -0,0 +1,120 @@
package sqlite
// Lookup returns the named statement.
func Lookup(name string) string {
return index[name]
}
var index = map[string]string{
"files-find-build": filesFindBuild,
"files-find-proc-name": filesFindProcName,
"files-find-proc-name-data": filesFindProcNameData,
"procs-find-build": procsFindBuild,
"procs-find-build-pid": procsFindBuildPid,
"procs-find-build-ppid": procsFindBuildPpid,
}
var filesFindBuild = `
SELECT
file_id
,file_build_id
,file_proc_id
,file_name
,file_mime
,file_size
,file_time
FROM files
WHERE file_build_id = ?
`
var filesFindProcName = `
SELECT
file_id
,file_build_id
,file_proc_id
,file_name
,file_mime
,file_size
,file_time
FROM files
WHERE file_proc_id = ?
AND file_name = ?
`
var filesFindProcNameData = `
SELECT
file_id
,file_build_id
,file_proc_id
,file_name
,file_mime
,file_size
,file_time
,file_data
FROM files
WHERE file_proc_id = ?
AND file_name = ?
`
var procsFindBuild = `
SELECT
proc_id
,proc_build_id
,proc_pid
,proc_ppid
,proc_pgid
,proc_name
,proc_state
,proc_error
,proc_exit_code
,proc_started
,proc_stopped
,proc_machine
,proc_platform
,proc_environ
FROM procs
WHERE proc_build_id = ?
`
var procsFindBuildPid = `
SELECT
proc_id
,proc_build_id
,proc_pid
,proc_ppid
,proc_pgid
,proc_name
,proc_state
,proc_error
,proc_exit_code
,proc_started
,proc_stopped
,proc_machine
,proc_platform
,proc_environ
FROM procs
WHERE proc_build_id = ?
AND proc_pid = ?
`
var procsFindBuildPpid = `
SELECT
proc_id
,proc_build_id
,proc_pid
,proc_ppid
,proc_pgid
,proc_name
,proc_state
,proc_error
,proc_exit_code
,proc_started
,proc_stopped
,proc_machine
,proc_platform
,proc_environ
FROM procs
WHERE proc_build_id = ?
AND proc_ppid = ?
AND proc_name = ?
`

View file

@ -144,6 +144,17 @@ type Store interface {
UpdateAgent(*model.Agent) error
DeleteAgent(*model.Agent) error
ProcFind(*model.Build, int) (*model.Proc, error)
ProcChild(*model.Build, int, string) (*model.Proc, error)
ProcList(*model.Build) ([]*model.Proc, error)
ProcCreate([]*model.Proc) error
ProcUpdate(*model.Proc) error
FileList(*model.Build) ([]*model.File, error)
FileFind(*model.Proc, string) (*model.File, error)
FileRead(*model.Proc, string) (io.ReadCloser, error)
FileCreate(*model.File, io.Reader) error
}
const globalTeamName = "__global__"