From 2f579e4b7d2fe48263f8d2b39f3edc4537abbe84 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Wed, 3 May 2017 23:25:33 +0200 Subject: [PATCH] setup store in main() --- .gitignore | 1 + drone/server/server.go | 57 +++++++++++++++- drone/server/setup.go | 24 +++++++ router/middleware/store.go | 68 +------------------ store/datastore/repos.go | 9 +-- store/datastore/sql/postgres/files/counts.sql | 14 ++++ store/datastore/sql/postgres/sql_gen.go | 18 +++++ store/datastore/sql/sqlite/files/counts.sql | 14 ++++ store/datastore/sql/sqlite/sql_gen.go | 18 +++++ store/datastore/users.go | 9 +-- 10 files changed, 156 insertions(+), 76 deletions(-) create mode 100644 drone/server/setup.go create mode 100644 store/datastore/sql/postgres/files/counts.sql create mode 100644 store/datastore/sql/sqlite/files/counts.sql diff --git a/.gitignore b/.gitignore index f48fd3263..e722eade2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ drone/drone *.sqlite *_gen.go +*_ee.go !store/datastore/sql/sqlite/sql_gen.go !store/datastore/sql/mysql/sql_gen.go !store/datastore/sql/postgres/sql_gen.go diff --git a/drone/server/server.go b/drone/server/server.go index dd69fef11..398e60936 100644 --- a/drone/server/server.go +++ b/drone/server/server.go @@ -1,6 +1,7 @@ package server import ( + "context" "net/http" "net/url" "time" @@ -8,8 +9,17 @@ import ( "golang.org/x/crypto/acme/autocert" "golang.org/x/sync/errgroup" + "github.com/cncd/logging" + "github.com/cncd/pubsub" + "github.com/cncd/queue" + "github.com/drone/drone/model" + "github.com/drone/drone/plugins/registry" + "github.com/drone/drone/plugins/secrets" + "github.com/drone/drone/plugins/sender" "github.com/drone/drone/router" "github.com/drone/drone/router/middleware" + droneserver "github.com/drone/drone/server" + "github.com/drone/drone/store" "github.com/Sirupsen/logrus" "github.com/gin-gonic/contrib/ginrus" @@ -325,13 +335,16 @@ func server(c *cli.Context) error { logrus.SetLevel(logrus.WarnLevel) } + s := setupStore(c) + setupEvilGlobals(c, s) + // setup the server and start the listener handler := router.Load( ginrus.Ginrus(logrus.StandardLogger(), time.RFC3339, true), middleware.Version, middleware.Config(c), middleware.Cache(c), - middleware.Store(c), + middleware.Store(c, s), middleware.Remote(c), ) @@ -370,3 +383,45 @@ func server(c *cli.Context) error { return g.Wait() } + +// HACK please excuse the message during this period of heavy refactoring. +// We are currently transitioning from storing services (ie database, queue) +// in the gin.Context to storing them in a struct. We are also moving away +// from gin to gorilla. We will temporarily use global during our refactoring +// which will be removing in the final implementation. +func setupEvilGlobals(c *cli.Context, v store.Store) { + + // storage + droneserver.Config.Storage.Files = v + + // services + droneserver.Config.Services.Queue = model.WithTaskStore(queue.New(), v) + droneserver.Config.Services.Logs = logging.New() + droneserver.Config.Services.Pubsub = pubsub.New() + droneserver.Config.Services.Pubsub.Create(context.Background(), "topic/events") + droneserver.Config.Services.Registries = registry.New(v) + droneserver.Config.Services.Secrets = secrets.New(v) + droneserver.Config.Services.Senders = sender.New(v) + if endpoint := c.String("registry-service"); endpoint != "" { + droneserver.Config.Services.Registries = registry.NewRemote(endpoint) + } + if endpoint := c.String("secret-service"); endpoint != "" { + droneserver.Config.Services.Secrets = secrets.NewRemote(endpoint) + } + if endpoint := c.String("gating-service"); endpoint != "" { + droneserver.Config.Services.Senders = sender.NewRemote(endpoint) + } + + // server configuration + droneserver.Config.Server.Cert = c.String("server-cert") + droneserver.Config.Server.Key = c.String("server-key") + droneserver.Config.Server.Pass = c.String("agent-secret") + droneserver.Config.Server.Host = c.String("server-host") + droneserver.Config.Server.Port = c.String("server-addr") + droneserver.Config.Pipeline.Networks = c.StringSlice("network") + droneserver.Config.Pipeline.Volumes = c.StringSlice("volumes") + droneserver.Config.Pipeline.Privileged = c.StringSlice("escalate") + // droneserver.Config.Server.Open = cli.Bool("open") + // droneserver.Config.Server.Orgs = sliceToMap(cli.StringSlice("orgs")) + // droneserver.Config.Server.Admins = sliceToMap(cli.StringSlice("admin")) +} diff --git a/drone/server/setup.go b/drone/server/setup.go new file mode 100644 index 000000000..d35c84b22 --- /dev/null +++ b/drone/server/setup.go @@ -0,0 +1,24 @@ +// +build !enterprise + +package server + +import ( + "github.com/drone/drone/store" + "github.com/drone/drone/store/datastore" + + "github.com/urfave/cli" +) + +func setupStore(c *cli.Context) store.Store { + return datastore.New( + c.String("driver"), + c.String("datasource"), + ) +} + +func setupQueue(c *cli.Context) {} +func setupPubsub(c *cli.Context) {} +func setupStream(c *cli.Command) {} +func setupRegistryService(c *cli.Command) {} +func setupSecretService(c *cli.Command) {} +func setupGatingService(c *cli.Command) {} diff --git a/router/middleware/store.go b/router/middleware/store.go index 50ff877ba..09e52d387 100644 --- a/router/middleware/store.go +++ b/router/middleware/store.go @@ -1,18 +1,7 @@ package middleware import ( - "context" - - "github.com/cncd/logging" - "github.com/cncd/pubsub" - "github.com/cncd/queue" - "github.com/drone/drone/model" - "github.com/drone/drone/plugins/registry" - "github.com/drone/drone/plugins/secrets" - "github.com/drone/drone/plugins/sender" - "github.com/drone/drone/server" "github.com/drone/drone/store" - "github.com/drone/drone/store/datastore" "github.com/urfave/cli" "github.com/gin-gonic/gin" @@ -20,64 +9,9 @@ import ( // Store is a middleware function that initializes the Datastore and attaches to // the context of every http.Request. -func Store(cli *cli.Context) gin.HandlerFunc { - v := setupStore(cli) - - // HACK during refactor period. Please ignore my mess. - - // storage - server.Config.Storage.Files = v - - // services - server.Config.Services.Queue = model.WithTaskStore(queue.New(), v) - server.Config.Services.Logs = logging.New() - server.Config.Services.Pubsub = pubsub.New() - server.Config.Services.Pubsub.Create(context.Background(), "topic/events") - server.Config.Services.Registries = registry.New(v) - server.Config.Services.Secrets = secrets.New(v) - server.Config.Services.Senders = sender.New(v) - if endpoint := cli.String("registry-service"); endpoint != "" { - server.Config.Services.Registries = registry.NewRemote(endpoint) - } - if endpoint := cli.String("secret-service"); endpoint != "" { - server.Config.Services.Secrets = secrets.NewRemote(endpoint) - } - if endpoint := cli.String("gating-service"); endpoint != "" { - server.Config.Services.Senders = sender.NewRemote(endpoint) - } - - // server configuration - server.Config.Server.Cert = cli.String("server-cert") - server.Config.Server.Key = cli.String("server-key") - server.Config.Server.Pass = cli.String("agent-secret") - server.Config.Server.Host = cli.String("server-host") - server.Config.Server.Port = cli.String("server-addr") - server.Config.Pipeline.Networks = cli.StringSlice("network") - server.Config.Pipeline.Volumes = cli.StringSlice("volumes") - server.Config.Pipeline.Privileged = cli.StringSlice("escalate") - // server.Config.Server.Open = cli.Bool("open") - // server.Config.Server.Orgs = sliceToMap(cli.StringSlice("orgs")) - // server.Config.Server.Admins = sliceToMap(cli.StringSlice("admin")) - +func Store(cli *cli.Context, v store.Store) gin.HandlerFunc { return func(c *gin.Context) { store.ToContext(c, v) c.Next() } } - -// helper function to create the datastore from the CLI context. -func setupStore(c *cli.Context) store.Store { - return datastore.New( - c.String("driver"), - c.String("datasource"), - ) -} - -// helper function to convert a string slice to a map. -func sliceToMap(s []string) map[string]struct{} { - v := map[string]struct{}{} - for _, ss := range s { - v[ss] = struct{}{} - } - return v -} diff --git a/store/datastore/repos.go b/store/datastore/repos.go index c0e5c6490..9302da74d 100644 --- a/store/datastore/repos.go +++ b/store/datastore/repos.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/drone/drone/model" + "github.com/drone/drone/store/datastore/sql" "github.com/russross/meddler" ) @@ -38,10 +39,10 @@ func (db *datastore) GetRepoListOf(listof []*model.RepoLite) ([]*model.Repo, err return repos, err } -func (db *datastore) GetRepoCount() (int, error) { - var count int - var err = db.QueryRow(rebind(repoCountQuery)).Scan(&count) - return count, err +func (db *datastore) GetRepoCount() (count int, err error) { + stmt := sql.Lookup(db.driver, "count-repos") + err = meddler.QueryAll(db, &count, stmt) + return } func (db *datastore) CreateRepo(repo *model.Repo) error { diff --git a/store/datastore/sql/postgres/files/counts.sql b/store/datastore/sql/postgres/files/counts.sql new file mode 100644 index 000000000..4adbd8730 --- /dev/null +++ b/store/datastore/sql/postgres/files/counts.sql @@ -0,0 +1,14 @@ +-- name: count-users + +SELECT reltuples +FROM pg_class WHERE relname = 'users'; + +-- name: count-repos + +SELECT reltuples +FROM pg_class WHERE relname = 'repos'; + +-- name: count-builds + +SELECT reltuples +FROM pg_class WHERE relname = 'builds'; diff --git a/store/datastore/sql/postgres/sql_gen.go b/store/datastore/sql/postgres/sql_gen.go index 7fc85becc..08be6c7da 100644 --- a/store/datastore/sql/postgres/sql_gen.go +++ b/store/datastore/sql/postgres/sql_gen.go @@ -6,6 +6,9 @@ func Lookup(name string) string { } var index = map[string]string{ + "count-users": countUsers, + "count-repos": countRepos, + "count-builds": countBuilds, "files-find-build": filesFindBuild, "files-find-proc-name": filesFindProcName, "files-find-proc-name-data": filesFindProcNameData, @@ -30,6 +33,21 @@ var index = map[string]string{ "task-delete": taskDelete, } +var countUsers = ` +SELECT reltuples +FROM pg_class WHERE relname = 'users'; +` + +var countRepos = ` +SELECT reltuples +FROM pg_class WHERE relname = 'repos'; +` + +var countBuilds = ` +SELECT reltuples +FROM pg_class WHERE relname = 'builds'; +` + var filesFindBuild = ` SELECT file_id diff --git a/store/datastore/sql/sqlite/files/counts.sql b/store/datastore/sql/sqlite/files/counts.sql new file mode 100644 index 000000000..23b73935b --- /dev/null +++ b/store/datastore/sql/sqlite/files/counts.sql @@ -0,0 +1,14 @@ +-- name: count-users + +SELECT count(1) +FROM users + +-- name: count-repos + +SELECT count(1) +FROM repos + +-- name: count-builds + +SELECT count(1) +FROM builds diff --git a/store/datastore/sql/sqlite/sql_gen.go b/store/datastore/sql/sqlite/sql_gen.go index 2ed2de0a5..c2e082dd8 100644 --- a/store/datastore/sql/sqlite/sql_gen.go +++ b/store/datastore/sql/sqlite/sql_gen.go @@ -6,6 +6,9 @@ func Lookup(name string) string { } var index = map[string]string{ + "count-users": countUsers, + "count-repos": countRepos, + "count-builds": countBuilds, "files-find-build": filesFindBuild, "files-find-proc-name": filesFindProcName, "files-find-proc-name-data": filesFindProcNameData, @@ -30,6 +33,21 @@ var index = map[string]string{ "task-delete": taskDelete, } +var countUsers = ` +SELECT count(1) +FROM users +` + +var countRepos = ` +SELECT count(1) +FROM repos +` + +var countBuilds = ` +SELECT count(1) +FROM builds +` + var filesFindBuild = ` SELECT file_id diff --git a/store/datastore/users.go b/store/datastore/users.go index 0fe8d668f..520da0dbd 100644 --- a/store/datastore/users.go +++ b/store/datastore/users.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/drone/drone/model" + "github.com/drone/drone/store/datastore/sql" "github.com/russross/meddler" ) @@ -65,10 +66,10 @@ func (db *datastore) GetUserFeedLatest(listof []*model.RepoLite) ([]*model.Feed, return feed, err } -func (db *datastore) GetUserCount() (int, error) { - var count int - var err = db.QueryRow(rebind(userCountQuery)).Scan(&count) - return count, err +func (db *datastore) GetUserCount() (count int, err error) { + stmt := sql.Lookup(db.driver, "count-users") + err = meddler.QueryAll(db, &count, stmt) + return } func (db *datastore) CreateUser(user *model.User) error {