mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-11-22 18:01:02 +00:00
Code Issues Projects Releases Wiki Activity

Fetch repositories with remote ID if possible (#1078)

Browse source 
Use IDs of the forge to fetch repositories instead of their names and owner names. This improves handling of renamed and transferred repos.

TODO

- [ ] try to support as many forges as possible
    - [x] Gogs (no API)
    - [ ] Bitbucket Server
    - [x] Coding (no API?)
- [x] update repo every time it is fetched or received from the forge
- [x] if repo remote IDs are not available, use owner / name to get it
- [x] handle redirections (redirect a renamed repo to its new path)
- [x] ~~pull all repos once during migration to update ID (?)~~ issue fixed by on-demand loading of remote IDs
- [x] handle redirections in web UI
- [ ] improve handling of hooks after a repo was renamed (currently it checks for a redirection to the repo)
- [x] tests
- [x] `UNIQUE` constraint for remote IDs after migration shouldn't work (all repos have an empty string as remote ID)

close #854
close #648 partial
close https://codeberg.org/Codeberg-CI/feedback/issues/46

Possible follow-up PRs
- apply the same scheme on everything fetched from the remote (currently only users)

Co-authored-by: 6543 <6543@obermui.de>
This commit is contained in:
qwerty287 2022-09-05 17:08:51 +02:00 committed by GitHub
parent 3b0263442a
commit 52d3652f2e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
51 changed files with 668 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
Makefile
View file

@ -57,8 +57,8 @@ vendor:
go mod tidy
go mod vendor
format:
@gofmt -s -w ${GOFILES_NOVENDOR}
format: install-tools
@gofumpt -extra -w ${GOFILES_NOVENDOR}
.PHONY: docs
docs:
@ -140,6 +140,9 @@ install-tools:
fi ; \
hash lint > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
go install github.com/rs/zerolog/cmd/lint@latest; \
fi ; \
hash gofumpt > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
go install mvdan.cc/gofumpt@latest; \
fi
cross-compile-server:

35
server/api/hook.go
View file

@ -69,8 +69,9 @@ func BlockTilQueueHasRunningItem(c *gin.Context) {
// PostHook start a pipeline triggered by a forges post webhook
func PostHook(c *gin.Context) {
_store := store.FromContext(c)
remote := server.Config.Services.Remote
tmpRepo, tmpBuild, err := server.Config.Services.Remote.Hook(c, c.Request)
tmpRepo, tmpBuild, err := remote.Hook(c, c.Request)
if err != nil {
msg := "failure to parse hook"
log.Debug().Err(err).Msg(msg)
@ -100,7 +101,7 @@ func PostHook(c *gin.Context) {
return
}
repo, err := _store.GetRepoName(tmpRepo.Owner + "/" + tmpRepo.Name)
repo, err := _store.GetRepoNameFallback(tmpRepo.RemoteID, tmpRepo.FullName)
if err != nil {
msg := fmt.Sprintf("failure to get repo %s from store", tmpRepo.FullName)
log.Error().Err(err).Msg(msg)
@ -114,6 +115,23 @@ func PostHook(c *gin.Context) {
return
}
oldFullName := repo.FullName
if oldFullName != tmpRepo.FullName {
// create a redirection
err = _store.CreateRedirection(&model.Redirection{RepoID: repo.ID, FullName: repo.FullName})
if err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
}
repo.Update(tmpRepo)
err = _store.UpdateRepo(repo)
if err != nil {
c.String(http.StatusInternalServerError, err.Error())
return
}
// get the token and verify the hook is authorized
parsed, err := token.ParseRequest(c.Request, func(_ *token.Token) (string, error) {
return repo.Hash, nil
@ -124,7 +142,18 @@ func PostHook(c *gin.Context) {
c.String(http.StatusBadRequest, msg)
return
}
if parsed.Text != repo.FullName {
verifiedKey := parsed.Text == oldFullName
if !verifiedKey {
verifiedKey, err = _store.HasRedirectionForRepo(repo.ID, repo.FullName)
if err != nil {
msg := "failure to verify token from hook. Could not check for redirections of the repo"
log.Error().Err(err).Msg(msg)
c.String(http.StatusInternalServerError, msg)
return
}
}
if !verifiedKey {
msg := fmt.Sprintf("failure to verify token from hook. Expected %s, got %s", repo.FullName, parsed.Text)
log.Debug().Msg(msg)
c.String(http.StatusForbidden, msg)

72
server/api/repo.go
View file

@ -86,17 +86,25 @@ func PostRepo(c *gin.Context) {
sig,
)
from, err := remote.Repo(c, user, repo.RemoteID, repo.Owner, repo.Name)
if err == nil {
if repo.FullName != from.FullName {
// create a redirection
err = _store.CreateRedirection(&model.Redirection{RepoID: repo.ID, FullName: repo.FullName})
if err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
}
repo.Update(from)
}
err = remote.Activate(c, user, repo, link)
if err != nil {
c.String(http.StatusInternalServerError, err.Error())
return
}
from, err := remote.Repo(c, user, repo.Owner, repo.Name)
if err == nil {
repo.Update(from)
}
err = _store.UpdateRepo(repo)
if err != nil {
c.String(http.StatusInternalServerError, err.Error())
@ -251,35 +259,36 @@ func RepairRepo(c *gin.Context) {
sig,
)
if err := remote.Deactivate(c, user, repo, host); err != nil {
log.Trace().Err(err).Msgf("deactivate repo '%s' to repair failed", repo.FullName)
}
if err := remote.Activate(c, user, repo, link); err != nil {
c.String(500, err.Error())
return
}
from, err := remote.Repo(c, user, repo.Owner, repo.Name)
from, err := remote.Repo(c, user, repo.RemoteID, repo.Owner, repo.Name)
if err != nil {
log.Error().Err(err).Msgf("get repo '%s/%s' from remote", repo.Owner, repo.Name)
c.AbortWithStatus(http.StatusInternalServerError)
return
}
repo.Name = from.Name
repo.Owner = from.Owner
repo.FullName = from.FullName
repo.Avatar = from.Avatar
repo.Link = from.Link
repo.Clone = from.Clone
repo.IsSCMPrivate = from.IsSCMPrivate
if repo.IsSCMPrivate != from.IsSCMPrivate {
repo.ResetVisibility()
if repo.FullName != from.FullName {
// create a redirection
err = _store.CreateRedirection(&model.Redirection{RepoID: repo.ID, FullName: repo.FullName})
if err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
}
repo.Update(from)
if err := _store.UpdateRepo(repo); err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
if err := remote.Deactivate(c, user, repo, host); err != nil {
log.Trace().Err(err).Msgf("deactivate repo '%s' to repair failed", repo.FullName)
}
if err := remote.Activate(c, user, repo, link); err != nil {
c.String(500, err.Error())
return
}
c.Writer.WriteHeader(http.StatusOK)
}
@ -302,7 +311,7 @@ func MoveRepo(c *gin.Context) {
return
}
from, err := remote.Repo(c, user, owner, name)
from, err := remote.Repo(c, user, "", owner, name)
if err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
@ -312,17 +321,14 @@ func MoveRepo(c *gin.Context) {
return
}
repo.Name = from.Name
repo.Owner = from.Owner
repo.FullName = from.FullName
repo.Avatar = from.Avatar
repo.Link = from.Link
repo.Clone = from.Clone
repo.IsSCMPrivate = from.IsSCMPrivate
if repo.IsSCMPrivate != from.IsSCMPrivate {
repo.ResetVisibility()
err = _store.CreateRedirection(&model.Redirection{RepoID: repo.ID, FullName: repo.FullName})
if err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
repo.Update(from)
errStore := _store.UpdateRepo(repo)
if errStore != nil {
_ = c.AbortWithError(http.StatusInternalServerError, errStore)

18
server/model/perm.go
View file

@ -25,15 +25,15 @@ type PermStore interface {
// Perm defines a repository permission for an individual user.
type Perm struct {
UserID int64 `json:"-" xorm:"UNIQUE(s) INDEX NOT NULL 'perm_user_id'"`
RepoID int64 `json:"-" xorm:"UNIQUE(s) INDEX NOT NULL 'perm_repo_id'"`
Repo string `json:"-" xorm:"-"` // TODO: better caching (use type *Repo)
Pull bool `json:"pull" xorm:"perm_pull"`
Push bool `json:"push" xorm:"perm_push"`
Admin bool `json:"admin" xorm:"perm_admin"`
Synced int64 `json:"synced" xorm:"perm_synced"`
Created int64 `json:"created" xorm:"created"`
Updated int64 `json:"updated" xorm:"updated"`
UserID int64 `json:"-" xorm:"UNIQUE(s) INDEX NOT NULL 'perm_user_id'"`
RepoID int64 `json:"-" xorm:"UNIQUE(s) INDEX NOT NULL 'perm_repo_id'"`
Repo *Repo `json:"-" xorm:"-"`
Pull bool `json:"pull" xorm:"perm_pull"`
Push bool `json:"push" xorm:"perm_push"`
Admin bool `json:"admin" xorm:"perm_admin"`
Synced int64 `json:"synced" xorm:"perm_synced"`
Created int64 `json:"created" xorm:"created"`
Updated int64 `json:"updated" xorm:"updated"`
}
// TableName return database table name for xorm

11
server/model/redirection.go Normal file
View file

@ -0,0 +1,11 @@
package model
type Redirection struct {
ID int64 `xorm:"pk autoincr 'redirection_id'"`
RepoID int64 `xorm:"'repo_id'"`
FullName string `xorm:"UNIQUE INDEX 'repo_full_name'"`
}
func (r Redirection) TableName() string {
return "redirections"
}

11
server/model/repo.go
View file

@ -26,6 +26,7 @@ import (
type Repo struct {
ID int64 `json:"id,omitempty" xorm:"pk autoincr 'repo_id'"`
UserID int64 `json:"-" xorm:"repo_user_id"`
RemoteID RemoteID `json:"-" xorm:"'remote_id'"`
Owner string `json:"owner" xorm:"UNIQUE(name) 'repo_owner'"`
Name string `json:"name" xorm:"UNIQUE(name) 'repo_name'"`
FullName string `json:"full_name" xorm:"UNIQUE 'repo_full_name'"`
@ -74,6 +75,10 @@ func ParseRepo(str string) (user, repo string, err error) {
// Update updates the repository with values from the given Repo.
func (r *Repo) Update(from *Repo) {
r.RemoteID = from.RemoteID
r.Owner = from.Owner
r.Name = from.Name
r.FullName = from.FullName
r.Avatar = from.Avatar
r.Link = from.Link
r.SCMKind = from.SCMKind
@ -99,3 +104,9 @@ type RepoPatch struct {
AllowPull *bool `json:"allow_pr,omitempty"`
CancelPreviousPipelineEvents *[]WebhookEvent `json:"cancel_previous_pipeline_events"`
}
type RemoteID string
func (r RemoteID) IsValid() bool {
return r != "" && r != "0"
}

5
server/remote/bitbucket/bitbucket.go
View file

@ -142,7 +142,10 @@ func (c *config) Teams(ctx context.Context, u *model.User) ([]*model.Team, error
}
// Repo returns the named Bitbucket repository.
func (c *config) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) {
func (c *config) Repo(ctx context.Context, u *model.User, id model.RemoteID, owner, name string) (*model.Repo, error) {
if id.IsValid() {
name = string(id)
}
repo, err := c.newClient(ctx, u).FindRepo(owner, name)
if err != nil {
return nil, err

4
server/remote/bitbucket/bitbucket_test.go
View file

@ -126,12 +126,12 @@ func Test_bitbucket(t *testing.T) {
g.Describe("When requesting a repository", func() {
g.It("Should return the details", func() {
repo, err := c.Repo(ctx, fakeUser, fakeRepo.Owner, fakeRepo.Name)
repo, err := c.Repo(ctx, fakeUser, "", fakeRepo.Owner, fakeRepo.Name)
g.Assert(err).IsNil()
g.Assert(repo.FullName).Equal(fakeRepo.FullName)
})
g.It("Should handle not found errors", func() {
_, err := c.Repo(ctx, fakeUser, fakeRepoNotFound.Owner, fakeRepoNotFound.Name)
_, err := c.Repo(ctx, fakeUser, "", fakeRepoNotFound.Owner, fakeRepoNotFound.Name)
g.Assert(err).IsNotNil()
})
})

1
server/remote/bitbucket/convert.go
View file

@ -49,6 +49,7 @@ func convertStatus(status model.StatusValue) string {
// structure to the common Woodpecker repository structure.
func convertRepo(from *internal.Repo) *model.Repo {
repo := model.Repo{
RemoteID: model.RemoteID(from.UUID),
Clone: cloneLink(from),
Owner: strings.Split(from.FullName, "/")[0],
Name: strings.Split(from.FullName, "/")[1],

1
server/remote/bitbucket/internal/types.go
View file

@ -89,6 +89,7 @@ type LinkClone struct {
}
type Repo struct {
UUID string `json:"uuid"`
Owner Account `json:"owner"`
Name string `json:"name"`
FullName string `json:"full_name"`

2
server/remote/bitbucketserver/bitbucketserver.go
View file

@ -151,7 +151,7 @@ func (*Config) TeamPerm(u *model.User, org string) (*model.Perm, error) {
return nil, nil
}
func (c *Config) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) {
func (c *Config) Repo(ctx context.Context, u *model.User, _ model.RemoteID, owner, name string) (*model.Repo, error) {
repo, err := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token).FindRepo(owner, name)
if err != nil {
return nil, err

1
server/remote/bitbucketserver/convert.go
View file

@ -51,6 +51,7 @@ func convertStatus(status model.StatusValue) string {
// structure to the common Woodpecker repository structure.
func convertRepo(from *internal.Repo) *model.Repo {
repo := model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(from.ID)),
Name: from.Slug,
Owner: from.Project.Key,
Branch: "master",

1
server/remote/bitbucketserver/parse.go
View file

@ -32,6 +32,7 @@ func parseHook(r *http.Request, baseURL string) (*model.Repo, *model.Build, erro
}
build := convertPushHook(hook, baseURL)
repo := &model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(hook.Repository.ID)),
Name: hook.Repository.Slug,
Owner: hook.Repository.Project.Key,
FullName: fmt.Sprintf("%s/%s", hook.Repository.Project.Key, hook.Repository.Slug),

6
server/remote/coding/coding.go
View file

@ -160,8 +160,8 @@ func (c *Coding) TeamPerm(u *model.User, org string) (*model.Perm, error) {
return nil, nil
}
// Repo fetches the named repository from the remote system.
func (c *Coding) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) {
// Repo fetches the repository from the remote system.
func (c *Coding) Repo(ctx context.Context, u *model.User, _ model.RemoteID, owner, name string) (*model.Repo, error) {
client := c.newClient(ctx, u)
project, err := client.GetProject(owner, name)
if err != nil {
@ -172,6 +172,7 @@ func (c *Coding) Repo(ctx context.Context, u *model.User, owner, name string) (*
return nil, err
}
return &model.Repo{
// TODO(1138) RemoteID: project.ID,
Owner: project.Owner,
Name: project.Name,
FullName: projectFullName(project.Owner, project.Name),
@ -199,6 +200,7 @@ func (c *Coding) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error
return nil, err
}
repo := &model.Repo{
// TODO(1138) RemoteID: project.ID,
Owner: project.Owner,
Name: project.Name,
FullName: projectFullName(project.Owner, project.Name),

4
server/remote/coding/coding_test.go
View file

@ -108,7 +108,7 @@ func Test_coding(t *testing.T) {
g.Describe("When requesting a repository", func() {
g.It("Should return the details", func() {
repo, err := c.Repo(ctx, fakeUser, fakeRepo.Owner, fakeRepo.Name)
repo, err := c.Repo(ctx, fakeUser, "", fakeRepo.Owner, fakeRepo.Name)
g.Assert(err).IsNil()
g.Assert(repo.FullName).Equal(fakeRepo.FullName)
g.Assert(repo.Avatar).Equal(s.URL + fakeRepo.Avatar)
@ -119,7 +119,7 @@ func Test_coding(t *testing.T) {
g.Assert(repo.IsSCMPrivate).Equal(fakeRepo.IsSCMPrivate)
})
g.It("Should handle not found errors", func() {
_, err := c.Repo(ctx, fakeUser, fakeRepoNotFound.Owner, fakeRepoNotFound.Name)
_, err := c.Repo(ctx, fakeUser, "", fakeRepoNotFound.Owner, fakeRepoNotFound.Name)
g.Assert(err).IsNotNil()
})
})

1
server/remote/coding/hook.go
View file

@ -137,6 +137,7 @@ func convertRepository(repo *Repository) (*model.Repo, error) {
}
return &model.Repo{
// TODO RemoteID: repo.ID,
Owner: matches[1],
Name: repo.Name,
FullName: projectFullName(repo.Owner.GlobalKey, repo.Name),

4
server/remote/coding/internal/coding.go
View file

@ -49,12 +49,12 @@ func NewClient(ctx context.Context, baseURL, apiPath, token, agent string, clien
}
}
// Generic GET for requesting Coding OAuth API
// Get Generic GET for requesting Coding OAuth API
func (c *Client) Get(u string, params url.Values) ([]byte, error) {
return c.Do(http.MethodGet, u, params)
}
// Generic method for requesting Coding OAuth API
// Do Generic method for requesting Coding OAuth API
func (c *Client) Do(method, u string, params url.Values) ([]byte, error) {
if params == nil {
params = url.Values{}

12
server/remote/gitea/fixtures/handler.go
View file

@ -27,6 +27,7 @@ func Handler() http.Handler {
e := gin.New()
e.GET("/api/v1/repos/:owner/:name", getRepo)
e.GET("/api/v1/repositories/:id", getRepoByID)
e.GET("/api/v1/repos/:owner/:name/raw/:commit/:file", getRepoFile)
e.POST("/api/v1/repos/:owner/:name/hooks", createRepoHook)
e.GET("/api/v1/repos/:owner/:name/hooks", listRepoHooks)
@ -51,6 +52,15 @@ func getRepo(c *gin.Context) {
}
}
func getRepoByID(c *gin.Context) {
switch c.Param("id") {
case "repo_not_found":
c.String(404, "")
default:
c.String(200, repoPayload)
}
}
func createRepoCommitStatus(c *gin.Context) {
if c.Param("commit") == "v1.0.0" || c.Param("commit") == "9ecad50" {
c.String(200, repoPayload)
@ -124,6 +134,7 @@ const listRepoHookPayloads = `
const repoPayload = `
{
"id": 5,
"owner": {
"login": "test_name",
"email": "octocat@github.com",
@ -146,6 +157,7 @@ const repoFilePayload = `{ platform: linux/amd64 }`
const userRepoPayload = `
[
{
"id": 5,
"owner": {
"login": "test_name",
"email": "octocat@github.com",

17
server/remote/gitea/gitea.go
View file

@ -26,6 +26,7 @@ import (
"net/url"
"path"
"path/filepath"
"strconv"
"strings"
"time"
@ -225,13 +226,25 @@ func (c *Gitea) TeamPerm(u *model.User, org string) (*model.Perm, error) {
return nil, nil
}
// Repo returns the named Gitea repository.
func (c *Gitea) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) {
// Repo returns the Gitea repository.
func (c *Gitea) Repo(ctx context.Context, u *model.User, id model.RemoteID, owner, name string) (*model.Repo, error) {
client, err := c.newClientToken(ctx, u.Token)
if err != nil {
return nil, err
}
if id.IsValid() {
intID, err := strconv.ParseInt(string(id), 10, 64)
if err != nil {
return nil, err
}
repo, _, err := client.GetRepoByID(intID)
if err != nil {
return nil, err
}
return toRepo(repo), nil
}
repo, _, err := client.GetRepo(owner, name)
if err != nil {
return nil, err

6
server/remote/gitea/gitea_test.go
View file

@ -76,7 +76,7 @@ func Test_gitea(t *testing.T) {
g.Describe("Requesting a repository", func() {
g.It("Should return the repository details", func() {
repo, err := c.Repo(ctx, fakeUser, fakeRepo.Owner, fakeRepo.Name)
repo, err := c.Repo(ctx, fakeUser, fakeRepo.RemoteID, fakeRepo.Owner, fakeRepo.Name)
g.Assert(err).IsNil()
g.Assert(repo.Owner).Equal(fakeRepo.Owner)
g.Assert(repo.Name).Equal(fakeRepo.Name)
@ -86,7 +86,7 @@ func Test_gitea(t *testing.T) {
g.Assert(repo.Link).Equal("http://localhost/test_name/repo_name")
})
g.It("Should handle a not found error", func() {
_, err := c.Repo(ctx, fakeUser, fakeRepoNotFound.Owner, fakeRepoNotFound.Name)
_, err := c.Repo(ctx, fakeUser, "0", fakeRepoNotFound.Owner, fakeRepoNotFound.Name)
g.Assert(err).IsNotNil()
})
})
@ -109,6 +109,7 @@ func Test_gitea(t *testing.T) {
g.It("Should return the repository list", func() {
repos, err := c.Repos(ctx, fakeUser)
g.Assert(err).IsNil()
g.Assert(repos[0].RemoteID).Equal(fakeRepo.RemoteID)
g.Assert(repos[0].Owner).Equal(fakeRepo.Owner)
g.Assert(repos[0].Name).Equal(fakeRepo.Name)
g.Assert(repos[0].FullName).Equal(fakeRepo.Owner + "/" + fakeRepo.Name)
@ -168,6 +169,7 @@ var (
fakeRepo = &model.Repo{
Clone: "http://gitea.com/test_name/repo_name.git",
RemoteID: "5",
Owner: "test_name",
Name: "repo_name",
FullName: "test_name/repo_name",

3
server/remote/gitea/helper.go
View file

@ -36,6 +36,7 @@ func toRepo(from *gitea.Repository) *model.Repo {
from.Owner.AvatarURL,
)
return &model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(from.ID)),
SCMKind: model.RepoGit,
Name: name,
Owner: from.Owner.UserName,
@ -185,6 +186,7 @@ func buildFromPullRequest(hook *pullRequestHook) *model.Build {
// helper function that extracts the Repository data from a Gitea push hook
func repoFromPush(hook *pushHook) *model.Repo {
return &model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(hook.Repo.ID)),
Name: hook.Repo.Name,
Owner: hook.Repo.Owner.Username,
FullName: hook.Repo.FullName,
@ -195,6 +197,7 @@ func repoFromPush(hook *pushHook) *model.Repo {
// helper function that extracts the Repository data from a Gitea pull_request hook
func repoFromPullRequest(hook *pullRequestHook) *model.Repo {
return &model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(hook.Repo.ID)),
Name: hook.Repo.Name,
Owner: hook.Repo.Owner.Username,
FullName: hook.Repo.FullName,

4
server/remote/github/convert.go
View file

@ -15,6 +15,8 @@
package github
import (
"fmt"
"github.com/google/go-github/v39/github"
"github.com/woodpecker-ci/woodpecker/server/model"
@ -82,6 +84,7 @@ func convertDesc(status model.StatusValue) string {
// structure to the common Woodpecker repository structure.
func convertRepo(from *github.Repository) *model.Repo {
repo := &model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(from.GetID())),
Name: from.GetName(),
FullName: from.GetFullName(),
Link: from.GetHTMLURL(),
@ -142,6 +145,7 @@ func convertTeam(from *github.Organization) *model.Team {
// from a webhook and convert to the common Woodpecker repository structure.
func convertRepoHook(eventRepo *github.PushEventRepository) *model.Repo {
repo := &model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(eventRepo.GetID())),
Owner: eventRepo.GetOwner().GetLogin(),
Name: eventRepo.GetName(),
FullName: eventRepo.GetFullName(),

11
server/remote/github/fixtures/handler.go
View file

@ -27,6 +27,7 @@ func Handler() http.Handler {
e := gin.New()
e.GET("/api/v3/repos/:owner/:name", getRepo)
e.GET("/api/v3/repositories/:id", getRepoByID)
e.GET("/api/v3/orgs/:org/memberships/:user", getMembership)
e.GET("/api/v3/user/memberships/orgs/:org", getMembership)
@ -42,6 +43,15 @@ func getRepo(c *gin.Context) {
}
}
func getRepoByID(c *gin.Context) {
switch c.Param("id") {
case "repo_not_found":
c.String(404, "")
default:
c.String(200, repoPayload)
}
}
func getMembership(c *gin.Context) {
switch c.Param("org") {
case "org_not_found":
@ -55,6 +65,7 @@ func getMembership(c *gin.Context) {
var repoPayload = `
{
"id": 5,
"owner": {
"login": "octocat",
"avatar_url": "https://github.com/images/error/octocat_happy.gif"

19
server/remote/github/github.go
View file

@ -163,9 +163,22 @@ func (c *client) Teams(ctx context.Context, u *model.User) ([]*model.Team, error
return teams, nil
}
// Repo returns the named GitHub repository.
func (c *client) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) {
// Repo returns the GitHub repository.
func (c *client) Repo(ctx context.Context, u *model.User, id model.RemoteID, owner, name string) (*model.Repo, error) {
client := c.newClientToken(ctx, u.Token)
if id.IsValid() {
intID, err := strconv.ParseInt(string(id), 10, 64)
if err != nil {
return nil, err
}
repo, _, err := client.Repositories.GetByID(ctx, intID)
if err != nil {
return nil, err
}
return convertRepo(repo), nil
}
repo, _, err := client.Repositories.Get(ctx, owner, name)
if err != nil {
return nil, err
@ -531,7 +544,7 @@ func (c *client) loadChangedFilesFromPullRequest(ctx context.Context, pull *gith
return build, nil
}
repo, err := _store.GetRepoName(tmpRepo.Owner + "/" + tmpRepo.Name)
repo, err := _store.GetRepoNameFallback(tmpRepo.RemoteID, tmpRepo.FullName)
if err != nil {
return nil, err
}

6
server/remote/github/github_test.go
View file

@ -77,8 +77,9 @@ func Test_github(t *testing.T) {
g.Describe("Requesting a repository", func() {
g.It("Should return the repository details", func() {
repo, err := c.Repo(ctx, fakeUser, fakeRepo.Owner, fakeRepo.Name)
repo, err := c.Repo(ctx, fakeUser, fakeRepo.RemoteID, fakeRepo.Owner, fakeRepo.Name)
g.Assert(err).IsNil()
g.Assert(repo.RemoteID).Equal(fakeRepo.RemoteID)
g.Assert(repo.Owner).Equal(fakeRepo.Owner)
g.Assert(repo.Name).Equal(fakeRepo.Name)
g.Assert(repo.FullName).Equal(fakeRepo.FullName)
@ -87,7 +88,7 @@ func Test_github(t *testing.T) {
g.Assert(repo.Link).Equal(fakeRepo.Link)
})
g.It("Should handle a not found error", func() {
_, err := c.Repo(ctx, fakeUser, fakeRepoNotFound.Owner, fakeRepoNotFound.Name)
_, err := c.Repo(ctx, fakeUser, "0", fakeRepoNotFound.Owner, fakeRepoNotFound.Name)
g.Assert(err).IsNotNil()
})
})
@ -131,6 +132,7 @@ var (
}
fakeRepo = &model.Repo{
RemoteID: "5",
Owner: "octocat",
Name: "Hello-World",
FullName: "octocat/Hello-World",

4
server/remote/gitlab/convert.go
View file

@ -37,6 +37,7 @@ func (g *Gitlab) convertGitlabRepo(_repo *gitlab.Project) (*model.Repo, error) {
owner := strings.Join(parts[:len(parts)-1], "/")
name := parts[len(parts)-1]
repo := &model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(_repo.ID)),
Owner: owner,
Name: name,
FullName: _repo.PathWithNamespace,
@ -87,6 +88,7 @@ func convertMergeRequestHook(hook *gitlab.MergeEvent, req *http.Request) (int, *
repo.FullName = fmt.Sprintf("%s/%s", repo.Owner, repo.Name)
}
repo.RemoteID = model.RemoteID(fmt.Sprint(obj.TargetProjectID))
repo.Link = target.WebURL
if target.GitHTTPURL != "" {
@ -140,6 +142,7 @@ func convertPushHook(hook *gitlab.PushEvent) (*model.Repo, *model.Build, error)
return nil, nil, err
}
repo.RemoteID = model.RemoteID(fmt.Sprint(hook.ProjectID))
repo.Avatar = hook.Project.AvatarURL
repo.Link = hook.Project.WebURL
repo.Clone = hook.Project.GitHTTPURL
@ -191,6 +194,7 @@ func convertTagHook(hook *gitlab.TagEvent) (*model.Repo, *model.Build, error) {
return nil, nil, err
}
repo.RemoteID = model.RemoteID(fmt.Sprint(hook.ProjectID))
repo.Avatar = hook.Project.AvatarURL
repo.Link = hook.Project.WebURL
repo.Clone = hook.Project.GitHTTPURL

19
server/remote/gitlab/gitlab.go
View file

@ -21,6 +21,7 @@ import (
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
@ -225,13 +226,25 @@ func (g *Gitlab) getProject(ctx context.Context, client *gitlab.Client, owner, n
return repo, nil
}
// Repo fetches the named repository from the remote system.
func (g *Gitlab) Repo(ctx context.Context, user *model.User, owner, name string) (*model.Repo, error) {
// Repo fetches the repository from the remote system.
func (g *Gitlab) Repo(ctx context.Context, user *model.User, id model.RemoteID, owner, name string) (*model.Repo, error) {
client, err := newClient(g.URL, user.Token, g.SkipVerify)
if err != nil {
return nil, err
}
if id.IsValid() {
intID, err := strconv.ParseInt(string(id), 10, 64)
if err != nil {
return nil, err
}
_repo, _, err := client.Projects.GetProject(int(intID), nil, gitlab.WithContext(ctx))
if err != nil {
return nil, err
}
return g.convertGitlabRepo(_repo)
}
_repo, err := g.getProject(ctx, client, owner, name)
if err != nil {
return nil, err
@ -667,7 +680,7 @@ func (g *Gitlab) loadChangedFilesFromMergeRequest(ctx context.Context, tmpRepo *
return build, nil
}
repo, err := _store.GetRepoName(tmpRepo.Owner + "/" + tmpRepo.Name)
repo, err := _store.GetRepoNameFallback(tmpRepo.RemoteID, tmpRepo.FullName)
if err != nil {
return nil, err
}

4
server/remote/gitlab/gitlab_test.go
View file

@ -93,7 +93,7 @@ func Test_Gitlab(t *testing.T) {
// Test repository method
g.Describe("Repo", func() {
g.It("Should return valid repo", func() {
_repo, err := client.Repo(ctx, &user, "diaspora", "diaspora-client")
_repo, err := client.Repo(ctx, &user, "0", "diaspora", "diaspora-client")
assert.NoError(t, err)
assert.Equal(t, "diaspora-client", _repo.Name)
assert.Equal(t, "diaspora", _repo.Owner)
@ -101,7 +101,7 @@ func Test_Gitlab(t *testing.T) {
})
g.It("Should return error, when repo not exist", func() {
_, err := client.Repo(ctx, &user, "not-existed", "not-existed")
_, err := client.Repo(ctx, &user, "0", "not-existed", "not-existed")
assert.Error(t, err)
})
})

2
server/remote/gogs/fixtures/handler.go
View file

@ -83,6 +83,7 @@ func getUserRepos(c *gin.Context) {
const repoPayload = `
{
"id": 5,
"owner": {
"username": "test_name",
"email": "octocat@github.com",
@ -105,6 +106,7 @@ const repoFilePayload = `{ platform: linux/amd64 }`
const userRepoPayload = `
[
{
"id": 5,
"owner": {
"username": "test_name",
"email": "octocat@github.com",

4
server/remote/gogs/gogs.go
View file

@ -148,7 +148,7 @@ func (c *client) Teams(ctx context.Context, u *model.User) ([]*model.Team, error
}
// Repo returns the named Gogs repository.
func (c *client) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) {
func (c *client) Repo(ctx context.Context, u *model.User, _ model.RemoteID, owner, name string) (*model.Repo, error) {
client := c.newClientToken(u.Token)
repo, err := client.GetRepo(owner, name)
if err != nil {
@ -160,7 +160,7 @@ func (c *client) Repo(ctx context.Context, u *model.User, owner, name string) (*
// Repos returns a list of all repositories for the Gogs account, including
// organization repositories.
func (c *client) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error) {
repos := []*model.Repo{}
var repos []*model.Repo
client := c.newClientToken(u.Token)
all, err := client.ListMyRepos()

6
server/remote/gogs/gogs_test.go
View file

@ -85,7 +85,7 @@ func Test_gogs(t *testing.T) {
g.Describe("Requesting a repository", func() {
g.It("Should return the repository details", func() {
repo, err := c.Repo(ctx, fakeUser, fakeRepo.Owner, fakeRepo.Name)
repo, err := c.Repo(ctx, fakeUser, fakeRepo.RemoteID, fakeRepo.Owner, fakeRepo.Name)
g.Assert(err).IsNil()
g.Assert(repo.Owner).Equal(fakeRepo.Owner)
g.Assert(repo.Name).Equal(fakeRepo.Name)
@ -95,7 +95,7 @@ func Test_gogs(t *testing.T) {
g.Assert(repo.Link).Equal("http://localhost/test_name/repo_name")
})
g.It("Should handle a not found error", func() {
_, err := c.Repo(ctx, fakeUser, fakeRepoNotFound.Owner, fakeRepoNotFound.Name)
_, err := c.Repo(ctx, fakeUser, "0", fakeRepoNotFound.Owner, fakeRepoNotFound.Name)
g.Assert(err).IsNotNil()
})
})
@ -118,6 +118,7 @@ func Test_gogs(t *testing.T) {
g.It("Should return the repository list", func() {
repos, err := c.Repos(ctx, fakeUser)
g.Assert(err).IsNil()
g.Assert(repos[0].RemoteID).Equal(fakeRepo.RemoteID)
g.Assert(repos[0].Owner).Equal(fakeRepo.Owner)
g.Assert(repos[0].Name).Equal(fakeRepo.Name)
g.Assert(repos[0].FullName).Equal(fakeRepo.Owner + "/" + fakeRepo.Name)
@ -181,6 +182,7 @@ var (
}
fakeRepo = &model.Repo{
RemoteID: "5",
Clone: "http://gogs.com/test_name/repo_name.git",
Owner: "test_name",
Name: "repo_name",

3
server/remote/gogs/helper.go
View file

@ -39,6 +39,7 @@ func toRepo(from *gogs.Repository, privateMode bool) *model.Repo {
private = true
}
return &model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(from.ID)),
SCMKind: model.RepoGit,
Name: name,
Owner: from.Owner.UserName,
@ -159,6 +160,7 @@ func buildFromPullRequest(hook *pullRequestHook) *model.Build {
// helper function that extracts the Repository data from a Gogs push hook
func repoFromPush(hook *pushHook) *model.Repo {
return &model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(hook.Repo.ID)),
Name: hook.Repo.Name,
Owner: hook.Repo.Owner.Username,
FullName: hook.Repo.FullName,
@ -169,6 +171,7 @@ func repoFromPush(hook *pushHook) *model.Repo {
// helper function that extracts the Repository data from a Gogs pull_request hook
func repoFromPullRequest(hook *pullRequestHook) *model.Repo {
return &model.Repo{
RemoteID: model.RemoteID(fmt.Sprint(hook.Repo.ID)),
Name: hook.Repo.Name,
Owner: hook.Repo.Owner.Username,
FullName: hook.Repo.FullName,

14
server/remote/mocks/remote.go
View file

@ -295,13 +295,13 @@ func (_m *Remote) Perm(ctx context.Context, u *model.User, r *model.Repo) (*mode
return r0, r1
}
// Repo provides a mock function with given fields: ctx, u, owner, name
func (_m *Remote) Repo(ctx context.Context, u *model.User, owner string, name string) (*model.Repo, error) {
ret := _m.Called(ctx, u, owner, name)
// Repo provides a mock function with given fields: ctx, u, id, owner, name
func (_m *Remote) Repo(ctx context.Context, u *model.User, id model.RemoteID, owner string, name string) (*model.Repo, error) {
ret := _m.Called(ctx, u, id, owner, name)
var r0 *model.Repo
if rf, ok := ret.Get(0).(func(context.Context, *model.User, string, string) *model.Repo); ok {
r0 = rf(ctx, u, owner, name)
if rf, ok := ret.Get(0).(func(context.Context, *model.User, model.RemoteID, string, string) *model.Repo); ok {
r0 = rf(ctx, u, id, owner, name)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.Repo)
@ -309,8 +309,8 @@ func (_m *Remote) Repo(ctx context.Context, u *model.User, owner string, name st
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *model.User, string, string) error); ok {
r1 = rf(ctx, u, owner, name)
if rf, ok := ret.Get(1).(func(context.Context, *model.User, model.RemoteID, string, string) error); ok {
r1 = rf(ctx, u, id, owner, name)
} else {
r1 = ret.Error(1)
}

4
server/remote/remote.go
View file

@ -42,8 +42,8 @@ type Remote interface {
// Teams fetches a list of team memberships from the remote system.
Teams(ctx context.Context, u *model.User) ([]*model.Team, error)
// Repo fetches the named repository from the remote system.
Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error)
// Repo fetches the repository from the remote system, preferred is using the ID, fallback is owner/name.
Repo(ctx context.Context, u *model.User, id model.RemoteID, owner, name string) (*model.Repo, error)
// Repos fetches a list of repos from the remote system.
Repos(ctx context.Context, u *model.User) ([]*model.Repo, error)

2
server/router/middleware/session/repo.go
View file

@ -100,7 +100,7 @@ func SetPerm() gin.HandlerFunc {
perm, err = server.Config.Services.Remote.Perm(c, user, repo)
if err == nil {
log.Debug().Msgf("Synced user permission for %s %s", user.Login, repo.FullName)
perm.Repo = repo.FullName
perm.Repo = repo
perm.UserID = user.ID
perm.Synced = time.Now().Unix()
if err := _store.PermUpsert(perm); err != nil {

2
server/shared/userSyncer.go
View file

@ -74,7 +74,7 @@ func (s *Syncer) Sync(ctx context.Context, user *model.User, flatPermissions boo
repo.Perm = &model.Perm{
UserID: user.ID,
RepoID: repo.ID,
Repo: repo.FullName,
Repo: repo,
Synced: unix,
}

7
server/store/datastore/feed_test.go
View file

@ -37,18 +37,21 @@ func TestRepoListLatest(t *testing.T) {
Owner: "bradrydzewski",
Name: "test",
FullName: "bradrydzewski/test",
RemoteID: "1",
IsActive: true,
}
repo2 := &model.Repo{
Owner: "test",
Name: "test",
FullName: "test/test",
RemoteID: "2",
IsActive: true,
}
repo3 := &model.Repo{
Owner: "octocat",
Name: "hello-world",
FullName: "octocat/hello-world",
RemoteID: "3",
IsActive: true,
}
assert.NoError(t, store.CreateRepo(repo1))
@ -56,8 +59,8 @@ func TestRepoListLatest(t *testing.T) {
assert.NoError(t, store.CreateRepo(repo3))
for _, perm := range []*model.Perm{
{UserID: user.ID, Repo: repo1.FullName, Push: true, Admin: false},
{UserID: user.ID, Repo: repo2.FullName, Push: true, Admin: true},
{UserID: user.ID, Repo: repo1, Push: true, Admin: false},
{UserID: user.ID, Repo: repo2, Push: true, Admin: true},
} {
assert.NoError(t, store.PermUpsert(perm))
}

1
server/store/datastore/migration/migration.go
View file

@ -53,6 +53,7 @@ var allBeans = []interface{}{
new(model.User),
new(model.ServerConfig),
new(model.Cron),
new(model.Redirection),
}
type migrations struct {

8
server/store/datastore/permission.go
View file

@ -45,13 +45,13 @@ func (s storage) PermUpsert(perm *model.Perm) error {
}
func (s storage) permUpsert(sess *xorm.Session, perm *model.Perm) error {
if perm.RepoID == 0 && len(perm.Repo) == 0 {
if perm.RepoID == 0 && perm.Repo == nil {
return fmt.Errorf("could not determine repo for permission: %v", perm)
}
// lookup repo based on name if possible
if perm.RepoID == 0 && len(perm.Repo) != 0 {
r, err := s.getRepoName(sess, perm.Repo)
// lookup repo based on name or remote ID if possible
if perm.RepoID == 0 && perm.Repo != nil {
r, err := s.getRepoNameFallback(sess, perm.Repo.RemoteID, perm.Repo.FullName)
if err != nil {
return err
}

11
server/store/datastore/permission_test.go
View file

@ -32,6 +32,7 @@ func TestPermFind(t *testing.T) {
FullName: "bradrydzewski/test",
Owner: "bradrydzewski",
Name: "test",
RemoteID: "1",
}
assert.NoError(t, store.CreateRepo(repo))
@ -39,7 +40,7 @@ func TestPermFind(t *testing.T) {
&model.Perm{
UserID: user.ID,
RepoID: repo.ID,
Repo: repo.FullName,
Repo: repo,
Pull: true,
Push: false,
Admin: false,
@ -76,6 +77,7 @@ func TestPermUpsert(t *testing.T) {
FullName: "bradrydzewski/test",
Owner: "bradrydzewski",
Name: "test",
RemoteID: "1",
}
assert.NoError(t, store.CreateRepo(repo))
@ -83,7 +85,7 @@ func TestPermUpsert(t *testing.T) {
&model.Perm{
UserID: user.ID,
RepoID: repo.ID,
Repo: repo.FullName,
Repo: repo,
Pull: true,
Push: false,
Admin: false,
@ -118,7 +120,7 @@ func TestPermUpsert(t *testing.T) {
&model.Perm{
UserID: user.ID,
RepoID: repo.ID,
Repo: repo.FullName,
Repo: repo,
Pull: true,
Push: true,
Admin: true,
@ -155,6 +157,7 @@ func TestPermDelete(t *testing.T) {
FullName: "bradrydzewski/test",
Owner: "bradrydzewski",
Name: "test",
RemoteID: "1",
}
assert.NoError(t, store.CreateRepo(repo))
@ -162,7 +165,7 @@ func TestPermDelete(t *testing.T) {
&model.Perm{
UserID: user.ID,
RepoID: repo.ID,
Repo: repo.FullName,
Repo: repo,
Pull: true,
Push: false,
Admin: false,

47
server/store/datastore/redirection.go Normal file
View file

@ -0,0 +1,47 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datastore
import (
"github.com/woodpecker-ci/woodpecker/server/model"
"xorm.io/xorm"
)
func (s storage) GetRedirection(fullName string) (*model.Redirection, error) {
sess := s.engine.NewSession()
defer sess.Close()
return s.getRedirection(sess, fullName)
}
func (s storage) getRedirection(e *xorm.Session, fullName string) (*model.Redirection, error) {
repo := new(model.Redirection)
return repo, wrapGet(e.Where("repo_full_name = ?", fullName).Get(repo))
}
func (s storage) CreateRedirection(redirect *model.Redirection) error {
sess := s.engine.NewSession()
defer sess.Close()
return s.createRedirection(sess, redirect)
}
func (s storage) createRedirection(e *xorm.Session, redirect *model.Redirection) error {
// only Insert set auto created ID back to object
_, err := e.Insert(redirect)
return err
}
func (s storage) HasRedirectionForRepo(repoID int64, fullName string) (bool, error) {
return s.engine.Where("repo_id = ? ", repoID).And("repo_full_name = ?", fullName).Get(new(model.Redirection))
}

84
server/store/datastore/redirection_test.go Normal file
View file

@ -0,0 +1,84 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datastore
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/woodpecker-ci/woodpecker/server/model"
)
func TestGetRedirection(t *testing.T) {
store, closer := newTestStore(t, new(model.Redirection))
defer closer()
redirection := &model.Redirection{
RepoID: 1,
FullName: "foo/bar",
}
if err := store.CreateRedirection(redirection); err != nil {
t.Errorf("Unexpected error: insert redirection: %s", err)
return
}
redirectionFromStore, err := store.GetRedirection("foo/bar")
assert.NoError(t, err)
assert.NotNil(t, redirectionFromStore)
assert.Equal(t, redirection.RepoID, redirectionFromStore.RepoID)
_, err = store.GetRedirection("foo/baz")
assert.ErrorIs(t, err, RecordNotExist)
}
func TestCreateRedirection(t *testing.T) {
store, closer := newTestStore(t, new(model.Redirection))
defer closer()
redirection := &model.Redirection{
RepoID: 1,
FullName: "foo/bar",
}
assert.NoError(t, store.CreateRedirection(redirection))
}
func TestHasRedirectionForRepo(t *testing.T) {
store, closer := newTestStore(t, new(model.Redirection))
defer closer()
redirection := &model.Redirection{
RepoID: 1,
FullName: "foo/bar",
}
if err := store.CreateRedirection(redirection); err != nil {
t.Errorf("Unexpected error: insert redirection: %s", err)
return
}
has, err := store.HasRedirectionForRepo(1, "foo/bar")
if err != nil {
t.Error(err)
return
}
if !has {
t.Errorf("Expected a redirection for %s", redirection.FullName)
}
has, err = store.HasRedirectionForRepo(1, "foo/baz")
if err != nil {
t.Error(err)
return
}
if has {
t.Errorf("Expected not finding a redirection for %s", redirection.FullName)
}
}

77
server/store/datastore/repo.go
View file

@ -27,10 +27,44 @@ func (s storage) GetRepo(id int64) (*model.Repo, error) {
return repo, wrapGet(s.engine.ID(id).Get(repo))
}
func (s storage) GetRepoRemoteID(id model.RemoteID) (*model.Repo, error) {
sess := s.engine.NewSession()
defer sess.Close()
return s.getRepoRemoteID(sess, id)
}
func (s storage) getRepoRemoteID(e *xorm.Session, id model.RemoteID) (*model.Repo, error) {
repo := new(model.Repo)
return repo, wrapGet(e.Where("remote_id = ?", id).Get(repo))
}
func (s storage) GetRepoNameFallback(remoteID model.RemoteID, fullName string) (*model.Repo, error) {
sess := s.engine.NewSession()
defer sess.Close()
return s.getRepoNameFallback(sess, remoteID, fullName)
}
func (s storage) getRepoNameFallback(e *xorm.Session, remoteID model.RemoteID, fullName string) (*model.Repo, error) {
repo, err := s.getRepoRemoteID(e, remoteID)
if err == RecordNotExist {
return s.getRepoName(e, fullName)
}
return repo, err
}
func (s storage) GetRepoName(fullName string) (*model.Repo, error) {
sess := s.engine.NewSession()
defer sess.Close()
return s.getRepoName(sess, fullName)
repo, err := s.getRepoName(sess, fullName)
if err == RecordNotExist {
// the repository does not exist, so look for a redirection
redirect, err := s.getRedirection(sess, fullName)
if err != nil {
return nil, err
}
return s.GetRepo(redirect.RepoID)
}
return repo, err
}
func (s storage) getRepoName(e *xorm.Session, fullName string) (*model.Repo, error) {
@ -73,6 +107,9 @@ func (s storage) DeleteRepo(repo *model.Repo) error {
if _, err := sess.Where("secret_repo_id = ?", repo.ID).Delete(new(model.Secret)); err != nil {
return err
}
if _, err := sess.Where("repo_id = ?", repo.ID).Delete(new(model.Redirection)); err != nil {
return err
}
// delete related builds
for startBuilds := 0; ; startBuilds += batchSize {
@ -128,23 +165,39 @@ func (s storage) RepoBatch(repos []*model.Repo) error {
continue
}
exist, err := sess.
Where("repo_owner = ? AND repo_name = ?", repos[i].Owner, repos[i].Name).
Exist(new(model.Repo))
if err != nil {
repo, err := s.getRepoNameFallback(sess, repos[i].RemoteID, repos[i].FullName)
if err != nil && err != RecordNotExist {
return err
}
exist := err == nil // if there's an error, it must be a RecordNotExist
if exist {
if _, err := sess.
Where("repo_owner = ? AND repo_name = ?", repos[i].Owner, repos[i].Name).
Cols("repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch").
Update(repos[i]); err != nil {
return err
if repos[i].FullName != repo.FullName {
// create redirection
err := s.createRedirection(sess, &model.Redirection{RepoID: repo.ID, FullName: repo.FullName})
if err != nil {
return err
}
}
if repos[i].RemoteID.IsValid() {
if _, err := sess.
Where("remote_id = ?", repos[i].RemoteID).
Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "remote_id").
Update(repos[i]); err != nil {
return err
}
} else {
if _, err := sess.
Where("repo_owner = ?", repos[i].Owner).
And(" repo_name = ?", repos[i].Name).
Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "remote_id").
Update(repos[i]); err != nil {
return err
}
}
_, err := sess.
Where("repo_owner = ? AND repo_name = ?", repos[i].Owner, repos[i].Name).
Where("remote_id = ?", repos[i].RemoteID).
Get(repos[i])
if err != nil {
return err
@ -158,7 +211,7 @@ func (s storage) RepoBatch(repos []*model.Repo) error {
if repos[i].Perm != nil {
repos[i].Perm.RepoID = repos[i].ID
repos[i].Perm.Repo = repos[i].FullName
repos[i].Perm.Repo = repos[i]
if err := s.permUpsert(sess, repos[i].Perm); err != nil {
return err
}

73
server/store/datastore/repo_test.go
View file

@ -138,24 +138,27 @@ func TestRepoList(t *testing.T) {
Owner: "bradrydzewski",
Name: "test",
FullName: "bradrydzewski/test",
RemoteID: "1",
}
repo2 := &model.Repo{
Owner: "test",
Name: "test",
FullName: "test/test",
RemoteID: "2",
}
repo3 := &model.Repo{
Owner: "octocat",
Name: "hello-world",
FullName: "octocat/hello-world",
RemoteID: "3",
}
assert.NoError(t, store.CreateRepo(repo1))
assert.NoError(t, store.CreateRepo(repo2))
assert.NoError(t, store.CreateRepo(repo3))
for _, perm := range []*model.Perm{
{UserID: user.ID, Repo: repo1.FullName},
{UserID: user.ID, Repo: repo2.FullName},
{UserID: user.ID, Repo: repo1},
{UserID: user.ID, Repo: repo2},
} {
assert.NoError(t, store.PermUpsert(perm))
}
@ -191,21 +194,25 @@ func TestOwnedRepoList(t *testing.T) {
Owner: "bradrydzewski",
Name: "test",
FullName: "bradrydzewski/test",
RemoteID: "1",
}
repo2 := &model.Repo{
Owner: "test",
Name: "test",
FullName: "test/test",
RemoteID: "2",
}
repo3 := &model.Repo{
Owner: "octocat",
Name: "hello-world",
FullName: "octocat/hello-world",
RemoteID: "3",
}
repo4 := &model.Repo{
Owner: "demo",
Name: "demo",
FullName: "demo/demo",
RemoteID: "4",
}
assert.NoError(t, store.CreateRepo(repo1))
assert.NoError(t, store.CreateRepo(repo2))
@ -213,10 +220,10 @@ func TestOwnedRepoList(t *testing.T) {
assert.NoError(t, store.CreateRepo(repo4))
for _, perm := range []*model.Perm{
{UserID: user.ID, Repo: repo1.FullName, Push: true, Admin: false},
{UserID: user.ID, Repo: repo2.FullName, Push: false, Admin: true},
{UserID: user.ID, Repo: repo3.FullName},
{UserID: user.ID, Repo: repo4.FullName},
{UserID: user.ID, Repo: repo1, Push: true, Admin: false},
{UserID: user.ID, Repo: repo2, Push: false, Admin: true},
{UserID: user.ID, Repo: repo3},
{UserID: user.ID, Repo: repo4},
} {
assert.NoError(t, store.PermUpsert(perm))
}
@ -270,10 +277,11 @@ func TestRepoCount(t *testing.T) {
}
func TestRepoBatch(t *testing.T) {
store, closer := newTestStore(t, new(model.Repo), new(model.User), new(model.Perm))
store, closer := newTestStore(t, new(model.Repo), new(model.User), new(model.Perm), new(model.Redirection))
defer closer()
if !assert.NoError(t, store.CreateRepo(&model.Repo{
RemoteID: "5",
UserID: 1,
FullName: "foo/bar",
Owner: "foo",
@ -285,6 +293,7 @@ func TestRepoBatch(t *testing.T) {
repos := []*model.Repo{
{
RemoteID: "5",
UserID: 1,
FullName: "foo/bar",
Owner: "foo",
@ -299,6 +308,7 @@ func TestRepoBatch(t *testing.T) {
},
},
{
RemoteID: "6",
UserID: 1,
FullName: "bar/baz",
Owner: "bar",
@ -306,6 +316,7 @@ func TestRepoBatch(t *testing.T) {
IsActive: true,
},
{
RemoteID: "7",
UserID: 1,
FullName: "baz/qux",
Owner: "baz",
@ -313,6 +324,7 @@ func TestRepoBatch(t *testing.T) {
IsActive: true,
},
{
RemoteID: "8",
UserID: 0, // not activated repos do hot have a user id assigned
FullName: "baz/notes",
Owner: "baz",
@ -330,6 +342,7 @@ func TestRepoBatch(t *testing.T) {
assert.True(t, perm.Admin)
repo := &model.Repo{
RemoteID: "5",
FullName: "foo/bar",
Owner: "foo",
Name: "bar",
@ -373,7 +386,8 @@ func TestRepoCrud(t *testing.T) {
new(model.File),
new(model.Secret),
new(model.Registry),
new(model.Config))
new(model.Config),
new(model.Redirection))
defer closer()
repo := model.Repo{
@ -420,3 +434,46 @@ func TestRepoCrud(t *testing.T) {
assert.NoError(t, err)
assert.EqualValues(t, 1, buildCount)
}
func TestRepoRedirection(t *testing.T) {
store, closer := newTestStore(t,
new(model.Repo),
new(model.Redirection))
defer closer()
repo := model.Repo{
UserID: 1,
RemoteID: "1",
FullName: "bradrydzewski/test",
Owner: "bradrydzewski",
Name: "test",
}
assert.NoError(t, store.CreateRepo(&repo))
repoUpdated := model.Repo{
RemoteID: "1",
FullName: "bradrydzewski/test-renamed",
Owner: "bradrydzewski",
Name: "test-renamed",
}
assert.NoError(t, store.RepoBatch([]*model.Repo{&repoUpdated}))
// test redirection from old repo name
repoFromStore, err := store.GetRepoNameFallback("1", "bradrydzewski/test")
assert.NoError(t, err)
assert.Equal(t, repoFromStore.FullName, repoUpdated.FullName)
// test getting repo without remote ID (use name fallback)
repo = model.Repo{
UserID: 1,
FullName: "bradrydzewski/test-no-remote-id",
Owner: "bradrydzewski",
Name: "test-no-remote-id",
}
assert.NoError(t, store.CreateRepo(&repo))
repoFromStore, err = store.GetRepoNameFallback("", "bradrydzewski/test-no-remote-id")
assert.NoError(t, err)
assert.Equal(t, repoFromStore.FullName, repo.FullName)
}

7
server/store/datastore/users_test.go
View file

@ -195,26 +195,29 @@ func TestUsers(t *testing.T) {
Name: "test",
FullName: "bradrydzewski/test",
IsActive: true,
RemoteID: "1",
}
repo2 := &model.Repo{
Owner: "test",
Name: "test",
FullName: "test/test",
IsActive: true,
RemoteID: "2",
}
repo3 := &model.Repo{
Owner: "octocat",
Name: "hello-world",
FullName: "octocat/hello-world",
IsActive: true,
RemoteID: "3",
}
g.Assert(store.CreateRepo(repo1)).IsNil()
g.Assert(store.CreateRepo(repo2)).IsNil()
g.Assert(store.CreateRepo(repo3)).IsNil()
for _, perm := range []*model.Perm{
{UserID: user.ID, Repo: repo1.FullName, Push: true, Admin: false},
{UserID: user.ID, Repo: repo2.FullName, Push: false, Admin: true},
{UserID: user.ID, Repo: repo1, Push: true, Admin: false},
{UserID: user.ID, Repo: repo2, Push: false, Admin: true},
} {
g.Assert(store.PermUpsert(perm)).IsNil()
}

104
server/store/mocks/store.go
View file

@ -144,6 +144,20 @@ func (_m *Store) CreateBuild(_a0 *model.Build, _a1 ...*model.Proc) error {
return r0
}
// CreateRedirection provides a mock function with given fields: redirection
func (_m *Store) CreateRedirection(redirection *model.Redirection) error {
ret := _m.Called(redirection)
var r0 error
if rf, ok := ret.Get(0).(func(*model.Redirection) error); ok {
r0 = rf(redirection)
} else {
r0 = ret.Error(0)
}
return r0
}
// CreateRepo provides a mock function with given fields: _a0
func (_m *Store) CreateRepo(_a0 *model.Repo) error {
ret := _m.Called(_a0)
@ -643,6 +657,29 @@ func (_m *Store) GetBuildRef(_a0 *model.Repo, _a1 string) (*model.Build, error)
return r0, r1
}
// GetRedirection provides a mock function with given fields: _a0
func (_m *Store) GetRedirection(_a0 string) (*model.Redirection, error) {
ret := _m.Called(_a0)
var r0 *model.Redirection
if rf, ok := ret.Get(0).(func(string) *model.Redirection); ok {
r0 = rf(_a0)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.Redirection)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(_a0)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetRepo provides a mock function with given fields: _a0
func (_m *Store) GetRepo(_a0 int64) (*model.Repo, error) {
ret := _m.Called(_a0)
@ -710,6 +747,52 @@ func (_m *Store) GetRepoName(_a0 string) (*model.Repo, error) {
return r0, r1
}
// GetRepoNameFallback provides a mock function with given fields: remoteID, fullName
func (_m *Store) GetRepoNameFallback(remoteID model.RemoteID, fullName string) (*model.Repo, error) {
ret := _m.Called(remoteID, fullName)
var r0 *model.Repo
if rf, ok := ret.Get(0).(func(model.RemoteID, string) *model.Repo); ok {
r0 = rf(remoteID, fullName)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.Repo)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(model.RemoteID, string) error); ok {
r1 = rf(remoteID, fullName)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetRepoRemoteID provides a mock function with given fields: _a0
func (_m *Store) GetRepoRemoteID(_a0 model.RemoteID) (*model.Repo, error) {
ret := _m.Called(_a0)
var r0 *model.Repo
if rf, ok := ret.Get(0).(func(model.RemoteID) *model.Repo); ok {
r0 = rf(_a0)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.Repo)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(model.RemoteID) error); ok {
r1 = rf(_a0)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetUser provides a mock function with given fields: _a0
func (_m *Store) GetUser(_a0 int64) (*model.User, error) {
ret := _m.Called(_a0)
@ -846,6 +929,27 @@ func (_m *Store) GlobalSecretList() ([]*model.Secret, error) {
return r0, r1
}
// HasRedirectionForRepo provides a mock function with given fields: _a0, _a1
func (_m *Store) HasRedirectionForRepo(_a0 int64, _a1 string) (bool, error) {
ret := _m.Called(_a0, _a1)
var r0 bool
if rf, ok := ret.Get(0).(func(int64, string) bool); ok {
r0 = rf(_a0, _a1)
} else {
r0 = ret.Get(0).(bool)
}
var r1 error
if rf, ok := ret.Get(1).(func(int64, string) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// LogFind provides a mock function with given fields: _a0
func (_m *Store) LogFind(_a0 *model.Proc) (io.ReadCloser, error) {
ret := _m.Called(_a0)

14
server/store/store.go
View file

@ -46,6 +46,10 @@ type Store interface {
// Repos
// GetRepo gets a repo by unique ID.
GetRepo(int64) (*model.Repo, error)
// GetRepoRemoteID gets a repo by its remote ID.
GetRepoRemoteID(model.RemoteID) (*model.Repo, error)
// GetRepoNameFallback gets the repo by its remote ID and if this doesn't exist by its full name.
GetRepoNameFallback(remoteID model.RemoteID, fullName string) (*model.Repo, error)
// GetRepoName gets a repo by its full name.
GetRepoName(string) (*model.Repo, error)
// GetRepoCount gets a count of all repositories in the system.
@ -57,6 +61,14 @@ type Store interface {
// DeleteRepo deletes a user repository.
DeleteRepo(*model.Repo) error
// Redirections
// GetRedirection returns the redirection for the given full repo name
GetRedirection(string) (*model.Redirection, error)
// CreateRedirection creates a redirection
CreateRedirection(redirection *model.Redirection) error
// HasRedirectionForRepo checks if there's a redirection for the given repo and full name
HasRedirectionForRepo(int64, string) (bool, error)
// Builds
// GetBuild gets a build by unique ID.
GetBuild(int64) (*model.Build, error)
@ -87,7 +99,7 @@ type Store interface {
// Feeds
UserFeed(*model.User) ([]*model.Feed, error)
// Repositorys
// Repositories
// RepoList TODO: paginate
RepoList(user *model.User, owned bool) ([]*model.Repo, error)
RepoListLatest(*model.User) ([]*model.Feed, error)

14
server/swagger/doc.go
View file

@ -14,15 +14,15 @@
// Package classification Drone API.
//
// Schemes: http, https
// BasePath: /api
// Version: 1.0.0
// Schemes: http, https
// BasePath: /api
// Version: 1.0.0
//
// Consumes:
// - application/json
// Consumes:
// - application/json
//
// Produces:
// - application/json
// Produces:
// - application/json
//
// swagger:meta
package swagger

30
server/swagger/swagger.go
View file

@ -24,54 +24,48 @@ import (
//
// Get the user with the matching login.
//
// Responses:
// 200: user
//
// Responses:
// 200: user
func userFind(w http.ResponseWriter, r *http.Request) {}
// swagger:route GET /user user getCurrentUser
//
// Get the currently authenticated user.
//
// Responses:
// 200: user
//
// Responses:
// 200: user
func userCurrent(w http.ResponseWriter, r *http.Request) {}
// swagger:route GET /users user getUserList
//
// Get the list of all registered users.
//
// Responses:
// 200: user
//
// Responses:
// 200: user
func userList(w http.ResponseWriter, r *http.Request) {}
// swagger:route GET /user/feed user getUserFeed
//
// Get the currently authenticated user's build feed.
//
// Responses:
// 200: feed
//
// Responses:
// 200: feed
func userFeed(w http.ResponseWriter, r *http.Request) {}
// swagger:route DELETE /users/{login} user deleteUserLogin
//
// Delete the user with the matching login.
//
// Responses:
// 200: user
//
// Responses:
// 200: user
func userDelete(w http.ResponseWriter, r *http.Request) {}
// swagger:route GET /user/repos user getUserRepos
//
// Get the currently authenticated user's active repository list.
//
// Responses:
// 200: repos
//
// Responses:
// 200: repos
func repoList(w http.ResponseWriter, r *http.Request) {}
// swagger:response user

1
web/src/store/repos.ts
View file

@ -32,6 +32,7 @@ export default defineStore({
async loadRepo(owner: string, name: string) {
const repo = await apiClient.getRepo(owner, name);
this.repos[repoSlug(repo)] = repo;
return repo;
},
async loadRepos() {
const repos = await apiClient.getRepoList();

9
web/src/views/repo/RepoWrapper.vue
View file

@ -100,7 +100,14 @@ async function loadRepo() {
return;
}
await repoStore.loadRepo(repoOwner.value, repoName.value);
const apiRepo = await repoStore.loadRepo(repoOwner.value, repoName.value);
if (apiRepo.full_name !== `${repoOwner.value}/${repoName.value}`) {
await router.replace({
name: route.name ? route.name : 'repo',
params: { repoOwner: apiRepo.owner, repoName: apiRepo.name },
});
return;
}
await buildStore.loadBuilds(repoOwner.value, repoName.value);
}