mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-09-22 11:40:17 +00:00
Code Issues Projects Releases Wiki Activity

Do not store inactive repos (#1658)

Browse source 
Do not sync repos with forge if the repo is not necessary in DB.

In the DB, only repos that were active once or repos that are currently
active are stored. When trying to enable new repos, the repos list is
fetched from the forge instead and displayed directly. In addition to
this, the forge func `Perm` was removed and is now merged with `Repo`.

Solves a TODO on RepoBatch.

---------

Co-authored-by: Anbraten <anton@ju60.de>
This commit is contained in:
qwerty287 2023-03-21 23:01:59 +01:00 committed by GitHub
parent a95a5b43bf
commit 0970f35df5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
49 changed files with 329 additions and 730 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/server/flags.go
View file

@ -487,16 +487,6 @@ var flags = []cli.Flag{
Hidden: true,
},
//
// misc
//
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_FLAT_PERMISSIONS"},
Name: "flat-permissions",
Usage: "no forge call for permissions should be made",
Hidden: true,
// TODO(485) temporary workaround to not hit api rate limits
},
//
// secrets encryption in DB
//
&cli.StringFlag{

3
cmd/server/server.go
View file

@ -360,7 +360,4 @@ func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) {
// prometheus
server.Config.Prometheus.AuthToken = c.String("prometheus-auth-token")
// TODO(485) temporary workaround to not hit api rate limits
server.Config.FlatPermissions = c.Bool("flat-permissions")
}

83
pipeline/rpc/proto/woodpecker_grpc.pb.go
View file

@ -1,6 +1,21 @@
// Copyright 2021 Woodpecker Authors
// Copyright 2011 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.12
// source: woodpecker.proto
@ -18,6 +33,20 @@ import (
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
const (
Woodpecker_Version_FullMethodName = "/proto.Woodpecker/Version"
Woodpecker_Next_FullMethodName = "/proto.Woodpecker/Next"
Woodpecker_Init_FullMethodName = "/proto.Woodpecker/Init"
Woodpecker_Wait_FullMethodName = "/proto.Woodpecker/Wait"
Woodpecker_Done_FullMethodName = "/proto.Woodpecker/Done"
Woodpecker_Extend_FullMethodName = "/proto.Woodpecker/Extend"
Woodpecker_Update_FullMethodName = "/proto.Woodpecker/Update"
Woodpecker_Upload_FullMethodName = "/proto.Woodpecker/Upload"
Woodpecker_Log_FullMethodName = "/proto.Woodpecker/Log"
Woodpecker_RegisterAgent_FullMethodName = "/proto.Woodpecker/RegisterAgent"
Woodpecker_ReportHealth_FullMethodName = "/proto.Woodpecker/ReportHealth"
)
// WoodpeckerClient is the client API for Woodpecker service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
@ -45,7 +74,7 @@ func NewWoodpeckerClient(cc grpc.ClientConnInterface) WoodpeckerClient {
func (c *woodpeckerClient) Version(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*VersionResponse, error) {
out := new(VersionResponse)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/Version", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_Version_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -54,7 +83,7 @@ func (c *woodpeckerClient) Version(ctx context.Context, in *Empty, opts ...grpc.
func (c *woodpeckerClient) Next(ctx context.Context, in *NextRequest, opts ...grpc.CallOption) (*NextResponse, error) {
out := new(NextResponse)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/Next", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_Next_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -63,7 +92,7 @@ func (c *woodpeckerClient) Next(ctx context.Context, in *NextRequest, opts ...gr
func (c *woodpeckerClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/Init", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_Init_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -72,7 +101,7 @@ func (c *woodpeckerClient) Init(ctx context.Context, in *InitRequest, opts ...gr
func (c *woodpeckerClient) Wait(ctx context.Context, in *WaitRequest, opts ...grpc.CallOption) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/Wait", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_Wait_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -81,7 +110,7 @@ func (c *woodpeckerClient) Wait(ctx context.Context, in *WaitRequest, opts ...gr
func (c *woodpeckerClient) Done(ctx context.Context, in *DoneRequest, opts ...grpc.CallOption) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/Done", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_Done_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -90,7 +119,7 @@ func (c *woodpeckerClient) Done(ctx context.Context, in *DoneRequest, opts ...gr
func (c *woodpeckerClient) Extend(ctx context.Context, in *ExtendRequest, opts ...grpc.CallOption) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/Extend", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_Extend_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -99,7 +128,7 @@ func (c *woodpeckerClient) Extend(ctx context.Context, in *ExtendRequest, opts .
func (c *woodpeckerClient) Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/Update", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_Update_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -108,7 +137,7 @@ func (c *woodpeckerClient) Update(ctx context.Context, in *UpdateRequest, opts .
func (c *woodpeckerClient) Upload(ctx context.Context, in *UploadRequest, opts ...grpc.CallOption) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/Upload", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_Upload_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -117,7 +146,7 @@ func (c *woodpeckerClient) Upload(ctx context.Context, in *UploadRequest, opts .
func (c *woodpeckerClient) Log(ctx context.Context, in *LogRequest, opts ...grpc.CallOption) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/Log", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_Log_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -126,7 +155,7 @@ func (c *woodpeckerClient) Log(ctx context.Context, in *LogRequest, opts ...grpc
func (c *woodpeckerClient) RegisterAgent(ctx context.Context, in *RegisterAgentRequest, opts ...grpc.CallOption) (*RegisterAgentResponse, error) {
out := new(RegisterAgentResponse)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/RegisterAgent", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_RegisterAgent_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -135,7 +164,7 @@ func (c *woodpeckerClient) RegisterAgent(ctx context.Context, in *RegisterAgentR
func (c *woodpeckerClient) ReportHealth(ctx context.Context, in *ReportHealthRequest, opts ...grpc.CallOption) (*Empty, error) {
out := new(Empty)
err := c.cc.Invoke(ctx, "/proto.Woodpecker/ReportHealth", in, out, opts...)
err := c.cc.Invoke(ctx, Woodpecker_ReportHealth_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -220,7 +249,7 @@ func _Woodpecker_Version_Handler(srv interface{}, ctx context.Context, dec func(
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/Version",
FullMethod: Woodpecker_Version_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).Version(ctx, req.(*Empty))
@ -238,7 +267,7 @@ func _Woodpecker_Next_Handler(srv interface{}, ctx context.Context, dec func(int
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/Next",
FullMethod: Woodpecker_Next_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).Next(ctx, req.(*NextRequest))
@ -256,7 +285,7 @@ func _Woodpecker_Init_Handler(srv interface{}, ctx context.Context, dec func(int
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/Init",
FullMethod: Woodpecker_Init_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).Init(ctx, req.(*InitRequest))
@ -274,7 +303,7 @@ func _Woodpecker_Wait_Handler(srv interface{}, ctx context.Context, dec func(int
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/Wait",
FullMethod: Woodpecker_Wait_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).Wait(ctx, req.(*WaitRequest))
@ -292,7 +321,7 @@ func _Woodpecker_Done_Handler(srv interface{}, ctx context.Context, dec func(int
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/Done",
FullMethod: Woodpecker_Done_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).Done(ctx, req.(*DoneRequest))
@ -310,7 +339,7 @@ func _Woodpecker_Extend_Handler(srv interface{}, ctx context.Context, dec func(i
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/Extend",
FullMethod: Woodpecker_Extend_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).Extend(ctx, req.(*ExtendRequest))
@ -328,7 +357,7 @@ func _Woodpecker_Update_Handler(srv interface{}, ctx context.Context, dec func(i
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/Update",
FullMethod: Woodpecker_Update_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).Update(ctx, req.(*UpdateRequest))
@ -346,7 +375,7 @@ func _Woodpecker_Upload_Handler(srv interface{}, ctx context.Context, dec func(i
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/Upload",
FullMethod: Woodpecker_Upload_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).Upload(ctx, req.(*UploadRequest))
@ -364,7 +393,7 @@ func _Woodpecker_Log_Handler(srv interface{}, ctx context.Context, dec func(inte
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/Log",
FullMethod: Woodpecker_Log_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).Log(ctx, req.(*LogRequest))
@ -382,7 +411,7 @@ func _Woodpecker_RegisterAgent_Handler(srv interface{}, ctx context.Context, dec
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/RegisterAgent",
FullMethod: Woodpecker_RegisterAgent_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).RegisterAgent(ctx, req.(*RegisterAgentRequest))
@ -400,7 +429,7 @@ func _Woodpecker_ReportHealth_Handler(srv interface{}, ctx context.Context, dec
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.Woodpecker/ReportHealth",
FullMethod: Woodpecker_ReportHealth_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerServer).ReportHealth(ctx, req.(*ReportHealthRequest))
@ -464,6 +493,10 @@ var Woodpecker_ServiceDesc = grpc.ServiceDesc{
Metadata: "woodpecker.proto",
}
const (
WoodpeckerAuth_Auth_FullMethodName = "/proto.WoodpeckerAuth/Auth"
)
// WoodpeckerAuthClient is the client API for WoodpeckerAuth service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
@ -481,7 +514,7 @@ func NewWoodpeckerAuthClient(cc grpc.ClientConnInterface) WoodpeckerAuthClient {
func (c *woodpeckerAuthClient) Auth(ctx context.Context, in *AuthRequest, opts ...grpc.CallOption) (*AuthResponse, error) {
out := new(AuthResponse)
err := c.cc.Invoke(ctx, "/proto.WoodpeckerAuth/Auth", in, out, opts...)
err := c.cc.Invoke(ctx, WoodpeckerAuth_Auth_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -526,7 +559,7 @@ func _WoodpeckerAuth_Auth_Handler(srv interface{}, ctx context.Context, dec func
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/proto.WoodpeckerAuth/Auth",
FullMethod: WoodpeckerAuth_Auth_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(WoodpeckerAuthServer).Auth(ctx, req.(*AuthRequest))

78
server/api/repo.go
View file

@ -17,9 +17,11 @@ package api
import (
"encoding/base32"
"errors"
"fmt"
"net/http"
"strconv"
"time"
"github.com/gin-gonic/gin"
"github.com/gorilla/securecookie"
@ -29,6 +31,7 @@ import (
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/router/middleware/session"
"github.com/woodpecker-ci/woodpecker/server/store"
"github.com/woodpecker-ci/woodpecker/server/store/types"
"github.com/woodpecker-ci/woodpecker/shared/token"
)
@ -36,18 +39,38 @@ func PostRepo(c *gin.Context) {
forge := server.Config.Services.Forge
_store := store.FromContext(c)
user := session.User(c)
repo := session.Repo(c)
if repo.IsActive {
owner := c.Param("owner")
name := c.Param("name")
repo, err := _store.GetRepoName(owner + "/" + name)
enabledOnce := err == nil // if there's no error, the repo was found and enabled once already
if enabledOnce && repo.IsActive {
c.String(http.StatusConflict, "Repository is already active.")
return
} else if err != nil && !errors.Is(err, types.RecordNotExist) {
c.String(http.StatusInternalServerError, err.Error())
return
}
from, err := forge.Repo(c, user, "0", owner, name)
if err != nil {
c.String(http.StatusInternalServerError, "Could not fetch repository from forge.")
return
}
if !from.Perm.Admin {
c.String(http.StatusForbidden, "User has to be a admin of this repository")
}
if enabledOnce {
repo.Update(from)
} else {
repo = from
repo.AllowPull = true
repo.NetrcOnlyTrusted = true
repo.CancelPreviousPipelineEvents = server.Config.Pipeline.DefaultCancelPreviousPipelineEvents
}
repo.IsActive = true
repo.UserID = user.ID
repo.AllowPull = true
repo.NetrcOnlyTrusted = true
repo.CancelPreviousPipelineEvents = server.Config.Pipeline.DefaultCancelPreviousPipelineEvents
if repo.Visibility == "" {
repo.Visibility = model.VisibilityPublic
@ -82,26 +105,27 @@ func PostRepo(c *gin.Context) {
sig,
)
from, err := forge.Repo(c, user, repo.ForgeRemoteID, repo.Owner, repo.Name)
if err == nil {
if repo.FullName != from.FullName {
// create a redirection
err = _store.CreateRedirection(&model.Redirection{RepoID: repo.ID, FullName: repo.FullName})
if err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
}
repo.Update(from)
}
err = forge.Activate(c, user, repo, link)
if err != nil {
c.String(http.StatusInternalServerError, err.Error())
return
}
err = _store.UpdateRepo(repo)
if enabledOnce {
err = _store.UpdateRepo(repo)
} else {
err = _store.CreateRepo(repo)
}
if err != nil {
c.String(http.StatusInternalServerError, err.Error())
return
}
repo.Perm = from.Perm
repo.Perm.Synced = time.Now().Unix()
repo.Perm.UserID = user.ID
repo.Perm.RepoID = repo.ID
repo.Perm.Repo = repo
err = _store.PermUpsert(repo.Perm)
if err != nil {
c.String(http.StatusInternalServerError, err.Error())
return
@ -248,7 +272,7 @@ func DeleteRepo(c *gin.Context) {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
c.JSON(200, repo)
c.JSON(http.StatusOK, repo)
}
func RepairRepo(c *gin.Context) {
@ -294,6 +318,13 @@ func RepairRepo(c *gin.Context) {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
repo.Perm.Pull = from.Perm.Pull
repo.Perm.Push = from.Perm.Push
repo.Perm.Admin = from.Perm.Admin
if err := _store.PermUpsert(repo.Perm); err != nil {
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
if err := forge.Deactivate(c, user, repo, host); err != nil {
log.Trace().Err(err).Msgf("deactivate repo '%s' to repair failed", repo.FullName)
@ -342,12 +373,17 @@ func MoveRepo(c *gin.Context) {
}
repo.Update(from)
errStore := _store.UpdateRepo(repo)
if errStore != nil {
_ = c.AbortWithError(http.StatusInternalServerError, errStore)
return
}
repo.Perm = from.Perm
errStore = _store.PermUpsert(repo.Perm)
if errStore != nil {
_ = c.AbortWithError(http.StatusInternalServerError, errStore)
return
}
// creates the jwt token used to verify the repository
t := token.New(token.HookToken, repo.FullName)

78
server/api/user.go
View file

@ -18,14 +18,10 @@ import (
"encoding/base32"
"net/http"
"strconv"
"time"
"github.com/gin-gonic/gin"
"github.com/gorilla/securecookie"
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/forge"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/router/middleware/session"
"github.com/woodpecker-ci/woodpecker/server/store"
@ -38,35 +34,10 @@ func GetSelf(c *gin.Context) {
func GetFeed(c *gin.Context) {
_store := store.FromContext(c)
_forge := server.Config.Services.Forge
user := session.User(c)
latest, _ := strconv.ParseBool(c.Query("latest"))
if time.Unix(user.Synced, 0).Add(time.Hour * 72).Before(time.Now()) {
log.Debug().Msgf("sync begin: %s", user.Login)
user.Synced = time.Now().Unix()
if err := _store.UpdateUser(user); err != nil {
log.Error().Err(err).Msg("UpdateUser")
return
}
config := ToConfig(c)
sync := forge.Syncer{
Forge: _forge,
Store: _store,
Perms: _store,
Match: forge.NamespaceFilter(config.OwnersWhitelist),
}
if err := sync.Sync(c, user, server.Config.FlatPermissions); err != nil {
log.Debug().Msgf("sync error: %s: %s", user.Login, err)
} else {
log.Debug().Msgf("sync complete: %s", user.Login)
}
}
if latest {
feed, err := _store.RepoListLatest(user)
if err != nil {
@ -91,45 +62,40 @@ func GetRepos(c *gin.Context) {
user := session.User(c)
all, _ := strconv.ParseBool(c.Query("all"))
flush, _ := strconv.ParseBool(c.Query("flush"))
if flush || time.Unix(user.Synced, 0).Add(time.Hour*72).Before(time.Now()) {
log.Debug().Msgf("sync begin: %s", user.Login)
user.Synced = time.Now().Unix()
if err := _store.UpdateUser(user); err != nil {
log.Err(err).Msgf("update user '%s'", user.Login)
return
}
config := ToConfig(c)
sync := forge.Syncer{
Forge: _forge,
Store: _store,
Perms: _store,
Match: forge.NamespaceFilter(config.OwnersWhitelist),
}
if err := sync.Sync(c, user, server.Config.FlatPermissions); err != nil {
log.Debug().Msgf("sync error: %s: %s", user.Login, err)
} else {
log.Debug().Msgf("sync complete: %s", user.Login)
}
}
repos, err := _store.RepoList(user, true)
dbRepos, err := _store.RepoList(user, true)
if err != nil {
c.String(http.StatusInternalServerError, "Error fetching repository list. %s", err)
return
}
if all {
active := map[string]bool{}
for _, r := range dbRepos {
active[r.FullName] = r.IsActive
}
_repos, err := _forge.Repos(c, user)
if err != nil {
c.String(http.StatusInternalServerError, "Error fetching repository list. %s", err)
return
}
var repos []*model.Repo
for _, r := range _repos {
if r.Perm.Push {
if active[r.FullName] {
r.IsActive = true
}
repos = append(repos, r)
}
}
c.JSON(http.StatusOK, repos)
return
}
active := make([]*model.Repo, 0)
for _, repo := range repos {
for _, repo := range dbRepos {
if repo.IsActive {
active = append(active, repo)
}

1
server/config.go
View file

@ -84,5 +84,4 @@ var Config = struct {
DefaultTimeout int64
MaxTimeout int64
}
FlatPermissions bool // TODO(485) temporary workaround to not hit api rate limits
}{}

48
server/forge/bitbucket/bitbucket.go
View file

@ -148,11 +148,16 @@ func (c *config) Repo(ctx context.Context, u *model.User, remoteID model.ForgeRe
if remoteID.IsValid() {
name = string(remoteID)
}
repo, err := c.newClient(ctx, u).FindRepo(owner, name)
client := c.newClient(ctx, u)
repo, err := client.FindRepo(owner, name)
if err != nil {
return nil, err
}
return convertRepo(repo), nil
perm, err := client.GetPermission(repo.FullName)
if err != nil {
return nil, err
}
return convertRepo(repo, perm), nil
}
// Repos returns a list of all repositories for Bitbucket account, including
@ -176,44 +181,17 @@ func (c *config) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error
return all, err
}
for _, repo := range repos {
all = append(all, convertRepo(repo))
perm, err := client.GetPermission(repo.FullName)
if err != nil {
return nil, err
}
all = append(all, convertRepo(repo, perm))
}
}
return all, nil
}
// Perm returns the user permissions for the named repository. Because Bitbucket
// does not have an endpoint to access user permissions, we attempt to fetch
// the repository hook list, which is restricted to administrators to calculate
// administrative access to a repository.
func (c *config) Perm(ctx context.Context, u *model.User, r *model.Repo) (*model.Perm, error) {
client := c.newClient(ctx, u)
perms := new(model.Perm)
repo, err := client.FindRepo(r.Owner, r.Name)
if err != nil {
return perms, err
}
perm, err := client.GetPermission(repo.FullName)
if err != nil {
return perms, err
}
switch perm.Permission {
case "admin":
perms.Admin = true
fallthrough
case "write":
perms.Push = true
fallthrough
default:
perms.Pull = true
}
return perms, nil
}
// File fetches the file from the Bitbucket repository and returns its contents.
func (c *config) File(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]byte, error) {
config, err := c.newClient(ctx, u).FindSource(r.Owner, r.Name, p.Commit, f)

46
server/forge/bitbucket/bitbucket_test.go
View file

@ -137,34 +137,6 @@ func Test_bitbucket(t *testing.T) {
})
})
g.Describe("When requesting repository permissions", func() {
g.It("Should handle not found errors", func() {
_, err := c.Perm(ctx, fakeUser, fakeRepoNotFound)
g.Assert(err).IsNotNil()
})
g.It("Should authorize read access", func() {
perm, err := c.Perm(ctx, fakeUser, fakeRepoReadOnly)
g.Assert(err).IsNil()
g.Assert(perm.Pull).IsTrue()
g.Assert(perm.Push).IsFalse()
g.Assert(perm.Admin).IsFalse()
})
g.It("Should authorize write access", func() {
perm, err := c.Perm(ctx, fakeUser, fakeRepoWriteOnly)
g.Assert(err).IsNil()
g.Assert(perm.Pull).IsTrue()
g.Assert(perm.Push).IsTrue()
g.Assert(perm.Admin).IsFalse()
})
g.It("Should authorize admin access", func() {
perm, err := c.Perm(ctx, fakeUser, fakeRepoAdmin)
g.Assert(err).IsNil()
g.Assert(perm.Pull).IsTrue()
g.Assert(perm.Push).IsTrue()
g.Assert(perm.Admin).IsTrue()
})
})
g.Describe("When requesting user repositories", func() {
g.It("Should return the details", func() {
repos, err := c.Repos(ctx, fakeUser)
@ -333,24 +305,6 @@ var (
FullName: "test_name/hook_empty",
}
fakeRepoReadOnly = &model.Repo{
Owner: "test_name",
Name: "permission_read",
FullName: "test_name/permission_read",
}
fakeRepoWriteOnly = &model.Repo{
Owner: "test_name",
Name: "permission_write",
FullName: "test_name/permission_write",
}
fakeRepoAdmin = &model.Repo{
Owner: "test_name",
Name: "permission_admin",
FullName: "test_name/permission_admin",
}
fakePipeline = &model.Pipeline{
Commit: "9ecad50",
}

18
server/forge/bitbucket/convert.go
View file

@ -48,7 +48,7 @@ func convertStatus(status model.StatusValue) string {
// convertRepo is a helper function used to convert a Bitbucket repository
// structure to the common Woodpecker repository structure.
func convertRepo(from *internal.Repo) *model.Repo {
func convertRepo(from *internal.Repo, perm *internal.RepoPerm) *model.Repo {
repo := model.Repo{
ForgeRemoteID: model.ForgeRemoteID(from.UUID),
Clone: cloneLink(from),
@ -60,6 +60,7 @@ func convertRepo(from *internal.Repo) *model.Repo {
Avatar: from.Owner.Links.Avatar.Href,
SCMKind: model.SCMKind(from.Scm),
Branch: "master",
Perm: convertPerm(perm),
}
if repo.SCMKind == model.RepoHg {
repo.Branch = "default"
@ -67,6 +68,21 @@ func convertRepo(from *internal.Repo) *model.Repo {
return &repo
}
func convertPerm(from *internal.RepoPerm) *model.Perm {
perms := new(model.Perm)
switch from.Permission {
case "admin":
perms.Admin = true
fallthrough
case "write":
perms.Push = true
fallthrough
default:
perms.Pull = true
}
return perms
}
// cloneLink is a helper function that tries to extract the clone url from the
// repository object.
func cloneLink(repo *internal.Repo) string {

7
server/forge/bitbucket/convert_test.go
View file

@ -52,8 +52,11 @@ func Test_helper(t *testing.T) {
}
from.Owner.Links.Avatar.Href = "http://..."
from.Links.HTML.Href = "https://bitbucket.org/foo/bar"
fromPerm := &internal.RepoPerm{
Permission: "write",
}
to := convertRepo(from)
to := convertRepo(from, fromPerm)
g.Assert(to.Avatar).Equal(from.Owner.Links.Avatar.Href)
g.Assert(to.FullName).Equal(from.FullName)
g.Assert(to.Owner).Equal("octocat")
@ -63,6 +66,8 @@ func Test_helper(t *testing.T) {
g.Assert(to.IsSCMPrivate).Equal(from.IsPrivate)
g.Assert(to.Clone).Equal(from.Links.HTML.Href)
g.Assert(to.Link).Equal(from.Links.HTML.Href)
g.Assert(to.Perm.Push).IsTrue()
g.Assert(to.Perm.Admin).IsFalse()
})
g.It("should convert team", func() {

4
server/forge/bitbucket/parse.go
View file

@ -59,7 +59,7 @@ func parsePushHook(payload []byte) (*model.Repo, *model.Pipeline, error) {
if change.New.Target.Hash == "" {
continue
}
return convertRepo(&hook.Repo), convertPushHook(&hook, &change), nil
return convertRepo(&hook.Repo, &internal.RepoPerm{}), convertPushHook(&hook, &change), nil
}
return nil, nil, nil
}
@ -75,5 +75,5 @@ func parsePullHook(payload []byte) (*model.Repo, *model.Pipeline, error) {
if hook.PullRequest.State != stateOpen {
return nil, nil, nil
}
return convertRepo(&hook.Repo), convertPullHook(&hook), nil
return convertRepo(&hook.Repo, &internal.RepoPerm{}), convertPullHook(&hook), nil
}

24
server/forge/bitbucketserver/bitbucketserver.go
View file

@ -154,32 +154,36 @@ func (*Config) TeamPerm(_ *model.User, _ string) (*model.Perm, error) {
}
func (c *Config) Repo(ctx context.Context, u *model.User, _ model.ForgeRemoteID, owner, name string) (*model.Repo, error) {
repo, err := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token).FindRepo(owner, name)
client := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token)
repo, err := client.FindRepo(owner, name)
if err != nil {
return nil, err
}
return convertRepo(repo), nil
perm, err := client.FindRepoPerms(repo.Project.Key, repo.Name)
if err != nil {
return nil, err
}
return convertRepo(repo, perm), nil
}
func (c *Config) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error) {
repos, err := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token).FindRepos()
client := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token)
repos, err := client.FindRepos()
if err != nil {
return nil, err
}
var all []*model.Repo
for _, repo := range repos {
all = append(all, convertRepo(repo))
perm, err := client.FindRepoPerms(repo.Project.Key, repo.Name)
if err != nil {
return nil, err
}
all = append(all, convertRepo(repo, perm))
}
return all, nil
}
func (c *Config) Perm(ctx context.Context, u *model.User, repo *model.Repo) (*model.Perm, error) {
client := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token)
return client.FindRepoPerms(repo.Owner, repo.Name)
}
func (c *Config) File(ctx context.Context, u *model.User, r *model.Repo, p *model.Pipeline, f string) ([]byte, error) {
client := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token)

3
server/forge/bitbucketserver/convert.go
View file

@ -50,7 +50,7 @@ func convertStatus(status model.StatusValue) string {
// convertRepo is a helper function used to convert a Bitbucket server repository
// structure to the common Woodpecker repository structure.
func convertRepo(from *internal.Repo) *model.Repo {
func convertRepo(from *internal.Repo, perm *model.Perm) *model.Repo {
repo := model.Repo{
ForgeRemoteID: model.ForgeRemoteID(fmt.Sprint(from.ID)),
Name: from.Slug,
@ -59,6 +59,7 @@ func convertRepo(from *internal.Repo) *model.Repo {
SCMKind: model.RepoGit,
IsSCMPrivate: true, // Since we have to use Netrc it has to always be private :/
FullName: fmt.Sprintf("%s/%s", from.Project.Key, from.Slug),
Perm: perm,
}
for _, item := range from.Links.Clone {

3
server/forge/bitbucketserver/convert_test.go
View file

@ -47,7 +47,7 @@ func Test_helper(t *testing.T) {
from.Links.Self = append(from.Links.Self, selfRef)
to := convertRepo(from)
to := convertRepo(from, &model.Perm{Pull: true})
g.Assert(to.FullName).Equal("octocat/hello-world")
g.Assert(to.Owner).Equal("octocat")
g.Assert(to.Name).Equal("hello-world")
@ -56,6 +56,7 @@ func Test_helper(t *testing.T) {
g.Assert(to.IsSCMPrivate).Equal(true)
g.Assert(to.Clone).Equal("https://server.org/foo/bar.git")
g.Assert(to.Link).Equal("https://server.org/foo/bar")
g.Assert(to.Perm.Pull).IsTrue()
})
g.It("should convert user", func() {

2
server/forge/bitbucketserver/parse.go
View file

@ -31,7 +31,7 @@ func parseHook(r *http.Request, baseURL string) (*model.Repo, *model.Pipeline, e
return nil, nil, err
}
pipeline := convertPushHook(hook, baseURL)
repo := convertRepo(&hook.Repository)
repo := convertRepo(&hook.Repository, &model.Perm{})
return repo, pipeline, nil
}

4
server/forge/forge.go
View file

@ -50,10 +50,6 @@ type Forge interface {
// Repos fetches a list of repos from the forge.
Repos(ctx context.Context, u *model.User) ([]*model.Repo, error)
// Perm fetches the named repository permissions from
// the forge for the specified user.
Perm(ctx context.Context, u *model.User, r *model.Repo) (*model.Perm, error)
// File fetches a file from the forge repository and returns in string
// format.
File(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error)

28
server/forge/gitea/fixtures/hooks.go
View file

@ -52,7 +52,12 @@ const HookPush = `
"login": "gordon",
"username": "gordon"
},
"private": true
"private": true,
"permissions": {
"admin": true,
"push": true,
"pull": true
}
},
"pusher": {
"name": "gordon",
@ -122,7 +127,12 @@ const HookPushBranch = `
"followers_count": 0,
"following_count": 0,
"starred_repos_count": 0,
"username": "meisam"
"username": "meisam",
"permissions": {
"admin": true,
"push": true,
"pull": true
}
},
"name": "woodpecktester",
"full_name": "meisam/woodpecktester",
@ -245,7 +255,12 @@ const HookPushTag = `{
"clone_url": "http://gitea.golang.org/gordon/hello-world.git",
"default_branch": "master",
"created_at": "2015-10-22T19:32:44Z",
"updated_at": "2016-11-24T13:37:16Z"
"updated_at": "2016-11-24T13:37:16Z",
"permissions": {
"admin": true,
"push": true,
"pull": true
}
},
"sender": {
"id": 1,
@ -300,7 +315,12 @@ const HookPullRequest = `{
"private": true,
"html_url": "http://gitea.golang.org/gordon/hello-world",
"clone_url": "https://gitea.golang.org/gordon/hello-world.git",
"default_branch": "master"
"default_branch": "master",
"permissions": {
"admin": true,
"push": true,
"pull": true
}
},
"sender": {
"id": 1,

14
server/forge/gitea/gitea.go
View file

@ -268,20 +268,6 @@ func (c *Gitea) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error)
})
}
// Perm returns the user permissions for the named Gitea repository.
func (c *Gitea) Perm(ctx context.Context, u *model.User, r *model.Repo) (*model.Perm, error) {
client, err := c.newClientToken(ctx, u.Token)
if err != nil {
return nil, err
}
repo, _, err := client.GetRepo(r.Owner, r.Name)
if err != nil {
return nil, err
}
return toPerm(repo.Permissions), nil
}
// File fetches the file from the Gitea repository and returns its contents.
func (c *Gitea) File(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error) {
client, err := c.newClientToken(ctx, u.Token)

14
server/forge/gitea/gitea_test.go
View file

@ -100,20 +100,6 @@ func Test_gitea(t *testing.T) {
})
})
g.Describe("Requesting repository permissions", func() {
g.It("Should return the permission details", func() {
perm, err := c.Perm(ctx, fakeUser, fakeRepo)
g.Assert(err).IsNil()
g.Assert(perm.Admin).IsTrue()
g.Assert(perm.Push).IsTrue()
g.Assert(perm.Pull).IsTrue()
})
g.It("Should handle a not found error", func() {
_, err := c.Perm(ctx, fakeUser, fakeRepoNotFound)
g.Assert(err).IsNotNil()
})
})
g.Describe("Requesting a repository list", func() {
g.It("Should return the repository list", func() {
repos, err := c.Repos(ctx, fakeUser)

1
server/forge/gitea/helper.go
View file

@ -47,6 +47,7 @@ func toRepo(from *gitea.Repository) *model.Repo {
IsSCMPrivate: from.Private || from.Owner.Visibility != gitea.VisibleTypePublic,
Clone: from.CloneURL,
Branch: from.DefaultBranch,
Perm: toPerm(from.Permissions),
}
}

2
server/forge/gitea/helper_test.go
View file

@ -202,6 +202,7 @@ func Test_parse(t *testing.T) {
HTMLURL: "http://gitea.golang.org/gophers/hello-world",
Private: true,
DefaultBranch: "master",
Permissions: &gitea.Permission{Admin: true},
}
repo := toRepo(&from)
g.Assert(repo.FullName).Equal(from.FullName)
@ -212,6 +213,7 @@ func Test_parse(t *testing.T) {
g.Assert(repo.Clone).Equal(from.CloneURL)
g.Assert(repo.Avatar).Equal(from.Owner.AvatarURL)
g.Assert(repo.IsSCMPrivate).Equal(from.Private)
g.Assert(repo.Perm.Admin).IsTrue()
})
g.It("Should correct a malformed avatar url", func() {

10
server/forge/github/github.go
View file

@ -209,16 +209,6 @@ func (c *client) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error
return repos, nil
}
// Perm returns the user permissions for the named GitHub repository.
func (c *client) Perm(ctx context.Context, u *model.User, r *model.Repo) (*model.Perm, error) {
client := c.newClientToken(ctx, u.Token)
repo, _, err := client.Repositories.Get(ctx, r.Owner, r.Name)
if err != nil {
return nil, err
}
return convertPerm(repo.GetPermissions()), nil
}
// File fetches the file from the GitHub repository and returns its contents.
func (c *client) File(ctx context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error) {
client := c.newClientToken(ctx, u.Token)

14
server/forge/github/github_test.go
View file

@ -94,20 +94,6 @@ func Test_github(t *testing.T) {
})
})
g.Describe("Requesting repository permissions", func() {
g.It("Should return the permission details", func() {
perm, err := c.Perm(ctx, fakeUser, fakeRepo)
g.Assert(err).IsNil()
g.Assert(perm.Admin).IsTrue()
g.Assert(perm.Push).IsTrue()
g.Assert(perm.Pull).IsTrue()
})
g.It("Should handle a not found error", func() {
_, err := c.Perm(ctx, fakeUser, fakeRepoNotFound)
g.Assert(err).IsNotNil()
})
})
g.It("Should return a user repository list")
g.It("Should return a user team list")

6
server/forge/gitlab/convert.go
View file

@ -33,7 +33,6 @@ const (
func (g *GitLab) convertGitLabRepo(_repo *gitlab.Project) (*model.Repo, error) {
parts := strings.Split(_repo.PathWithNamespace, "/")
// TODO(648) save repo id (support nested repos)
owner := strings.Join(parts[:len(parts)-1], "/")
name := parts[len(parts)-1]
repo := &model.Repo{
@ -47,6 +46,11 @@ func (g *GitLab) convertGitLabRepo(_repo *gitlab.Project) (*model.Repo, error) {
Branch: _repo.DefaultBranch,
Visibility: model.RepoVisibly(_repo.Visibility),
IsSCMPrivate: !_repo.Public,
Perm: &model.Perm{
Pull: isRead(_repo),
Push: isWrite(_repo),
Admin: isAdmin(_repo),
},
}
if len(repo.Branch) == 0 { // TODO: do we need that?

24
server/forge/gitlab/gitlab.go
View file

@ -335,30 +335,6 @@ func (g *GitLab) PullRequests(ctx context.Context, u *model.User, r *model.Repo,
return result, err
}
// Perm fetches the named repository from the forge.
func (g *GitLab) Perm(ctx context.Context, user *model.User, r *model.Repo) (*model.Perm, error) {
client, err := newClient(g.URL, user.Token, g.SkipVerify)
if err != nil {
return nil, err
}
repo, err := g.getProject(ctx, client, r.Owner, r.Name)
if err != nil {
return nil, err
}
// repo owner is granted full access
if repo.Owner != nil && repo.Owner.Username == user.Login {
return &model.Perm{Push: true, Pull: true, Admin: true}, nil
}
// return permission for current user
return &model.Perm{
Pull: isRead(repo),
Push: isWrite(repo),
Admin: isAdmin(repo),
}, nil
}
// File fetches a file from the forge repository and returns in string format.
func (g *GitLab) File(ctx context.Context, user *model.User, repo *model.Repo, pipeline *model.Pipeline, fileName string) ([]byte, error) {
client, err := newClient(g.URL, user.Token, g.SkipVerify)

29
server/forge/gitlab/gitlab_test.go
View file

@ -107,35 +107,6 @@ func Test_GitLab(t *testing.T) {
})
})
// Test permissions method
g.Describe("Perm", func() {
g.It("Should return repo permissions", func() {
perm, err := client.Perm(ctx, &user, &repo)
assert.NoError(t, err)
assert.True(t, perm.Admin)
assert.True(t, perm.Pull)
assert.True(t, perm.Push)
})
g.It("Should return repo permissions when user is admin", func() {
perm, err := client.Perm(ctx, &user, &model.Repo{
Owner: "brightbox",
Name: "puppet",
})
assert.NoError(t, err)
g.Assert(perm.Admin).Equal(true)
g.Assert(perm.Pull).Equal(true)
g.Assert(perm.Push).Equal(true)
})
g.It("Should return error, when repo is not exist", func() {
_, err := client.Perm(ctx, &user, &model.Repo{
Owner: "not-existed",
Name: "not-existed",
})
g.Assert(err).IsNotNil()
})
})
// Test activate method
g.Describe("Activate", func() {
g.It("Should be success", func() {

36
server/forge/gitlab/testdata/projects.go vendored
View file

@ -51,7 +51,17 @@ var allProjectsPayload = []byte(`
"path": "diaspora",
"updated_at": "2013-09-30T13:46:02Z"
},
"archived": false
"archived": false,
"permissions": {
"project_access": {
"access_level": 10,
"notification_level": 3
},
"group_access": {
"access_level": 50,
"notification_level": 3
}
}
},
{
"id": 6,
@ -87,7 +97,17 @@ var allProjectsPayload = []byte(`
"path": "brightbox",
"updated_at": "2013-09-30T13:46:02Z"
},
"archived": true
"archived": true,
"permissions": {
"project_access": {
"access_level": 10,
"notification_level": 3
},
"group_access": {
"access_level": 50,
"notification_level": 3
}
}
}
]
`)
@ -128,7 +148,17 @@ var notArchivedProjectsPayload = []byte(`
"path": "diaspora",
"updated_at": "2013-09-30T13:46:02Z"
},
"archived": false
"archived": false,
"permissions": {
"project_access": {
"access_level": 10,
"notification_level": 3
},
"group_access": {
"access_level": 50,
"notification_level": 3
}
}
}
]
`)

21
server/forge/gogs/fixtures/hooks.go
View file

@ -48,7 +48,12 @@ var HookPush = `
"email": "gordon@golang.org",
"username": "gordon"
},
"private": true
"private": true,
"permissions": {
"admin": true,
"push": true,
"pull": true
}
},
"pusher": {
"name": "gordon",
@ -88,7 +93,12 @@ var HookPushTag = `{
"clone_url": "http://gogs.golang.org/gordon/hello-world.git",
"default_branch": "master",
"created_at": "2015-10-22T19:32:44Z",
"updated_at": "2016-11-24T13:37:16Z"
"updated_at": "2016-11-24T13:37:16Z",
"permissions": {
"admin": true,
"push": true,
"pull": true
}
},
"sender": {
"id": 1,
@ -142,7 +152,12 @@ var HookPullRequest = `{
"private": true,
"html_url": "http://gogs.golang.org/gordon/hello-world",
"clone_url": "https://gogs.golang.org/gordon/hello-world.git",
"default_branch": "master"
"default_branch": "master",
"permissions": {
"admin": true,
"push": true,
"pull": true
}
},
"sender": {
"id": 1,

10
server/forge/gogs/gogs.go
View file

@ -176,16 +176,6 @@ func (c *client) Repos(_ context.Context, u *model.User) ([]*model.Repo, error)
return repos, err
}
// Perm returns the user permissions for the named Gogs repository.
func (c *client) Perm(_ context.Context, u *model.User, r *model.Repo) (*model.Perm, error) {
client := c.newClientToken(u.Token)
repo, err := client.GetRepo(r.Owner, r.Name)
if err != nil {
return nil, err
}
return toPerm(repo.Permissions), nil
}
// File fetches the file from the Gogs repository and returns its contents.
func (c *client) File(_ context.Context, u *model.User, r *model.Repo, b *model.Pipeline, f string) ([]byte, error) {
client := c.newClientToken(u.Token)

14
server/forge/gogs/gogs_test.go
View file

@ -101,20 +101,6 @@ func Test_gogs(t *testing.T) {
})
})
g.Describe("Requesting repository permissions", func() {
g.It("Should return the permission details", func() {
perm, err := c.Perm(ctx, fakeUser, fakeRepo)
g.Assert(err).IsNil()
g.Assert(perm.Admin).IsTrue()
g.Assert(perm.Push).IsTrue()
g.Assert(perm.Pull).IsTrue()
})
g.It("Should handle a not found error", func() {
_, err := c.Perm(ctx, fakeUser, fakeRepoNotFound)
g.Assert(err).IsNotNil()
})
})
g.Describe("Requesting a repository list", func() {
g.It("Should return the repository list", func() {
repos, err := c.Repos(ctx, fakeUser)

1
server/forge/gogs/helper.go
View file

@ -46,6 +46,7 @@ func toRepo(from *gogs.Repository, privateMode bool) *model.Repo {
IsSCMPrivate: from.Private || privateMode,
Clone: from.CloneURL,
Branch: from.DefaultBranch,
Perm: toPerm(from.Permissions),
}
}

2
server/forge/gogs/helper_test.go
View file

@ -174,6 +174,7 @@ func Test_parse(t *testing.T) {
HTMLURL: "http://gogs.golang.org/gophers/hello-world",
Private: true,
DefaultBranch: "master",
Permissions: &gogs.Permission{Admin: true},
}
repo := toRepo(&from, false)
g.Assert(repo.FullName).Equal(from.FullName)
@ -184,6 +185,7 @@ func Test_parse(t *testing.T) {
g.Assert(repo.Clone).Equal(from.CloneURL)
g.Assert(repo.Avatar).Equal(from.Owner.AvatarUrl)
g.Assert(repo.IsSCMPrivate).Equal(from.Private)
g.Assert(repo.Perm.Admin).IsTrue()
})
g.It("Should correct a malformed avatar url", func() {

28
server/forge/mocks/forge.go
View file

@ -1,4 +1,4 @@
// Code generated by mockery v2.22.1. DO NOT EDIT.
// Code generated by mockery v2.23.0. DO NOT EDIT.
package mocks
@ -300,32 +300,6 @@ func (_m *Forge) OrgMembership(ctx context.Context, u *model.User, owner string)
return r0, r1
}
// Perm provides a mock function with given fields: ctx, u, r
func (_m *Forge) Perm(ctx context.Context, u *model.User, r *model.Repo) (*model.Perm, error) {
ret := _m.Called(ctx, u, r)
var r0 *model.Perm
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo) (*model.Perm, error)); ok {
return rf(ctx, u, r)
}
if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo) *model.Perm); ok {
r0 = rf(ctx, u, r)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.Perm)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *model.User, *model.Repo) error); ok {
r1 = rf(ctx, u, r)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// PullRequests provides a mock function with given fields: ctx, u, r, p
func (_m *Forge) PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.PaginationData) ([]*model.PullRequest, error) {
ret := _m.Called(ctx, u, r, p)

115
server/forge/userSyncer.go
View file

@ -1,115 +0,0 @@
// Copyright 2022 Woodpecker Authors
// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package forge
import (
"context"
"fmt"
"time"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/store"
)
// UserSyncer syncs the user repository and permissions.
type UserSyncer interface {
Sync(ctx context.Context, user *model.User) error
}
type Syncer struct {
Forge Forge
Store store.Store
Perms model.PermStore
Match FilterFunc
}
// FilterFunc can be used to filter which repositories are
// synchronized with the local datastore.
type FilterFunc func(*model.Repo) bool
func NamespaceFilter(namespaces map[string]bool) FilterFunc {
if len(namespaces) == 0 {
return noopFilter
}
return func(repo *model.Repo) bool {
return namespaces[repo.Owner]
}
}
// noopFilter is a filter function that always returns true.
func noopFilter(*model.Repo) bool {
return true
}
// SetFilter sets the filter function.
func (s *Syncer) SetFilter(fn FilterFunc) {
s.Match = fn
}
func (s *Syncer) Sync(ctx context.Context, user *model.User, flatPermissions bool) error {
unix := time.Now().Unix() - (3601) // force immediate expiration. note 1 hour expiration is hard coded at the moment
repos, err := s.Forge.Repos(ctx, user)
if err != nil {
return err
}
forgeRepos := make([]*model.Repo, 0, len(repos))
for _, repo := range repos {
if s.Match(repo) {
repo.Perm = &model.Perm{
UserID: user.ID,
RepoID: repo.ID,
Repo: repo,
Synced: unix,
}
// TODO(485) temporary workaround to not hit api rate limits
if flatPermissions {
repo.Perm.Pull = true
repo.Perm.Push = true
repo.Perm.Admin = true
} else {
forgePerm, err := s.Forge.Perm(ctx, user, repo)
if err != nil {
return fmt.Errorf("could not fetch permission of repo '%s': %w", repo.FullName, err)
}
repo.Perm.Pull = forgePerm.Pull
repo.Perm.Push = forgePerm.Push
repo.Perm.Admin = forgePerm.Admin
}
forgeRepos = append(forgeRepos, repo)
}
}
err = s.Store.RepoBatch(forgeRepos)
if err != nil {
return err
}
// this is here as a precaution. I want to make sure that if an api
// call to the version control system fails and (for some reason) returns
// an empty list, we don't wipe out the user repository permissions.
//
// the side-effect of this code is that a user with 1 repository whose
// access is removed will still display in the feed, but they will not
// be able to access the actual repository data.
if len(repos) == 0 {
return nil
}
return s.Perms.PermFlush(user, unix)
}

1
server/model/repo.go
View file

@ -40,7 +40,6 @@ type Repo struct {
Visibility RepoVisibly `json:"visibility" xorm:"varchar(10) 'repo_visibility'"`
IsSCMPrivate bool `json:"private" xorm:"repo_private"`
IsTrusted bool `json:"trusted" xorm:"repo_trusted"`
IsStarred bool `json:"starred,omitempty" xorm:"-"`
IsGated bool `json:"gated" xorm:"repo_gated"`
IsActive bool `json:"active" xorm:"repo_active"`
AllowPull bool `json:"allow_pr" xorm:"repo_allow_pr"`

3
server/model/user.go
View file

@ -56,9 +56,6 @@ type User struct {
// the avatar url for this user.
Avatar string `json:"avatar_url" xorm:" varchar(500) 'user_avatar'"`
// Synced is the timestamp when the user was synced with the forge.
Synced int64 `json:"synced" xorm:"user_synced"`
// Admin indicates the user is a system administrator.
//
// NOTE: If the username is part of the WOODPECKER_ADMINS

2
server/router/api.go
View file

@ -61,6 +61,7 @@ func apiRoutes(e *gin.Engine) {
}
}
apiBase.POST("/repos/:owner/:name", session.MustUser(), api.PostRepo)
repoBase := apiBase.Group("/repos/:owner/:name")
{
repoBase.Use(session.SetRepo())
@ -72,7 +73,6 @@ func apiRoutes(e *gin.Engine) {
{
repo.Use(session.MustPull)
repo.POST("", session.MustRepoAdmin(), api.PostRepo)
repo.GET("", api.GetRepo)
repo.GET("/branches", api.GetRepoBranches)

14
server/router/middleware/session/repo.go
View file

@ -21,8 +21,8 @@ import (
"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/store"
"github.com/woodpecker-ci/woodpecker/server/store/types"
@ -94,8 +94,7 @@ func SetPerm() gin.HandlerFunc {
repo := Repo(c)
perm := new(model.Perm)
switch {
case user != nil:
if user != nil {
var err error
perm, err = _store.PermFind(user, repo)
if err != nil {
@ -103,10 +102,12 @@ func SetPerm() gin.HandlerFunc {
user.Login, repo.FullName, err)
}
if time.Unix(perm.Synced, 0).Add(time.Hour).Before(time.Now()) {
perm, err = server.Config.Services.Forge.Perm(c, user, repo)
_repo, err := server.Config.Services.Forge.Repo(c, user, repo.ForgeRemoteID, repo.Owner, repo.Name)
if err == nil {
log.Debug().Msgf("Synced user permission for %s %s", user.Login, repo.FullName)
perm = _repo.Perm
perm.Repo = repo
perm.RepoID = repo.ID
perm.UserID = user.ID
perm.Synced = time.Now().Unix()
if err := _store.PermUpsert(perm); err != nil {
@ -127,10 +128,7 @@ func SetPerm() gin.HandlerFunc {
perm.Admin = true
}
switch {
case repo.Visibility == model.VisibilityPublic:
perm.Pull = true
case repo.Visibility == model.VisibilityInternal && user != nil:
if repo.Visibility == model.VisibilityPublic || (repo.Visibility == model.VisibilityInternal && user != nil) {
perm.Pull = true
}

33
server/store/datastore/migration/015_remove_inactive_repos.go Normal file
View file

@ -0,0 +1,33 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package migration
import (
"xorm.io/xorm"
)
var removeInactiveRepos = task{
name: "remove-inactive-repos",
required: true,
fn: func(sess *xorm.Session) error {
// If the timeout is 0, the repo was never activated, so we remove it.
_, err := sess.Table("repos").Where("repo_active = ?", false).And("repo_timeout = ?", 0).Delete()
if err != nil {
return err
}
return dropTableColumns(sess, "users", "user_synced")
},
}

1
server/store/datastore/migration/migration.go
View file

@ -43,6 +43,7 @@ var migrationTasks = []*task{
&renameRemoteToForge,
&renameForgeIDToForgeRemoteID,
&removeActiveFromUsers,
&removeInactiveRepos,
}
var allBeans = []interface{}{

76
server/store/datastore/repo.go
View file

@ -18,7 +18,6 @@ import (
"errors"
"strings"
"github.com/rs/zerolog/log"
"xorm.io/builder"
"xorm.io/xorm"
@ -153,78 +152,3 @@ func (s storage) RepoList(user *model.User, owned bool) ([]*model.Repo, error) {
Asc("repo_full_name").
Find(&repos)
}
// RepoBatch Sync batch of repos from SCM (with permissions) to store (create if not exist else update)
// TODO: only store activated repos ...
func (s storage) RepoBatch(repos []*model.Repo) error {
sess := s.engine.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
for i := range repos {
if len(repos[i].Owner) == 0 || len(repos[i].Name) == 0 || len(repos[i].FullName) == 0 {
log.Debug().Msgf("skip insert/update repo: %#v", repos[i])
continue
}
exist := true
repo, err := s.getRepoNameFallback(sess, repos[i].ForgeRemoteID, repos[i].FullName)
if err != nil {
if errors.Is(err, types.RecordNotExist) {
exist = false
} else {
return err
}
}
if exist {
if repos[i].FullName != repo.FullName {
// create redirection
err := s.createRedirection(sess, &model.Redirection{RepoID: repo.ID, FullName: repo.FullName})
if err != nil {
return err
}
}
if repos[i].ForgeRemoteID.IsValid() {
if _, err := sess.
Where("forge_remote_id = ?", repos[i].ForgeRemoteID).
Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "forge_id").
Update(repos[i]); err != nil {
return err
}
} else {
if _, err := sess.
Where("repo_owner = ?", repos[i].Owner).
And(" repo_name = ?", repos[i].Name).
Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "forge_id").
Update(repos[i]); err != nil {
return err
}
}
_, err := sess.
Where("forge_remote_id = ?", repos[i].ForgeRemoteID).
Get(repos[i])
if err != nil {
return err
}
} else {
// only Insert on single object ref set auto created ID back to object
if _, err := sess.Insert(repos[i]); err != nil {
return err
}
}
if repos[i].Perm != nil {
repos[i].Perm.RepoID = repos[i].ID
repos[i].Perm.Repo = repos[i]
if err := s.permUpsert(sess, repos[i].Perm); err != nil {
return err
}
}
}
return sess.Commit()
}

106
server/store/datastore/repo_test.go
View file

@ -17,7 +17,6 @@ package datastore
import (
"testing"
"time"
"github.com/franela/goblin"
"github.com/stretchr/testify/assert"
@ -293,104 +292,6 @@ func TestRepoCount(t *testing.T) {
}
}
func TestRepoBatch(t *testing.T) {
store, closer := newTestStore(t, new(model.Repo), new(model.User), new(model.Perm), new(model.Redirection))
defer closer()
if !assert.NoError(t, store.CreateRepo(&model.Repo{
ForgeRemoteID: "5",
UserID: 1,
FullName: "foo/bar",
Owner: "foo",
Name: "bar",
IsActive: true,
})) {
return
}
repos := []*model.Repo{
{
ForgeRemoteID: "5",
UserID: 1,
FullName: "foo/bar",
Owner: "foo",
Name: "bar",
IsActive: true,
Perm: &model.Perm{
UserID: 1,
Pull: true,
Push: true,
Admin: true,
Synced: time.Now().Unix(),
},
},
{
ForgeRemoteID: "6",
UserID: 1,
FullName: "bar/baz",
Owner: "bar",
Name: "baz",
IsActive: true,
},
{
ForgeRemoteID: "7",
UserID: 1,
FullName: "baz/qux",
Owner: "baz",
Name: "qux",
IsActive: true,
},
{
ForgeRemoteID: "8",
UserID: 0, // not activated repos do hot have a user id assigned
FullName: "baz/notes",
Owner: "baz",
Name: "notes",
IsActive: false,
},
}
if !assert.NoError(t, store.RepoBatch(repos)) {
return
}
// check DB state
perm, err := store.PermFind(&model.User{ID: 1}, repos[0])
assert.NoError(t, err)
assert.True(t, perm.Admin)
repo := &model.Repo{
ForgeRemoteID: "5",
FullName: "foo/bar",
Owner: "foo",
Name: "bar",
Perm: &model.Perm{
UserID: 1,
Pull: true,
Push: true,
Admin: false,
Synced: time.Now().Unix(),
},
}
assert.NoError(t, store.RepoBatch([]*model.Repo{repo}))
assert.EqualValues(t, repos[0].ID, repo.ID)
// check current DB state
_, err = store.engine.ID(repo.ID).Get(repo)
assert.NoError(t, err)
assert.True(t, repo.IsActive)
perm, err = store.PermFind(&model.User{ID: 1}, repos[0])
assert.NoError(t, err)
assert.False(t, perm.Admin)
allRepos := make([]*model.Repo, 0, 4)
assert.NoError(t, store.engine.Find(&allRepos))
assert.Len(t, allRepos, 4)
count, err := store.GetRepoCount()
assert.NoError(t, err)
assert.EqualValues(t, 3, count)
}
func TestRepoCrud(t *testing.T) {
store, closer := newTestStore(t,
new(model.Repo),
@ -468,13 +369,18 @@ func TestRepoRedirection(t *testing.T) {
assert.NoError(t, store.CreateRepo(&repo))
repoUpdated := model.Repo{
ID: repo.ID,
ForgeRemoteID: "1",
FullName: "bradrydzewski/test-renamed",
Owner: "bradrydzewski",
Name: "test-renamed",
}
assert.NoError(t, store.RepoBatch([]*model.Repo{&repoUpdated}))
assert.NoError(t, store.UpdateRepo(&repoUpdated))
assert.NoError(t, store.CreateRedirection(&model.Redirection{
RepoID: repo.ID,
FullName: repo.FullName,
}))
// test redirection from old repo name
repoFromStore, err := store.GetRepoNameFallback("1", "bradrydzewski/test")

16
server/store/mocks/store.go
View file

@ -1,4 +1,4 @@
// Code generated by mockery v2.22.1. DO NOT EDIT.
// Code generated by mockery v2.23.0. DO NOT EDIT.
package mocks
@ -1451,20 +1451,6 @@ func (_m *Store) RegistryUpdate(_a0 *model.Registry) error {
return r0
}
// RepoBatch provides a mock function with given fields: _a0
func (_m *Store) RepoBatch(_a0 []*model.Repo) error {
ret := _m.Called(_a0)
var r0 error
if rf, ok := ret.Get(0).(func([]*model.Repo) error); ok {
r0 = rf(_a0)
} else {
r0 = ret.Error(0)
}
return r0
}
// RepoList provides a mock function with given fields: user, owned
func (_m *Store) RepoList(user *model.User, owned bool) ([]*model.Repo, error) {
ret := _m.Called(user, owned)

2
server/store/store.go
View file

@ -103,8 +103,6 @@ type Store interface {
// RepoList TODO: paginate
RepoList(user *model.User, owned bool) ([]*model.Repo, error)
RepoListLatest(*model.User) ([]*model.Feed, error)
// RepoBatch Sync batch of repos from SCM (with permissions) to store (create if not exist else update)
RepoBatch([]*model.Repo) error
// Permissions
PermFind(user *model.User, repo *model.Repo) (*model.Perm, error)

8
server/web/config.go
View file

@ -18,7 +18,6 @@ import (
"encoding/json"
"net/http"
"text/template"
"time"
"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log"
@ -40,15 +39,9 @@ func Config(c *gin.Context) {
).Sign(user.Hash)
}
var syncing bool
if user != nil {
syncing = time.Unix(user.Synced, 0).Add(time.Hour * 72).Before(time.Now())
}
configData := map[string]interface{}{
"user": user,
"csrf": csrf,
"syncing": syncing,
"docs": server.Config.Server.Docs,
"version": version.String(),
"forge": server.Config.Services.Forge.Name(),
@ -73,7 +66,6 @@ func Config(c *gin.Context) {
const configTemplate = `
window.WOODPECKER_USER = {{ json .user }};
window.WOODPECKER_SYNC = {{ .syncing }};
window.WOODPECKER_CSRF = "{{ .csrf }}";
window.WOODPECKER_VERSION = "{{ .version }}";
window.WOODPECKER_DOCS = "{{ .docs }}";

1
web/components.d.ts vendored
View file

@ -21,7 +21,6 @@ declare module '@vue/runtime-core' {
Button: typeof import('./src/components/atomic/Button.vue')['default']
Checkbox: typeof import('./src/components/form/Checkbox.vue')['default']
CheckboxesField: typeof import('./src/components/form/CheckboxesField.vue')['default']
copy: typeof import('./src/components/admin/settings/AdminAgentsTab copy.vue')['default']
CronTab: typeof import('./src/components/repo/settings/CronTab.vue')['default']
DeployPipelinePopup: typeof import('./src/components/layout/popups/DeployPipelinePopup.vue')['default']
DocsLink: typeof import('./src/components/atomic/DocsLink.vue')['default']

2
web/src/compositions/useConfig.ts
View file

@ -3,7 +3,6 @@ import { User } from '~/lib/api/types';
declare global {
interface Window {
WOODPECKER_USER: User | undefined;
WOODPECKER_SYNC: boolean | undefined;
WOODPECKER_DOCS: string | undefined;
WOODPECKER_VERSION: string | undefined;
WOODPECKER_CSRF: string | undefined;
@ -13,7 +12,6 @@ declare global {
export default () => ({
user: window.WOODPECKER_USER || null,
syncing: window.WOODPECKER_SYNC || null,
docs: window.WOODPECKER_DOCS || null,
version: window.WOODPECKER_VERSION,
csrf: window.WOODPECKER_CSRF || null,

1
web/src/lib/api/index.ts
View file

@ -20,7 +20,6 @@ import {
type RepoListOptions = {
all?: boolean;
flush?: boolean;
};
type PipelineOptions = {

12
web/src/views/RepoAdd.vue
View file

@ -4,10 +4,6 @@
{{ $t('repo.add') }}
</template>
<template #titleActions>
<Button start-icon="sync" :text="$t('repo.enable.reload')" :is-loading="isReloadingRepos" @click="reloadRepos" />
</template>
<div class="space-y-4">
<ListItem
v-for="repo in searchedRepos"
@ -68,12 +64,6 @@ export default defineComponent({
repos.value = await apiClient.getRepoList({ all: true });
});
const { doSubmit: reloadRepos, isLoading: isReloadingRepos } = useAsyncAction(async () => {
repos.value = undefined;
repos.value = await apiClient.getRepoList({ all: true, flush: true });
notifications.notify({ title: i18n.t('repo.enable.list_reloaded'), type: 'success' });
});
const { doSubmit: activateRepo, isLoading: isActivatingRepo } = useAsyncAction(async (repo: Repo) => {
repoToActivate.value = repo;
await apiClient.activateRepo(repo.owner, repo.name);
@ -85,11 +75,9 @@ export default defineComponent({
const goBack = useRouteBackOrDefault({ name: 'repos' });
return {
isReloadingRepos,
isActivatingRepo,
repoToActivate,
goBack,
reloadRepos,
activateRepo,
searchedRepos,
search,