diff --git a/server/remote/common/utils.go b/server/remote/common/utils.go index 224f03b7b..730c7c0cc 100644 --- a/server/remote/common/utils.go +++ b/server/remote/common/utils.go @@ -23,3 +23,28 @@ func ExtractHostFromCloneURL(cloneURL string) (string, error) { return host, nil } + +// Paginate iterates over a func call until it does not return new items and return it as list +func Paginate[T any](get func(page int) ([]T, error)) ([]T, error) { + items := make([]T, 0, 10) + page := 1 + lenFirstBatch := -1 + + for { + batch, err := get(page) + if err != nil { + return nil, err + } + items = append(items, batch...) + + if page == 1 { + lenFirstBatch = len(batch) + } else if len(batch) < lenFirstBatch || len(batch) == 0 { + break + } + + page++ + } + + return items, nil +} diff --git a/server/remote/common/utils_test.go b/server/remote/common/utils_test.go index 35a400060..81121b379 100644 --- a/server/remote/common/utils_test.go +++ b/server/remote/common/utils_test.go @@ -3,6 +3,7 @@ package common_test import ( "testing" + "github.com/stretchr/testify/assert" "github.com/woodpecker-ci/woodpecker/server/remote/common" ) @@ -16,3 +17,29 @@ func Test_Netrc(t *testing.T) { t.Errorf("Expected host to be git.example.com, got %s", host) } } + +func TestPaginate(t *testing.T) { + apiExec := 0 + apiMock := func(page int) []int { + apiExec++ + switch page { + case 0, 1: + return []int{11, 12, 13} + case 2: + return []int{21, 22, 23} + case 3: + return []int{31, 32} + default: + return []int{} + } + } + + result, _ := common.Paginate(func(page int) ([]int, error) { + return apiMock(page), nil + }) + + assert.EqualValues(t, 3, apiExec) + if assert.Len(t, result, 8) { + assert.EqualValues(t, []int{11, 12, 13, 21, 22, 23, 31, 32}, result) + } +} diff --git a/server/remote/gitea/gitea.go b/server/remote/gitea/gitea.go index d35f08d65..acc3234c9 100644 --- a/server/remote/gitea/gitea.go +++ b/server/remote/gitea/gitea.go @@ -192,10 +192,7 @@ func (c *Gitea) Teams(ctx context.Context, u *model.User) ([]*model.Team, error) return nil, err } - teams := make([]*model.Team, 0, perPage) - - page := 1 - for { + return common.Paginate(func(page int) ([]*model.Team, error) { orgs, _, err := client.ListMyOrgs( gitea.ListOrgsOptions{ ListOptions: gitea.ListOptions{ @@ -204,21 +201,12 @@ func (c *Gitea) Teams(ctx context.Context, u *model.User) ([]*model.Team, error) }, }, ) - if err != nil { - return nil, err - } - + teams := make([]*model.Team, 0, len(orgs)) for _, org := range orgs { teams = append(teams, toTeam(org, c.URL)) } - - if len(orgs) < perPage { - break - } - page++ - } - - return teams, nil + return teams, err + }) } // TeamPerm is not supported by the Gitea driver. @@ -255,17 +243,13 @@ func (c *Gitea) Repo(ctx context.Context, u *model.User, id model.RemoteID, owne // Repos returns a list of all repositories for the Gitea account, including // organization repositories. func (c *Gitea) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error) { - repos := make([]*model.Repo, 0, perPage) - client, err := c.newClientToken(ctx, u.Token) if err != nil { return nil, err } - // Gitea SDK forces us to read repo list paginated. - page := 1 - for { - all, _, err := client.ListMyRepos( + return common.Paginate(func(page int) ([]*model.Repo, error) { + repos, _, err := client.ListMyRepos( gitea.ListReposOptions{ ListOptions: gitea.ListOptions{ Page: page, @@ -273,22 +257,12 @@ func (c *Gitea) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error) }, }, ) - if err != nil { - return nil, err + result := make([]*model.Repo, 0, len(repos)) + for _, repo := range repos { + result = append(result, toRepo(repo)) } - - for _, repo := range all { - repos = append(repos, toRepo(repo)) - } - - if len(all) < perPage { - break - } - // Last page was not empty so more repos may be available - continue loop. - page++ - } - - return repos, nil + return result, err + }) } // Perm returns the user permissions for the named Gitea repository. @@ -462,27 +436,17 @@ func (c *Gitea) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]s return nil, err } - branches := make([]string, 0) - - page := 1 - - for { - giteaBranches, _, err := client.ListRepoBranches(r.Owner, r.Name, gitea.ListRepoBranchesOptions{ - ListOptions: gitea.ListOptions{ - Page: page, - }, - }) - if err != nil { - return nil, err - } - if len(giteaBranches) > 0 { - for _, branch := range giteaBranches { - branches = append(branches, branch.Name) - } - page++ - } else { - break + branches, err := common.Paginate(func(page int) ([]string, error) { + branches, _, err := client.ListRepoBranches(r.Owner, r.Name, + gitea.ListRepoBranchesOptions{ListOptions: gitea.ListOptions{Page: page}}) + result := make([]string, 0, len(branches)) + for i := range branches { + result[i] = branches[i].Name } + return result, err + }) + if err != nil { + return nil, err } return branches, nil