Add repo-sync-releases admin command (#3254)

* Add repo-sync-releases admin command

Will help recovering corrupted database, see #3247

* Load repos in chunks of 10, exit with error if unable to get a list, scan private repos, fix typo

* Fix debug output about num releases

* Introduce RepositoryListDefaultPageSize constant, set to 64

Use it from the new admin command

* Use RepositoryListDefaultPageSize in more places

* Document RepositoryListDefaultPageSize
This commit is contained in:
Sandro Santilli 2017-12-31 15:45:46 +01:00 committed by Lauris BH
parent ae9cc8f972
commit 8cd987af0c
5 changed files with 85 additions and 3 deletions

View file

@ -8,7 +8,9 @@ package cmd
import ( import (
"fmt" "fmt"
"code.gitea.io/git"
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"github.com/urfave/cli" "github.com/urfave/cli"
@ -24,6 +26,7 @@ to make automatic initialization process more smoothly`,
Subcommands: []cli.Command{ Subcommands: []cli.Command{
subcmdCreateUser, subcmdCreateUser,
subcmdChangePassword, subcmdChangePassword,
subcmdRepoSyncReleases,
}, },
} }
@ -76,6 +79,12 @@ to make automatic initialization process more smoothly`,
}, },
}, },
} }
subcmdRepoSyncReleases = cli.Command{
Name: "repo-sync-releases",
Usage: "Synchronize repository releases with tags",
Action: runRepoSyncReleases,
}
) )
func runChangePassword(c *cli.Context) error { func runChangePassword(c *cli.Context) error {
@ -145,3 +154,69 @@ func runCreateUser(c *cli.Context) error {
fmt.Printf("New user '%s' has been successfully created!\n", c.String("name")) fmt.Printf("New user '%s' has been successfully created!\n", c.String("name"))
return nil return nil
} }
func runRepoSyncReleases(c *cli.Context) error {
setting.NewContext()
models.LoadConfigs()
setting.NewXORMLogService(false)
if err := models.SetEngine(); err != nil {
return fmt.Errorf("models.SetEngine: %v", err)
}
log.Trace("Synchronizing repository releases (this may take a while)")
for page := 1; ; page++ {
repos, count, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
Page: page,
PageSize: models.RepositoryListDefaultPageSize,
Private: true,
})
if err != nil {
log.Fatal(4, "SearchRepositoryByName: %v", err)
return err
}
if len(repos) == 0 {
break
}
log.Trace("Processing next %d repos of %d", len(repos), count)
for _, repo := range repos {
log.Trace("Synchronizing repo %s with path %s", repo.FullName(), repo.RepoPath())
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
log.Warn("OpenRepository: %v", err)
continue
}
oldnum, err := models.GetReleaseCountByRepoID(repo.ID,
models.FindReleasesOptions{
IncludeDrafts: false,
IncludeTags: true,
})
if err != nil {
log.Warn(" GetReleaseCountByRepoID: %v", err)
}
log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)
if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
log.Warn(" SyncReleasesWithTags: %v", err)
continue
}
count, err = models.GetReleaseCountByRepoID(repo.ID,
models.FindReleasesOptions{
IncludeDrafts: false,
IncludeTags: true,
})
if err != nil {
log.Warn(" GetReleaseCountByRepoID: %v", err)
continue
}
log.Trace(" repo %s releases synchronized to tags: from %d to %d",
repo.FullName(), oldnum, count)
}
}
return nil
}

View file

@ -29,7 +29,7 @@ func populateIssueIndexer() error {
for page := 1; ; page++ { for page := 1; ; page++ {
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{ repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
Page: page, Page: page,
PageSize: 10, PageSize: RepositoryListDefaultPageSize,
OrderBy: SearchOrderByID, OrderBy: SearchOrderByID,
Private: true, Private: true,
Collaborate: util.OptionalBoolFalse, Collaborate: util.OptionalBoolFalse,

View file

@ -31,7 +31,7 @@ func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error {
// For the sake of SQLite3, we can't use x.Iterate here. // For the sake of SQLite3, we can't use x.Iterate here.
offset := 0 offset := 0
pageSize := 20 pageSize := models.RepositoryListDefaultPageSize
for { for {
repos := make([]*models.Repository, 0, pageSize) repos := make([]*models.Repository, 0, pageSize)
if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil { if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil {

View file

@ -81,7 +81,7 @@ func populateRepoIndexer() error {
for page := 1; ; page++ { for page := 1; ; page++ {
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{ repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
Page: page, Page: page,
PageSize: 10, PageSize: RepositoryListDefaultPageSize,
OrderBy: SearchOrderByID, OrderBy: SearchOrderByID,
Private: true, Private: true,
}) })

View file

@ -13,6 +13,13 @@ import (
"github.com/go-xorm/builder" "github.com/go-xorm/builder"
) )
// RepositoryListDefaultPageSize is the default number of repositories
// to load in memory when running administrative tasks on all (or almost
// all) of them.
// The number should be low enough to avoid filling up all RAM with
// repository data...
const RepositoryListDefaultPageSize = 64
// RepositoryList contains a list of repositories // RepositoryList contains a list of repositories
type RepositoryList []*Repository type RepositoryList []*Repository