Add retry for migration http/https requests (#9019)

* Add retry for migration http/https requests

* give the more suitable name for retry configuraion items

* fix docs and lint

* Only use retryDownloader when setting > 1
This commit is contained in:
Lunny Xiao 2019-11-16 16:30:06 +08:00 committed by GitHub
parent 6fbfffeeb8
commit 1b7182e5ec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 195 additions and 1 deletions

View file

@ -892,3 +892,9 @@ QUEUE_LENGTH = 1000
; Task queue connection string, available only when `QUEUE_TYPE` is `redis`. ; Task queue connection string, available only when `QUEUE_TYPE` is `redis`.
; If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`. ; If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`.
QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
[migrations]
; Max attempts per http/https request on migrations.
MAX_ATTEMPTS = 3
; Backoff time per http/https request retry (seconds)
RETRY_BACKOFF = 3

View file

@ -585,6 +585,11 @@ Two special environment variables are passed to the render command:
- `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`. - `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`.
- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If there redis needs a password, use `addrs=127.0.0.1:6379 password=123 db=0`. - `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If there redis needs a password, use `addrs=127.0.0.1:6379 password=123 db=0`.
## Migrations (`migrations`)
- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations.
- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds)
## Other (`other`) ## Other (`other`)
- `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.

View file

@ -272,6 +272,11 @@ IS_INPUT_FILE = false
- `QUEUE_LENGTH`: **1000**: 任务队列长度,当 `QUEUE_TYPE``channel` 时有效。 - `QUEUE_LENGTH`: **1000**: 任务队列长度,当 `QUEUE_TYPE``channel` 时有效。
- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串,当 `QUEUE_TYPE``redis` 时有效。如果redis有密码则可以 `addrs=127.0.0.1:6379 password=123 db=0` - `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串,当 `QUEUE_TYPE``redis` 时有效。如果redis有密码则可以 `addrs=127.0.0.1:6379 password=123 db=0`
## Migrations (`migrations`)
- `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。
- `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。
## Other (`other`) ## Other (`other`)
- `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。 - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。

View file

@ -5,7 +5,11 @@
package base package base
import "code.gitea.io/gitea/modules/structs" import (
"time"
"code.gitea.io/gitea/modules/structs"
)
// Downloader downloads the site repo informations // Downloader downloads the site repo informations
type Downloader interface { type Downloader interface {
@ -25,3 +29,148 @@ type DownloaderFactory interface {
New(opts MigrateOptions) (Downloader, error) New(opts MigrateOptions) (Downloader, error)
GitServiceType() structs.GitServiceType GitServiceType() structs.GitServiceType
} }
// RetryDownloader retry the downloads
type RetryDownloader struct {
Downloader
RetryTimes int // the total execute times
RetryDelay int // time to delay seconds
}
// NewRetryDownloader creates a retry downloader
func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
return &RetryDownloader{
Downloader: downloader,
RetryTimes: retryTimes,
RetryDelay: retryDelay,
}
}
// GetRepoInfo returns a repository information with retry
func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
var (
times = d.RetryTimes
repo *Repository
err error
)
for ; times > 0; times-- {
if repo, err = d.Downloader.GetRepoInfo(); err == nil {
return repo, nil
}
time.Sleep(time.Second * time.Duration(d.RetryDelay))
}
return nil, err
}
// GetTopics returns a repository's topics with retry
func (d *RetryDownloader) GetTopics() ([]string, error) {
var (
times = d.RetryTimes
topics []string
err error
)
for ; times > 0; times-- {
if topics, err = d.Downloader.GetTopics(); err == nil {
return topics, nil
}
time.Sleep(time.Second * time.Duration(d.RetryDelay))
}
return nil, err
}
// GetMilestones returns a repository's milestones with retry
func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
var (
times = d.RetryTimes
milestones []*Milestone
err error
)
for ; times > 0; times-- {
if milestones, err = d.Downloader.GetMilestones(); err == nil {
return milestones, nil
}
time.Sleep(time.Second * time.Duration(d.RetryDelay))
}
return nil, err
}
// GetReleases returns a repository's releases with retry
func (d *RetryDownloader) GetReleases() ([]*Release, error) {
var (
times = d.RetryTimes
releases []*Release
err error
)
for ; times > 0; times-- {
if releases, err = d.Downloader.GetReleases(); err == nil {
return releases, nil
}
time.Sleep(time.Second * time.Duration(d.RetryDelay))
}
return nil, err
}
// GetLabels returns a repository's labels with retry
func (d *RetryDownloader) GetLabels() ([]*Label, error) {
var (
times = d.RetryTimes
labels []*Label
err error
)
for ; times > 0; times-- {
if labels, err = d.Downloader.GetLabels(); err == nil {
return labels, nil
}
time.Sleep(time.Second * time.Duration(d.RetryDelay))
}
return nil, err
}
// GetIssues returns a repository's issues with retry
func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
var (
times = d.RetryTimes
issues []*Issue
isEnd bool
err error
)
for ; times > 0; times-- {
if issues, isEnd, err = d.Downloader.GetIssues(page, perPage); err == nil {
return issues, isEnd, nil
}
time.Sleep(time.Second * time.Duration(d.RetryDelay))
}
return nil, false, err
}
// GetComments returns a repository's comments with retry
func (d *RetryDownloader) GetComments(issueNumber int64) ([]*Comment, error) {
var (
times = d.RetryTimes
comments []*Comment
err error
)
for ; times > 0; times-- {
if comments, err = d.Downloader.GetComments(issueNumber); err == nil {
return comments, nil
}
time.Sleep(time.Second * time.Duration(d.RetryDelay))
}
return nil, err
}
// GetPullRequests returns a repository's pull requests with retry
func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, error) {
var (
times = d.RetryTimes
prs []*PullRequest
err error
)
for ; times > 0; times-- {
if prs, err = d.Downloader.GetPullRequests(page, perPage); err == nil {
return prs, nil
}
time.Sleep(time.Second * time.Duration(d.RetryDelay))
}
return nil, err
}

View file

@ -11,6 +11,7 @@ import (
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
) )
@ -63,6 +64,11 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
} }
uploader.gitServiceType = opts.GitServiceType uploader.gitServiceType = opts.GitServiceType
if setting.Migrations.MaxAttempts > 1 {
downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
}
if err := migrateRepository(downloader, uploader, opts); err != nil { if err := migrateRepository(downloader, uploader, opts); err != nil {
if err1 := uploader.Rollback(); err1 != nil { if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1) log.Error("rollback failed: %v", err1)

View file

@ -0,0 +1,22 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
var (
// Migrations settings
Migrations = struct {
MaxAttempts int
RetryBackoff int
}{
MaxAttempts: 3,
RetryBackoff: 3,
}
)
func newMigrationsService() {
sec := Cfg.Section("migrations")
Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts)
Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff)
}

View file

@ -1062,6 +1062,7 @@ func NewServices() {
newRegisterMailService() newRegisterMailService()
newNotifyMailService() newNotifyMailService()
newWebhookService() newWebhookService()
newMigrationsService()
newIndexerService() newIndexerService()
newTaskService() newTaskService()
} }