Add commit status summary table to reduce query from commit status table (#30223)

This PR adds a new table named commit status summary to reduce queries
from the commit status table. After this change, commit status summary
table will be used for the final result, commit status table will be for
details.

---------

Co-authored-by: Jason Song <i@wolfogre.com>
This commit is contained in:
Lunny Xiao 2024-04-12 09:41:50 +08:00 committed by Michael Kriese
parent 5a10eec50f
commit 6495741238
No known key found for this signature in database
GPG key ID: F8D7748549A5986A
6 changed files with 162 additions and 19 deletions

View file

@ -257,30 +257,27 @@ func GetLatestCommitStatus(ctx context.Context, repoID int64, sha string, listOp
}
// GetLatestCommitStatusForPairs returns all statuses with a unique context for a given list of repo-sha pairs
func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHAs map[int64]string, listOptions db.ListOptions) (map[int64][]*CommitStatus, error) {
func GetLatestCommitStatusForPairs(ctx context.Context, repoSHAs []RepoSHA) (map[int64][]*CommitStatus, error) {
type result struct {
Index int64
RepoID int64
SHA string
}
results := make([]result, 0, len(repoIDsToLatestCommitSHAs))
results := make([]result, 0, len(repoSHAs))
getBase := func() *xorm.Session {
return db.GetEngine(ctx).Table(&CommitStatus{})
}
// Create a disjunction of conditions for each repoID and SHA pair
conds := make([]builder.Cond, 0, len(repoIDsToLatestCommitSHAs))
for repoID, sha := range repoIDsToLatestCommitSHAs {
conds = append(conds, builder.Eq{"repo_id": repoID, "sha": sha})
conds := make([]builder.Cond, 0, len(repoSHAs))
for _, repoSHA := range repoSHAs {
conds = append(conds, builder.Eq{"repo_id": repoSHA.RepoID, "sha": repoSHA.SHA})
}
sess := getBase().Where(builder.Or(conds...)).
Select("max( `index` ) as `index`, repo_id").
GroupBy("context_hash, repo_id").OrderBy("max( `index` ) desc")
if !listOptions.IsListAll() {
sess = db.SetSessionPagination(sess, &listOptions)
}
Select("max( `index` ) as `index`, repo_id, sha").
GroupBy("context_hash, repo_id, sha").OrderBy("max( `index` ) desc")
err := sess.Find(&results)
if err != nil {
@ -297,7 +294,7 @@ func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHA
cond := builder.Eq{
"`index`": result.Index,
"repo_id": result.RepoID,
"sha": repoIDsToLatestCommitSHAs[result.RepoID],
"sha": result.SHA,
}
conds = append(conds, cond)
}

View file

@ -0,0 +1,84 @@
// Copyright 2024 Gitea. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"context"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
"xorm.io/builder"
)
// CommitStatusSummary holds the latest commit Status of a single Commit
type CommitStatusSummary struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX UNIQUE(repo_id_sha)"`
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
State api.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"`
}
func init() {
db.RegisterModel(new(CommitStatusSummary))
}
type RepoSHA struct {
RepoID int64
SHA string
}
func GetLatestCommitStatusForRepoAndSHAs(ctx context.Context, repoSHAs []RepoSHA) ([]*CommitStatus, error) {
cond := builder.NewCond()
for _, rs := range repoSHAs {
cond = cond.Or(builder.Eq{"repo_id": rs.RepoID, "sha": rs.SHA})
}
var summaries []CommitStatusSummary
if err := db.GetEngine(ctx).Where(cond).Find(&summaries); err != nil {
return nil, err
}
commitStatuses := make([]*CommitStatus, 0, len(repoSHAs))
for _, summary := range summaries {
commitStatuses = append(commitStatuses, &CommitStatus{
RepoID: summary.RepoID,
SHA: summary.SHA,
State: summary.State,
})
}
return commitStatuses, nil
}
func UpdateCommitStatusSummary(ctx context.Context, repoID int64, sha string) error {
commitStatuses, _, err := GetLatestCommitStatus(ctx, repoID, sha, db.ListOptionsAll)
if err != nil {
return err
}
state := CalcCommitStatus(commitStatuses)
// mysql will return 0 when update a record which state hasn't been changed which behaviour is different from other database,
// so we need to use insert in on duplicate
if setting.Database.Type.IsMySQL() {
_, err := db.GetEngine(ctx).Exec("INSERT INTO commit_status_summary (repo_id,sha,state) VALUES (?,?,?) ON DUPLICATE KEY UPDATE state=?",
repoID, sha, state.State, state.State)
return err
}
if cnt, err := db.GetEngine(ctx).Where("repo_id=? AND sha=?", repoID, sha).
Cols("state").
Update(&CommitStatusSummary{
State: state.State,
}); err != nil {
return err
} else if cnt == 0 {
_, err = db.GetEngine(ctx).Insert(&CommitStatusSummary{
RepoID: repoID,
SHA: sha,
State: state.State,
})
return err
}
return nil
}

View file

@ -578,7 +578,10 @@ var migrations = []Migration{
// Gitea 1.22.0 ends at 294
// v294 -> v295
NewMigration("Add unique index for project issue table", v1_23.AddUniqueIndexForProjectIssue),
// v295 -> v296
NewMigration("Add commit status summary table", v1_23.AddCommitStatusSummary),
}
// GetCurrentDBVersion returns the current db version

View file

@ -0,0 +1,18 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_23 //nolint
import "xorm.io/xorm"
func AddCommitStatusSummary(x *xorm.Engine) error {
type CommitStatusSummary struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX UNIQUE(repo_id_sha)"`
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
State string `xorm:"VARCHAR(7) NOT NULL"`
}
// there is no migrations because if there is no data on this table, it will fall back to get data
// from commit status
return x.Sync2(new(CommitStatusSummary))
}

View file

@ -7,6 +7,7 @@ import (
"context"
"crypto/sha256"
"fmt"
"slices"
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
@ -59,13 +60,19 @@ func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creato
sha = commit.ID.String()
}
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
Repo: repo,
Creator: creator,
SHA: commit.ID,
CommitStatus: status,
if err := db.WithTx(ctx, func(ctx context.Context) error {
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
Repo: repo,
Creator: creator,
SHA: commit.ID,
CommitStatus: status,
}); err != nil {
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
}
return git_model.UpdateCommitStatusSummary(ctx, repo.ID, commit.ID.String())
}); err != nil {
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
return err
}
defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
@ -114,8 +121,35 @@ func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Rep
return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err)
}
var repoSHAs []git_model.RepoSHA
for id, sha := range repoIDsToLatestCommitSHAs {
repoSHAs = append(repoSHAs, git_model.RepoSHA{RepoID: id, SHA: sha})
}
summaryResults, err := git_model.GetLatestCommitStatusForRepoAndSHAs(ctx, repoSHAs)
if err != nil {
return nil, fmt.Errorf("GetLatestCommitStatusForRepoAndSHAs: %v", err)
}
for _, summary := range summaryResults {
for i, repo := range repos {
if repo.ID == summary.RepoID {
results[i] = summary
_ = slices.DeleteFunc(repoSHAs, func(repoSHA git_model.RepoSHA) bool {
return repoSHA.RepoID == repo.ID
})
if results[i].State != "" {
if err := updateCommitStatusCache(ctx, repo.ID, repo.DefaultBranch, results[i].State); err != nil {
log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
}
}
break
}
}
}
// call the database O(1) times to get the commit statuses for all repos
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptionsAll)
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoSHAs)
if err != nil {
return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err)
}

View file

@ -12,6 +12,9 @@ import (
"testing"
auth_model "code.gitea.io/gitea/models/auth"
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
api "code.gitea.io/gitea/modules/structs"
"github.com/stretchr/testify/assert"
@ -90,6 +93,10 @@ func TestPullCreate_CommitStatus(t *testing.T) {
assert.True(t, ok)
assert.Contains(t, cls, statesIcons[status])
}
repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: "user1", Name: "repo1"})
css := unittest.AssertExistsAndLoadBean(t, &git_model.CommitStatusSummary{RepoID: repo1.ID, SHA: commitID})
assert.EqualValues(t, api.CommitStatusWarning, css.State)
})
}