Use a separate admin page to show global stats, remove actions stat (#25062)

Before, Gitea shows the database table stats on the `admin dashboard`
page.

It has some problems:

* `count(*)` is quite heavy. If tables have many records, this blocks
loading the admin page blocks for a long time
* Some users had even reported issues that they can't visit their admin
page because this page causes blocking or `50x error (reverse proxy
timeout)`
* The `actions` stat is not useful. The table is simply too large. Does
it really matter if it contains 1,000,000 rows or 9,999,999 rows?
* The translation `admin.dashboard.statistic_info` is difficult to
maintain.

So, this PR uses a separate page to show the stats and removes the
`actions` stat.


![image](https://github.com/go-gitea/gitea/assets/2114189/babf7c61-b93b-4a62-bfaa-22983636427e)

## ⚠️ BREAKING

The `actions` Prometheus metrics collector has been removed for the
reasons mentioned beforehand.
Please do not rely on its output anymore.
This commit is contained in:
wxiaoguang 2023-06-03 22:03:41 +08:00 committed by GitHub
parent 4486dd39e7
commit 520eb57d76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 57 additions and 53 deletions

View file

@ -21,7 +21,7 @@ import (
type Statistic struct {
Counter struct {
User, Org, PublicKey,
Repo, Watch, Star, Action, Access,
Repo, Watch, Star, Access,
Issue, IssueClosed, IssueOpen,
Comment, Oauth, Follow,
Mirror, Release, AuthSource, Webhook,
@ -55,7 +55,6 @@ func GetStatistic() (stats Statistic) {
stats.Counter.Repo, _ = repo_model.CountRepositories(db.DefaultContext, repo_model.CountRepositoryOptions{})
stats.Counter.Watch, _ = e.Count(new(repo_model.Watch))
stats.Counter.Star, _ = e.Count(new(repo_model.Star))
stats.Counter.Action, _ = db.EstimateCount(db.DefaultContext, new(Action))
stats.Counter.Access, _ = e.Count(new(access_model.Access))
type IssueCount struct {
@ -83,7 +82,7 @@ func GetStatistic() (stats Statistic) {
Find(&stats.Counter.IssueByRepository)
}
issueCounts := []IssueCount{}
var issueCounts []IssueCount
_ = e.Select("COUNT(*) AS count, is_closed").Table("issue").GroupBy("is_closed").Find(&issueCounts)
for _, c := range issueCounts {

View file

@ -9,7 +9,6 @@ import (
"xorm.io/builder"
"xorm.io/xorm"
"xorm.io/xorm/schemas"
)
// DefaultContext is the default context to run xorm queries in
@ -241,30 +240,6 @@ func TableName(bean interface{}) string {
return x.TableName(bean)
}
// EstimateCount returns an estimate of total number of rows in table
func EstimateCount(ctx context.Context, bean interface{}) (int64, error) {
e := GetEngine(ctx)
e.Context(ctx)
var rows int64
var err error
tablename := TableName(bean)
switch x.Dialect().URI().DBType {
case schemas.MYSQL:
_, err = e.Context(ctx).SQL("SELECT table_rows FROM information_schema.tables WHERE tables.table_name = ? AND tables.table_schema = ?;", tablename, x.Dialect().URI().DBName).Get(&rows)
case schemas.POSTGRES:
// the table can live in multiple schemas of a postgres database
// See https://wiki.postgresql.org/wiki/Count_estimate
tablename = x.TableName(bean, true)
_, err = e.Context(ctx).SQL("SELECT reltuples::bigint AS estimate FROM pg_class WHERE oid = ?::regclass;", tablename).Get(&rows)
case schemas.MSSQL:
_, err = e.Context(ctx).SQL("sp_spaceused ?;", tablename).Get(&rows)
default:
return e.Context(ctx).Count(tablename)
}
return rows, err
}
// InTransaction returns true if the engine is in a transaction otherwise return false
func InTransaction(ctx context.Context) bool {
_, ok := inTransaction(ctx)

View file

@ -18,7 +18,6 @@ const namespace = "gitea_"
// exposes gitea metrics for prometheus
type Collector struct {
Accesses *prometheus.Desc
Actions *prometheus.Desc
Attachments *prometheus.Desc
BuildInfo *prometheus.Desc
Comments *prometheus.Desc
@ -56,11 +55,6 @@ func NewCollector() Collector {
"Number of Accesses",
nil, nil,
),
Actions: prometheus.NewDesc(
namespace+"actions",
"Number of Actions",
nil, nil,
),
Attachments: prometheus.NewDesc(
namespace+"attachments",
"Number of Attachments",
@ -207,7 +201,6 @@ func NewCollector() Collector {
// Describe returns all possible prometheus.Desc
func (c Collector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.Accesses
ch <- c.Actions
ch <- c.Attachments
ch <- c.BuildInfo
ch <- c.Comments
@ -246,11 +239,6 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) {
prometheus.GaugeValue,
float64(stats.Counter.Access),
)
ch <- prometheus.MustNewConstMetric(
c.Actions,
prometheus.GaugeValue,
float64(stats.Counter.Action),
)
ch <- prometheus.MustNewConstMetric(
c.Attachments,
prometheus.GaugeValue,

View file

@ -2619,7 +2619,6 @@ dashboard.new_version_hint = Gitea %s is now available, you are running %s. Chec
dashboard.statistic = Summary
dashboard.operations = Maintenance Operations
dashboard.system_status = System Status
dashboard.statistic_info = The Gitea database holds <b>%d</b> users, <b>%d</b> organizations, <b>%d</b> public keys, <b>%d</b> repositories, <b>%d</b> watches, <b>%d</b> stars, ~<b>%d</b> actions, <b>%d</b> accesses, <b>%d</b> issues, <b>%d</b> comments, <b>%d</b> social accounts, <b>%d</b> follows, <b>%d</b> mirrors, <b>%d</b> releases, <b>%d</b> authentication sources, <b>%d</b> webhooks, <b>%d</b> milestones, <b>%d</b> labels, <b>%d</b> hook tasks, <b>%d</b> teams, <b>%d</b> update tasks, <b>%d</b> attachments.
dashboard.operation_name = Operation Name
dashboard.operation_switch = Switch
dashboard.operation_run = Run
@ -3060,6 +3059,8 @@ config.xorm_log_sql = Log SQL
config.get_setting_failed = Get setting %s failed
config.set_setting_failed = Set setting %s failed
monitor.stats = Stats
monitor.cron = Cron Tasks
monitor.name = Name
monitor.schedule = Schedule

View file

@ -8,11 +8,13 @@ import (
"fmt"
"net/http"
"runtime"
"sort"
"time"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/updatechecker"
"code.gitea.io/gitea/modules/web"
@ -26,6 +28,7 @@ const (
tplQueue base.TplName = "admin/queue"
tplStacktrace base.TplName = "admin/stacktrace"
tplQueueManage base.TplName = "admin/queue_manage"
tplStats base.TplName = "admin/stats"
)
var sysStatus struct {
@ -111,7 +114,6 @@ func updateSystemStatus() {
func Dashboard(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("admin.dashboard")
ctx.Data["PageIsAdminDashboard"] = true
ctx.Data["Stats"] = activities_model.GetStatistic()
ctx.Data["NeedUpdate"] = updatechecker.GetNeedUpdate()
ctx.Data["RemoteVersion"] = updatechecker.GetRemoteVersion()
// FIXME: update periodically
@ -126,7 +128,6 @@ func DashboardPost(ctx *context.Context) {
form := web.GetForm(ctx).(*forms.AdminDashboardForm)
ctx.Data["Title"] = ctx.Tr("admin.dashboard")
ctx.Data["PageIsAdminDashboard"] = true
ctx.Data["Stats"] = activities_model.GetStatistic()
updateSystemStatus()
ctx.Data["SysStatus"] = sysStatus
@ -153,3 +154,30 @@ func CronTasks(ctx *context.Context) {
ctx.Data["Entries"] = cron.ListTasks()
ctx.HTML(http.StatusOK, tplCron)
}
func MonitorStats(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("admin.monitor.stats")
ctx.Data["PageIsAdminMonitorStats"] = true
bs, err := json.Marshal(activities_model.GetStatistic().Counter)
if err != nil {
ctx.ServerError("MonitorStats", err)
return
}
statsCounter := map[string]any{}
err = json.Unmarshal(bs, &statsCounter)
if err != nil {
ctx.ServerError("MonitorStats", err)
return
}
statsKeys := make([]string, 0, len(statsCounter))
for k := range statsCounter {
if statsCounter[k] == nil {
continue
}
statsKeys = append(statsKeys, k)
}
sort.Strings(statsKeys)
ctx.Data["StatsKeys"] = statsKeys
ctx.Data["StatsCounter"] = statsCounter
ctx.HTML(http.StatusOK, tplStats)
}

View file

@ -538,8 +538,8 @@ func registerRoutes(m *web.Route) {
// ***** START: Admin *****
m.Group("/admin", func() {
m.Get("", adminReq, admin.Dashboard)
m.Post("", adminReq, web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost)
m.Get("", admin.Dashboard)
m.Post("", web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost)
m.Group("/config", func() {
m.Get("", admin.Config)
@ -548,6 +548,7 @@ func registerRoutes(m *web.Route) {
})
m.Group("/monitor", func() {
m.Get("/stats", admin.MonitorStats)
m.Get("/cron", admin.CronTasks)
m.Get("/stacktrace", admin.Stacktrace)
m.Post("/stacktrace/cancel/{pid}", admin.StacktraceCancel)

View file

@ -5,14 +5,6 @@
<p>{{(.locale.Tr "admin.dashboard.new_version_hint" .RemoteVersion AppVer) | Str2html}}</p>
</div>
{{end}}
<h4 class="ui top attached header">
{{.locale.Tr "admin.dashboard.statistic"}}
</h4>
<div class="ui attached segment">
<p>
{{.locale.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}}
</p>
</div>
<h4 class="ui top attached header">
{{.locale.Tr "admin.dashboard.operations"}}
</h4>

View file

@ -53,6 +53,9 @@
<div class="item">
{{.locale.Tr "admin.monitor"}}
<div class="menu">
<a class="{{if .PageIsAdminMonitorStats}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/stats">
{{.locale.Tr "admin.monitor.stats"}}
</a>
<a class="{{if .PageIsAdminMonitorCron}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/cron">
{{.locale.Tr "admin.monitor.cron"}}
</a>

View file

@ -0,0 +1,17 @@
{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}}
<div class="admin-setting-content">
<h4 class="ui top attached header">
{{.locale.Tr "admin.dashboard.statistic"}}
</h4>
<div class="ui attached table segment">
<table class="ui very basic striped table unstackable">
{{range $statsKey := .StatsKeys}}
<tr>
<td width="200">{{$statsKey}}</td>
<td>{{index $.StatsCounter $statsKey}}</td>
</tr>
{{end}}
</table>
</div>
</div>
{{template "admin/layout_footer" .}}