From 520eb57d7642a5fca3df319e5b5d1c7c9018087c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 3 Jun 2023 22:03:41 +0800 Subject: [PATCH] Use a separate admin page to show global stats, remove `actions` stat (#25062) Before, Gitea shows the database table stats on the `admin dashboard` page. It has some problems: * `count(*)` is quite heavy. If tables have many records, this blocks loading the admin page blocks for a long time * Some users had even reported issues that they can't visit their admin page because this page causes blocking or `50x error (reverse proxy timeout)` * The `actions` stat is not useful. The table is simply too large. Does it really matter if it contains 1,000,000 rows or 9,999,999 rows? * The translation `admin.dashboard.statistic_info` is difficult to maintain. So, this PR uses a separate page to show the stats and removes the `actions` stat. ![image](https://github.com/go-gitea/gitea/assets/2114189/babf7c61-b93b-4a62-bfaa-22983636427e) ## :warning: BREAKING The `actions` Prometheus metrics collector has been removed for the reasons mentioned beforehand. Please do not rely on its output anymore. --- models/activities/statistic.go | 5 ++--- models/db/context.go | 25 ------------------------- modules/metrics/collector.go | 12 ------------ options/locale/locale_en-US.ini | 3 ++- routers/web/admin/admin.go | 32 ++++++++++++++++++++++++++++++-- routers/web/web.go | 5 +++-- templates/admin/dashboard.tmpl | 8 -------- templates/admin/navbar.tmpl | 3 +++ templates/admin/stats.tmpl | 17 +++++++++++++++++ 9 files changed, 57 insertions(+), 53 deletions(-) create mode 100644 templates/admin/stats.tmpl diff --git a/models/activities/statistic.go b/models/activities/statistic.go index 138f4d8fe9..9d379cd0c4 100644 --- a/models/activities/statistic.go +++ b/models/activities/statistic.go @@ -21,7 +21,7 @@ import ( type Statistic struct { Counter struct { User, Org, PublicKey, - Repo, Watch, Star, Action, Access, + Repo, Watch, Star, Access, Issue, IssueClosed, IssueOpen, Comment, Oauth, Follow, Mirror, Release, AuthSource, Webhook, @@ -55,7 +55,6 @@ func GetStatistic() (stats Statistic) { stats.Counter.Repo, _ = repo_model.CountRepositories(db.DefaultContext, repo_model.CountRepositoryOptions{}) stats.Counter.Watch, _ = e.Count(new(repo_model.Watch)) stats.Counter.Star, _ = e.Count(new(repo_model.Star)) - stats.Counter.Action, _ = db.EstimateCount(db.DefaultContext, new(Action)) stats.Counter.Access, _ = e.Count(new(access_model.Access)) type IssueCount struct { @@ -83,7 +82,7 @@ func GetStatistic() (stats Statistic) { Find(&stats.Counter.IssueByRepository) } - issueCounts := []IssueCount{} + var issueCounts []IssueCount _ = e.Select("COUNT(*) AS count, is_closed").Table("issue").GroupBy("is_closed").Find(&issueCounts) for _, c := range issueCounts { diff --git a/models/db/context.go b/models/db/context.go index 670f6272aa..59be1e1389 100644 --- a/models/db/context.go +++ b/models/db/context.go @@ -9,7 +9,6 @@ import ( "xorm.io/builder" "xorm.io/xorm" - "xorm.io/xorm/schemas" ) // DefaultContext is the default context to run xorm queries in @@ -241,30 +240,6 @@ func TableName(bean interface{}) string { return x.TableName(bean) } -// EstimateCount returns an estimate of total number of rows in table -func EstimateCount(ctx context.Context, bean interface{}) (int64, error) { - e := GetEngine(ctx) - e.Context(ctx) - - var rows int64 - var err error - tablename := TableName(bean) - switch x.Dialect().URI().DBType { - case schemas.MYSQL: - _, err = e.Context(ctx).SQL("SELECT table_rows FROM information_schema.tables WHERE tables.table_name = ? AND tables.table_schema = ?;", tablename, x.Dialect().URI().DBName).Get(&rows) - case schemas.POSTGRES: - // the table can live in multiple schemas of a postgres database - // See https://wiki.postgresql.org/wiki/Count_estimate - tablename = x.TableName(bean, true) - _, err = e.Context(ctx).SQL("SELECT reltuples::bigint AS estimate FROM pg_class WHERE oid = ?::regclass;", tablename).Get(&rows) - case schemas.MSSQL: - _, err = e.Context(ctx).SQL("sp_spaceused ?;", tablename).Get(&rows) - default: - return e.Context(ctx).Count(tablename) - } - return rows, err -} - // InTransaction returns true if the engine is in a transaction otherwise return false func InTransaction(ctx context.Context) bool { _, ok := inTransaction(ctx) diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 94699c161c..33678256c3 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -18,7 +18,6 @@ const namespace = "gitea_" // exposes gitea metrics for prometheus type Collector struct { Accesses *prometheus.Desc - Actions *prometheus.Desc Attachments *prometheus.Desc BuildInfo *prometheus.Desc Comments *prometheus.Desc @@ -56,11 +55,6 @@ func NewCollector() Collector { "Number of Accesses", nil, nil, ), - Actions: prometheus.NewDesc( - namespace+"actions", - "Number of Actions", - nil, nil, - ), Attachments: prometheus.NewDesc( namespace+"attachments", "Number of Attachments", @@ -207,7 +201,6 @@ func NewCollector() Collector { // Describe returns all possible prometheus.Desc func (c Collector) Describe(ch chan<- *prometheus.Desc) { ch <- c.Accesses - ch <- c.Actions ch <- c.Attachments ch <- c.BuildInfo ch <- c.Comments @@ -246,11 +239,6 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { prometheus.GaugeValue, float64(stats.Counter.Access), ) - ch <- prometheus.MustNewConstMetric( - c.Actions, - prometheus.GaugeValue, - float64(stats.Counter.Action), - ) ch <- prometheus.MustNewConstMetric( c.Attachments, prometheus.GaugeValue, diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index bf6e4b7524..517d829c84 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2619,7 +2619,6 @@ dashboard.new_version_hint = Gitea %s is now available, you are running %s. Chec dashboard.statistic = Summary dashboard.operations = Maintenance Operations dashboard.system_status = System Status -dashboard.statistic_info = The Gitea database holds %d users, %d organizations, %d public keys, %d repositories, %d watches, %d stars, ~%d actions, %d accesses, %d issues, %d comments, %d social accounts, %d follows, %d mirrors, %d releases, %d authentication sources, %d webhooks, %d milestones, %d labels, %d hook tasks, %d teams, %d update tasks, %d attachments. dashboard.operation_name = Operation Name dashboard.operation_switch = Switch dashboard.operation_run = Run @@ -3060,6 +3059,8 @@ config.xorm_log_sql = Log SQL config.get_setting_failed = Get setting %s failed config.set_setting_failed = Set setting %s failed +monitor.stats = Stats + monitor.cron = Cron Tasks monitor.name = Name monitor.schedule = Schedule diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 1ada4deefc..797ba8798d 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -8,11 +8,13 @@ import ( "fmt" "net/http" "runtime" + "sort" "time" activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/updatechecker" "code.gitea.io/gitea/modules/web" @@ -26,6 +28,7 @@ const ( tplQueue base.TplName = "admin/queue" tplStacktrace base.TplName = "admin/stacktrace" tplQueueManage base.TplName = "admin/queue_manage" + tplStats base.TplName = "admin/stats" ) var sysStatus struct { @@ -111,7 +114,6 @@ func updateSystemStatus() { func Dashboard(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.dashboard") ctx.Data["PageIsAdminDashboard"] = true - ctx.Data["Stats"] = activities_model.GetStatistic() ctx.Data["NeedUpdate"] = updatechecker.GetNeedUpdate() ctx.Data["RemoteVersion"] = updatechecker.GetRemoteVersion() // FIXME: update periodically @@ -126,7 +128,6 @@ func DashboardPost(ctx *context.Context) { form := web.GetForm(ctx).(*forms.AdminDashboardForm) ctx.Data["Title"] = ctx.Tr("admin.dashboard") ctx.Data["PageIsAdminDashboard"] = true - ctx.Data["Stats"] = activities_model.GetStatistic() updateSystemStatus() ctx.Data["SysStatus"] = sysStatus @@ -153,3 +154,30 @@ func CronTasks(ctx *context.Context) { ctx.Data["Entries"] = cron.ListTasks() ctx.HTML(http.StatusOK, tplCron) } + +func MonitorStats(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("admin.monitor.stats") + ctx.Data["PageIsAdminMonitorStats"] = true + bs, err := json.Marshal(activities_model.GetStatistic().Counter) + if err != nil { + ctx.ServerError("MonitorStats", err) + return + } + statsCounter := map[string]any{} + err = json.Unmarshal(bs, &statsCounter) + if err != nil { + ctx.ServerError("MonitorStats", err) + return + } + statsKeys := make([]string, 0, len(statsCounter)) + for k := range statsCounter { + if statsCounter[k] == nil { + continue + } + statsKeys = append(statsKeys, k) + } + sort.Strings(statsKeys) + ctx.Data["StatsKeys"] = statsKeys + ctx.Data["StatsCounter"] = statsCounter + ctx.HTML(http.StatusOK, tplStats) +} diff --git a/routers/web/web.go b/routers/web/web.go index da6064257b..f5037a848e 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -538,8 +538,8 @@ func registerRoutes(m *web.Route) { // ***** START: Admin ***** m.Group("/admin", func() { - m.Get("", adminReq, admin.Dashboard) - m.Post("", adminReq, web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost) + m.Get("", admin.Dashboard) + m.Post("", web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost) m.Group("/config", func() { m.Get("", admin.Config) @@ -548,6 +548,7 @@ func registerRoutes(m *web.Route) { }) m.Group("/monitor", func() { + m.Get("/stats", admin.MonitorStats) m.Get("/cron", admin.CronTasks) m.Get("/stacktrace", admin.Stacktrace) m.Post("/stacktrace/cancel/{pid}", admin.StacktraceCancel) diff --git a/templates/admin/dashboard.tmpl b/templates/admin/dashboard.tmpl index e7d986ee17..20cf3ba7f4 100644 --- a/templates/admin/dashboard.tmpl +++ b/templates/admin/dashboard.tmpl @@ -5,14 +5,6 @@

{{(.locale.Tr "admin.dashboard.new_version_hint" .RemoteVersion AppVer) | Str2html}}

{{end}} -

- {{.locale.Tr "admin.dashboard.statistic"}} -

-
-

- {{.locale.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}} -

-

{{.locale.Tr "admin.dashboard.operations"}}

diff --git a/templates/admin/navbar.tmpl b/templates/admin/navbar.tmpl index eff5f1d83a..777fe29924 100644 --- a/templates/admin/navbar.tmpl +++ b/templates/admin/navbar.tmpl @@ -53,6 +53,9 @@
{{.locale.Tr "admin.monitor"}}
+ + {{.locale.Tr "admin.monitor.stats"}} + {{.locale.Tr "admin.monitor.cron"}} diff --git a/templates/admin/stats.tmpl b/templates/admin/stats.tmpl new file mode 100644 index 0000000000..c755969cb1 --- /dev/null +++ b/templates/admin/stats.tmpl @@ -0,0 +1,17 @@ +{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}} +
+

+ {{.locale.Tr "admin.dashboard.statistic"}} +

+
+ + {{range $statsKey := .StatsKeys}} + + + + + {{end}} +
{{$statsKey}}{{index $.StatsCounter $statsKey}}
+
+
+{{template "admin/layout_footer" .}}