feat(package): gitea: wire heatmap reindexing

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
2026-06-06 22:01:59 +00:00
parent 2eb23ea7ea
commit c2e0ba200c
9 changed files with 369 additions and 1 deletions
+1
View File
@@ -25,6 +25,7 @@ func newAdminCommand() *cli.Command {
Commands: []*cli.Command{ Commands: []*cli.Command{
newUserCommand(), newUserCommand(),
newRepoSyncReleasesCommand(), newRepoSyncReleasesCommand(),
newHeatmapCommand(),
newRegenerateCommand(), newRegenerateCommand(),
newAuthCommand(), newAuthCommand(),
newSendMailCommand(), newSendMailCommand(),
+131
View File
@@ -0,0 +1,131 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cmd
import (
"context"
"fmt"
"strings"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
repo_service "code.gitea.io/gitea/services/repository"
"github.com/urfave/cli/v3"
)
func newHeatmapCommand() *cli.Command {
return &cli.Command{
Name: "heatmap",
Usage: "Manage heatmap indexes",
Commands: []*cli.Command{
newHeatmapReindexCommand(),
},
}
}
func newHeatmapReindexCommand() *cli.Command {
return &cli.Command{
Name: "reindex",
Usage: "Reindex heatmap contributions from repository default branches",
Action: runHeatmapReindex,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "all",
Usage: "Reindex all repositories",
},
&cli.StringFlag{
Name: "repo",
Usage: "Repository to reindex as owner/name, or repository name when --owner is set",
},
&cli.StringFlag{
Name: "owner",
Usage: "Owner name for --repo",
},
},
}
}
func runHeatmapReindex(ctx context.Context, c *cli.Command) error {
if err := initDB(ctx); err != nil {
return err
}
if err := git.InitSimple(); err != nil {
return err
}
reindexAll := c.Bool("all")
repoName := c.String("repo")
ownerName := c.String("owner")
if reindexAll {
if repoName != "" || ownerName != "" {
return fmt.Errorf("--all cannot be combined with --repo or --owner")
}
return reindexAllHeatmapRepositories(ctx)
}
if repoName == "" {
return fmt.Errorf("either --all or --repo must be provided")
}
ownerName, repoName, err := parseHeatmapRepoSelector(ownerName, repoName)
if err != nil {
return err
}
repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, ownerName, repoName)
if err != nil {
return err
}
return reindexHeatmapRepository(ctx, repo)
}
func parseHeatmapRepoSelector(ownerName, repoName string) (string, string, error) {
if ownerName != "" {
if strings.Contains(repoName, "/") {
return "", "", fmt.Errorf("--repo must be a repository name when --owner is set")
}
return ownerName, repoName, nil
}
ownerName, repoName, ok := strings.Cut(repoName, "/")
if !ok || ownerName == "" || repoName == "" || strings.Contains(repoName, "/") {
return "", "", fmt.Errorf("--repo must be provided as owner/name unless --owner is set")
}
return ownerName, repoName, nil
}
func reindexAllHeatmapRepositories(ctx context.Context) error {
for page := 1; ; page++ {
repos, count, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{
ListOptions: db.ListOptions{
PageSize: repo_model.RepositoryListDefaultPageSize,
Page: page,
},
Private: true,
})
if err != nil {
return fmt.Errorf("SearchRepositoryByName: %w", err)
}
if len(repos) == 0 {
break
}
log.Trace("Processing next %d repos of %d", len(repos), count)
for _, repo := range repos {
if err := reindexHeatmapRepository(ctx, repo); err != nil {
log.Warn("Reindexing heatmap contributions for repo %s failed: %v", repo.FullName(), err)
continue
}
}
}
return nil
}
func reindexHeatmapRepository(ctx context.Context, repo *repo_model.Repository) error {
log.Trace("Reindexing heatmap contributions for repo %s", repo.FullName())
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
return fmt.Errorf("IndexDefaultBranchHeatmapContributions[%s]: %w", repo.FullName(), err)
}
return nil
}
@@ -0,0 +1,154 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cmd
import (
"fmt"
"strings"
"testing"
"time"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/timeutil"
repo_service "code.gitea.io/gitea/services/repository"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHeatmapAdminReindex(t *testing.T) {
repo, commits := prepareHeatmapAdminRepo(t, "heatmap-admin-reindex", []heatmapAdminTestCommit{
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
})
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
contributions := loadHeatmapAdminContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, commits["initial"], contributions[0].CommitSHA)
require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
newCommits := runHeatmapAdminFastImport(t, repo, []heatmapAdminTestCommit{
{Branch: "main", Mark: "forced", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
})
require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
require.NoError(t, reindexAllHeatmapRepositories(t.Context()))
contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 1)
assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
}
func TestHeatmapAdminRepoSelector(t *testing.T) {
owner, repo, err := parseHeatmapRepoSelector("", "user/repo")
require.NoError(t, err)
assert.Equal(t, "user", owner)
assert.Equal(t, "repo", repo)
owner, repo, err = parseHeatmapRepoSelector("user", "repo")
require.NoError(t, err)
assert.Equal(t, "user", owner)
assert.Equal(t, "repo", repo)
_, _, err = parseHeatmapRepoSelector("", "repo")
assert.ErrorContains(t, err, "owner/name")
_, _, err = parseHeatmapRepoSelector("user", "owner/repo")
assert.ErrorContains(t, err, "when --owner is set")
}
type heatmapAdminTestCommit struct {
Branch string
Mark string
Parent string
AuthorName string
AuthorEmail string
CommitterName string
CommitterEmail string
AuthorDate string
}
func prepareHeatmapAdminRepo(t *testing.T, repoName string, commits []heatmapAdminTestCommit) (*repo_model.Repository, map[string]string) {
t.Helper()
require.NoError(t, unittest.PrepareTestDatabase())
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
repo, err := repo_service.CreateRepositoryDirectly(t.Context(), owner, owner, repo_service.CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
require.NoError(t, err)
marks := runHeatmapAdminFastImport(t, repo, commits)
repo.IsEmpty = false
require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
return repo, marks
}
func runHeatmapAdminFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapAdminTestCommit) map[string]string {
t.Helper()
var stream strings.Builder
branchTips := make(map[string]string)
for i, commit := range commits {
mark := fmt.Sprintf(":%d", i+1)
branchRef := "refs/heads/" + commit.Branch
stream.WriteString("commit " + branchRef + "\n")
stream.WriteString("mark " + mark + "\n")
stream.WriteString(heatmapAdminSignatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
stream.WriteString(heatmapAdminSignatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
message := "heatmap admin " + commit.Mark
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
if parentMark := branchTips[commit.Branch]; parentMark != "" {
stream.WriteString("from " + parentMark + "\n")
} else if commit.Parent != "" {
stream.WriteString("from " + commit.Parent + "\n")
}
content := commit.Mark + "\n"
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
branchTips[commit.Branch] = mark
}
require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
WithDir(repo.RepoPath()).
WithStdinCopy(strings.NewReader(stream.String())).
Run(t.Context()))
commitSHAs := make(map[string]string, len(commits))
for _, commit := range commits {
stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
AddOptionFormat("--grep=%s", "heatmap admin "+commit.Mark).
AddDynamicArguments("refs/heads/" + commit.Branch).
WithDir(repo.RepoPath()).
RunStdString(t.Context())
require.NoError(t, err)
commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
}
return commitSHAs
}
func heatmapAdminSignatureLine(kind, name, email, date string) string {
parsed, err := time.Parse(time.RFC3339, date)
if err != nil {
panic(err)
}
return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
}
func loadHeatmapAdminContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
t.Helper()
contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
require.NoError(t, err)
return contributions
}
@@ -318,6 +318,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
defer gitRepo.Close() defer gitRepo.Close()
log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results)) log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
indexHeatmap := false
if len(results) > 0 { if len(results) > 0 {
if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok { if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok {
log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err) log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err)
@@ -330,6 +331,9 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
if result.RefName.IsPull() { if result.RefName.IsPull() {
continue continue
} }
if result.RefName.IsBranch() && result.RefName.BranchName() == m.Repo.DefaultBranch {
indexHeatmap = true
}
// Create reference // Create reference
if result.OldCommitID == "" { if result.OldCommitID == "" {
@@ -391,6 +395,11 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
NewCommitID: newCommitID, NewCommitID: newCommitID,
}, theCommits) }, theCommits)
} }
if indexHeatmap {
if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, m.Repo); err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to index heatmap contributions: %v", m.Repo, err)
}
}
log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo) log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)
isEmpty, err := gitRepo.IsEmpty() isEmpty, err := gitRepo.IsEmpty()
@@ -747,6 +747,9 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newB
} }
notify_service.ChangeDefaultBranch(ctx, repo) notify_service.ChangeDefaultBranch(ctx, repo)
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
}
return nil return nil
} }
@@ -337,6 +337,10 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User,
} }
} }
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
}
return repo, nil return repo, nil
} }
@@ -14,7 +14,9 @@ import (
repo_model "code.gitea.io/gitea/models/repo" repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/git/gitcmd"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -80,9 +82,56 @@ func TestHeatmapIndexIgnoresPusherAndNonDefaultBranch(t *testing.T) {
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID)) assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
} }
func TestHeatmapIndexOnPushDefaultBranch(t *testing.T) {
repo, commits := prepareHeatmapIndexRepo(t, "heatmap-push-default", []heatmapIndexTestCommit{
{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
})
timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
defer timeutil.MockUnset()
require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
assert.Len(t, loadHeatmapContributionsForRepo(t, repo.ID), 1)
newCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
{Branch: "main", Mark: "pushed", Parent: commits["initial"], AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
})
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
{
RefFullName: git.RefNameFromBranch("main"),
OldCommitID: commits["initial"],
NewCommitID: newCommits["pushed"],
PusherID: 1,
RepoUserName: repo.OwnerName,
RepoName: repo.Name,
},
}))
contributions := loadHeatmapContributionsForRepo(t, repo.ID)
require.Len(t, contributions, 2)
assert.Equal(t, newCommits["pushed"], contributions[1].CommitSHA)
require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
featureCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
{Branch: "feature", Mark: "feature-pushed", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
})
require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
{
RefFullName: git.RefNameFromBranch("feature"),
OldCommitID: git.Sha1ObjectFormat.EmptyObjectID().String(),
NewCommitID: featureCommits["feature-pushed"],
PusherID: 1,
RepoUserName: repo.OwnerName,
RepoName: repo.Name,
},
}))
assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
}
type heatmapIndexTestCommit struct { type heatmapIndexTestCommit struct {
Branch string Branch string
Mark string Mark string
Parent string
AuthorName string AuthorName string
AuthorEmail string AuthorEmail string
CommitterName string CommitterName string
@@ -121,6 +170,8 @@ func runFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapI
stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message)) stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
if parentMark := branchTips[commit.Branch]; parentMark != "" { if parentMark := branchTips[commit.Branch]; parentMark != "" {
stream.WriteString("from " + parentMark + "\n") stream.WriteString("from " + parentMark + "\n")
} else if commit.Parent != "" {
stream.WriteString("from " + commit.Parent + "\n")
} }
content := commit.Mark + "\n" content := commit.Mark + "\n"
stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark)) stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
@@ -173,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
} }
} }
return db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) { repo, err = db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
if opts.Mirror { if opts.Mirror {
remoteAddress, err := util.SanitizeURL(opts.CloneAddr) remoteAddress, err := util.SanitizeURL(opts.CloneAddr)
if err != nil { if err != nil {
@@ -255,6 +255,13 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
} }
return repo, nil return repo, nil
}) })
if err != nil {
return nil, err
}
if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
}
return repo, nil
} }
// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. // CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
@@ -167,6 +167,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
} }
} }
indexHeatmap := false
if !opts.IsDelRef() { if !opts.IsDelRef() {
branch := opts.RefFullName.BranchName() branch := opts.RefFullName.BranchName()
@@ -193,6 +194,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil { if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil {
log.Error("DelRepoDivergenceFromCache: %v", err) log.Error("DelRepoDivergenceFromCache: %v", err)
} }
indexHeatmap = true
} else { } else {
if err := DelDivergenceFromCache(repo.ID, branch); err != nil { if err := DelDivergenceFromCache(repo.ID, branch); err != nil {
log.Error("DelDivergenceFromCache: %v", err) log.Error("DelDivergenceFromCache: %v", err)
@@ -222,6 +224,12 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
pushDeleteBranch(ctx, repo, pusher, opts) pushDeleteBranch(ctx, repo, pusher, opts)
} }
if indexHeatmap {
if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
}
}
// Even if user delete a branch on a repository which he didn't watch, he will be watch that. // Even if user delete a branch on a repository which he didn't watch, he will be watch that.
if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil { if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil {
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)