feat(package): gitea: wire heatmap reindexing

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-06 22:01:59 +00:00
parent 2eb23ea7ea
commit c2e0ba200c
9 changed files with 369 additions and 1 deletions
@@ -25,6 +25,7 @@ func newAdminCommand() *cli.Command {
 		Commands: []*cli.Command{
 			newUserCommand(),
 			newRepoSyncReleasesCommand(),
+			newHeatmapCommand(),
 			newRegenerateCommand(),
 			newAuthCommand(),
 			newSendMailCommand(),
@@ -0,0 +1,131 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	repo_service "code.gitea.io/gitea/services/repository"
+
+	"github.com/urfave/cli/v3"
+)
+
+func newHeatmapCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "heatmap",
+		Usage: "Manage heatmap indexes",
+		Commands: []*cli.Command{
+			newHeatmapReindexCommand(),
+		},
+	}
+}
+
+func newHeatmapReindexCommand() *cli.Command {
+	return &cli.Command{
+		Name:   "reindex",
+		Usage:  "Reindex heatmap contributions from repository default branches",
+		Action: runHeatmapReindex,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:  "all",
+				Usage: "Reindex all repositories",
+			},
+			&cli.StringFlag{
+				Name:  "repo",
+				Usage: "Repository to reindex as owner/name, or repository name when --owner is set",
+			},
+			&cli.StringFlag{
+				Name:  "owner",
+				Usage: "Owner name for --repo",
+			},
+		},
+	}
+}
+
+func runHeatmapReindex(ctx context.Context, c *cli.Command) error {
+	if err := initDB(ctx); err != nil {
+		return err
+	}
+	if err := git.InitSimple(); err != nil {
+		return err
+	}
+
+	reindexAll := c.Bool("all")
+	repoName := c.String("repo")
+	ownerName := c.String("owner")
+	if reindexAll {
+		if repoName != "" || ownerName != "" {
+			return fmt.Errorf("--all cannot be combined with --repo or --owner")
+		}
+		return reindexAllHeatmapRepositories(ctx)
+	}
+	if repoName == "" {
+		return fmt.Errorf("either --all or --repo must be provided")
+	}
+
+	ownerName, repoName, err := parseHeatmapRepoSelector(ownerName, repoName)
+	if err != nil {
+		return err
+	}
+	repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, ownerName, repoName)
+	if err != nil {
+		return err
+	}
+	return reindexHeatmapRepository(ctx, repo)
+}
+
+func parseHeatmapRepoSelector(ownerName, repoName string) (string, string, error) {
+	if ownerName != "" {
+		if strings.Contains(repoName, "/") {
+			return "", "", fmt.Errorf("--repo must be a repository name when --owner is set")
+		}
+		return ownerName, repoName, nil
+	}
+
+	ownerName, repoName, ok := strings.Cut(repoName, "/")
+	if !ok || ownerName == "" || repoName == "" || strings.Contains(repoName, "/") {
+		return "", "", fmt.Errorf("--repo must be provided as owner/name unless --owner is set")
+	}
+	return ownerName, repoName, nil
+}
+
+func reindexAllHeatmapRepositories(ctx context.Context) error {
+	for page := 1; ; page++ {
+		repos, count, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{
+			ListOptions: db.ListOptions{
+				PageSize: repo_model.RepositoryListDefaultPageSize,
+				Page:     page,
+			},
+			Private: true,
+		})
+		if err != nil {
+			return fmt.Errorf("SearchRepositoryByName: %w", err)
+		}
+		if len(repos) == 0 {
+			break
+		}
+		log.Trace("Processing next %d repos of %d", len(repos), count)
+		for _, repo := range repos {
+			if err := reindexHeatmapRepository(ctx, repo); err != nil {
+				log.Warn("Reindexing heatmap contributions for repo %s failed: %v", repo.FullName(), err)
+				continue
+			}
+		}
+	}
+	return nil
+}
+
+func reindexHeatmapRepository(ctx context.Context, repo *repo_model.Repository) error {
+	log.Trace("Reindexing heatmap contributions for repo %s", repo.FullName())
+	if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
+		return fmt.Errorf("IndexDefaultBranchHeatmapContributions[%s]: %w", repo.FullName(), err)
+	}
+	return nil
+}
@@ -0,0 +1,154 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cmd
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	activities_model "code.gitea.io/gitea/models/activities"
+	"code.gitea.io/gitea/models/db"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/unittest"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git/gitcmd"
+	"code.gitea.io/gitea/modules/timeutil"
+	repo_service "code.gitea.io/gitea/services/repository"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHeatmapAdminReindex(t *testing.T) {
+	repo, commits := prepareHeatmapAdminRepo(t, "heatmap-admin-reindex", []heatmapAdminTestCommit{
+		{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
+	})
+
+	timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
+	defer timeutil.MockUnset()
+
+	require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
+	require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
+	contributions := loadHeatmapAdminContributionsForRepo(t, repo.ID)
+	require.Len(t, contributions, 1)
+	assert.Equal(t, commits["initial"], contributions[0].CommitSHA)
+
+	require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
+	newCommits := runHeatmapAdminFastImport(t, repo, []heatmapAdminTestCommit{
+		{Branch: "main", Mark: "forced", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
+	})
+	require.NoError(t, reindexHeatmapRepository(t.Context(), repo))
+	contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
+	require.Len(t, contributions, 1)
+	assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
+
+	require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
+	require.NoError(t, reindexAllHeatmapRepositories(t.Context()))
+	contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID)
+	require.Len(t, contributions, 1)
+	assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA)
+}
+
+func TestHeatmapAdminRepoSelector(t *testing.T) {
+	owner, repo, err := parseHeatmapRepoSelector("", "user/repo")
+	require.NoError(t, err)
+	assert.Equal(t, "user", owner)
+	assert.Equal(t, "repo", repo)
+
+	owner, repo, err = parseHeatmapRepoSelector("user", "repo")
+	require.NoError(t, err)
+	assert.Equal(t, "user", owner)
+	assert.Equal(t, "repo", repo)
+
+	_, _, err = parseHeatmapRepoSelector("", "repo")
+	assert.ErrorContains(t, err, "owner/name")
+	_, _, err = parseHeatmapRepoSelector("user", "owner/repo")
+	assert.ErrorContains(t, err, "when --owner is set")
+}
+
+type heatmapAdminTestCommit struct {
+	Branch         string
+	Mark           string
+	Parent         string
+	AuthorName     string
+	AuthorEmail    string
+	CommitterName  string
+	CommitterEmail string
+	AuthorDate     string
+}
+
+func prepareHeatmapAdminRepo(t *testing.T, repoName string, commits []heatmapAdminTestCommit) (*repo_model.Repository, map[string]string) {
+	t.Helper()
+	require.NoError(t, unittest.PrepareTestDatabase())
+	require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
+
+	owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
+	repo, err := repo_service.CreateRepositoryDirectly(t.Context(), owner, owner, repo_service.CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
+	require.NoError(t, err)
+
+	marks := runHeatmapAdminFastImport(t, repo, commits)
+	repo.IsEmpty = false
+	require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
+	return repo, marks
+}
+
+func runHeatmapAdminFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapAdminTestCommit) map[string]string {
+	t.Helper()
+
+	var stream strings.Builder
+	branchTips := make(map[string]string)
+	for i, commit := range commits {
+		mark := fmt.Sprintf(":%d", i+1)
+		branchRef := "refs/heads/" + commit.Branch
+		stream.WriteString("commit " + branchRef + "\n")
+		stream.WriteString("mark " + mark + "\n")
+		stream.WriteString(heatmapAdminSignatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
+		stream.WriteString(heatmapAdminSignatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
+		message := "heatmap admin " + commit.Mark
+		stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
+		if parentMark := branchTips[commit.Branch]; parentMark != "" {
+			stream.WriteString("from " + parentMark + "\n")
+		} else if commit.Parent != "" {
+			stream.WriteString("from " + commit.Parent + "\n")
+		}
+		content := commit.Mark + "\n"
+		stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
+		stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
+		branchTips[commit.Branch] = mark
+	}
+
+	require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
+		WithDir(repo.RepoPath()).
+		WithStdinCopy(strings.NewReader(stream.String())).
+		Run(t.Context()))
+
+	commitSHAs := make(map[string]string, len(commits))
+	for _, commit := range commits {
+		stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
+			AddOptionFormat("--grep=%s", "heatmap admin "+commit.Mark).
+			AddDynamicArguments("refs/heads/" + commit.Branch).
+			WithDir(repo.RepoPath()).
+			RunStdString(t.Context())
+		require.NoError(t, err)
+		commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
+	}
+	return commitSHAs
+}
+
+func heatmapAdminSignatureLine(kind, name, email, date string) string {
+	parsed, err := time.Parse(time.RFC3339, date)
+	if err != nil {
+		panic(err)
+	}
+	return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
+}
+
+func loadHeatmapAdminContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
+	t.Helper()
+	contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
+	require.NoError(t, err)
+	return contributions
+}
@@ -318,6 +318,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
 	defer gitRepo.Close()

 	log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
+	indexHeatmap := false
 	if len(results) > 0 {
 		if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok {
 			log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err)
@@ -330,6 +331,9 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
 		if result.RefName.IsPull() {
 			continue
 		}
+		if result.RefName.IsBranch() && result.RefName.BranchName() == m.Repo.DefaultBranch {
+			indexHeatmap = true
+		}

 		// Create reference
 		if result.OldCommitID == "" {
@@ -391,6 +395,11 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
 			NewCommitID: newCommitID,
 		}, theCommits)
 	}
+	if indexHeatmap {
+		if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, m.Repo); err != nil {
+			log.Error("SyncMirrors [repo: %-v]: failed to index heatmap contributions: %v", m.Repo, err)
+		}
+	}
 	log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)

 	isEmpty, err := gitRepo.IsEmpty()
@@ -747,6 +747,9 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newB
 	}

 	notify_service.ChangeDefaultBranch(ctx, repo)
+	if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
+		log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
+	}

 	return nil
 }
@@ -337,6 +337,10 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User,
 		}
 	}

+	if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
+		return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
+	}
+
 	return repo, nil
 }

@@ -14,7 +14,9 @@ import (
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/git/gitcmd"
+	repo_module "code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/timeutil"

 	"github.com/stretchr/testify/assert"
@@ -80,9 +82,56 @@ func TestHeatmapIndexIgnoresPusherAndNonDefaultBranch(t *testing.T) {
 	assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
 }

+func TestHeatmapIndexOnPushDefaultBranch(t *testing.T) {
+	repo, commits := prepareHeatmapIndexRepo(t, "heatmap-push-default", []heatmapIndexTestCommit{
+		{Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
+	})
+
+	timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
+	defer timeutil.MockUnset()
+
+	require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
+	assert.Len(t, loadHeatmapContributionsForRepo(t, repo.ID), 1)
+
+	newCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
+		{Branch: "main", Mark: "pushed", Parent: commits["initial"], AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
+	})
+	require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
+		{
+			RefFullName:  git.RefNameFromBranch("main"),
+			OldCommitID:  commits["initial"],
+			NewCommitID:  newCommits["pushed"],
+			PusherID:     1,
+			RepoUserName: repo.OwnerName,
+			RepoName:     repo.Name,
+		},
+	}))
+
+	contributions := loadHeatmapContributionsForRepo(t, repo.ID)
+	require.Len(t, contributions, 2)
+	assert.Equal(t, newCommits["pushed"], contributions[1].CommitSHA)
+
+	require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
+	featureCommits := runFastImport(t, repo, []heatmapIndexTestCommit{
+		{Branch: "feature", Mark: "feature-pushed", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
+	})
+	require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{
+		{
+			RefFullName:  git.RefNameFromBranch("feature"),
+			OldCommitID:  git.Sha1ObjectFormat.EmptyObjectID().String(),
+			NewCommitID:  featureCommits["feature-pushed"],
+			PusherID:     1,
+			RepoUserName: repo.OwnerName,
+			RepoName:     repo.Name,
+		},
+	}))
+	assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
+}
+
 type heatmapIndexTestCommit struct {
 	Branch         string
 	Mark           string
+	Parent         string
 	AuthorName     string
 	AuthorEmail    string
 	CommitterName  string
@@ -121,6 +170,8 @@ func runFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapI
 		stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
 		if parentMark := branchTips[commit.Branch]; parentMark != "" {
 			stream.WriteString("from " + parentMark + "\n")
+		} else if commit.Parent != "" {
+			stream.WriteString("from " + commit.Parent + "\n")
 		}
 		content := commit.Mark + "\n"
 		stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
@@ -173,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
 		}
 	}

-	return db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
+	repo, err = db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) {
 		if opts.Mirror {
 			remoteAddress, err := util.SanitizeURL(opts.CloneAddr)
 			if err != nil {
@@ -255,6 +255,13 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
 		}
 		return repo, nil
 	})
+	if err != nil {
+		return nil, err
+	}
+	if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
+		return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err)
+	}
+	return repo, nil
 }

 // CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
@@ -167,6 +167,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
 				}
 			}

+			indexHeatmap := false
 			if !opts.IsDelRef() {
 				branch := opts.RefFullName.BranchName()

@@ -193,6 +194,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
 					if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil {
 						log.Error("DelRepoDivergenceFromCache: %v", err)
 					}
+					indexHeatmap = true
 				} else {
 					if err := DelDivergenceFromCache(repo.ID, branch); err != nil {
 						log.Error("DelDivergenceFromCache: %v", err)
@@ -222,6 +224,12 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
 				pushDeleteBranch(ctx, repo, pusher, opts)
 			}

+			if indexHeatmap {
+				if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil {
+					log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err)
+				}
+			}
+
 			// Even if user delete a branch on a repository which he didn't watch, he will be watch that.
 			if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil {
 				log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)