diff --git a/package/gitea/source/cmd/admin.go b/package/gitea/source/cmd/admin.go index c0e21731..e5bde560 100644 --- a/package/gitea/source/cmd/admin.go +++ b/package/gitea/source/cmd/admin.go @@ -25,6 +25,7 @@ func newAdminCommand() *cli.Command { Commands: []*cli.Command{ newUserCommand(), newRepoSyncReleasesCommand(), + newHeatmapCommand(), newRegenerateCommand(), newAuthCommand(), newSendMailCommand(), diff --git a/package/gitea/source/cmd/admin_heatmap.go b/package/gitea/source/cmd/admin_heatmap.go new file mode 100644 index 00000000..ac2546f4 --- /dev/null +++ b/package/gitea/source/cmd/admin_heatmap.go @@ -0,0 +1,131 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cmd + +import ( + "context" + "fmt" + "strings" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + repo_service "code.gitea.io/gitea/services/repository" + + "github.com/urfave/cli/v3" +) + +func newHeatmapCommand() *cli.Command { + return &cli.Command{ + Name: "heatmap", + Usage: "Manage heatmap indexes", + Commands: []*cli.Command{ + newHeatmapReindexCommand(), + }, + } +} + +func newHeatmapReindexCommand() *cli.Command { + return &cli.Command{ + Name: "reindex", + Usage: "Reindex heatmap contributions from repository default branches", + Action: runHeatmapReindex, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "all", + Usage: "Reindex all repositories", + }, + &cli.StringFlag{ + Name: "repo", + Usage: "Repository to reindex as owner/name, or repository name when --owner is set", + }, + &cli.StringFlag{ + Name: "owner", + Usage: "Owner name for --repo", + }, + }, + } +} + +func runHeatmapReindex(ctx context.Context, c *cli.Command) error { + if err := initDB(ctx); err != nil { + return err + } + if err := git.InitSimple(); err != nil { + return err + } + + reindexAll := c.Bool("all") + repoName := c.String("repo") + ownerName := c.String("owner") + if reindexAll { + if repoName != "" || ownerName != "" { + return fmt.Errorf("--all cannot be combined with --repo or --owner") + } + return reindexAllHeatmapRepositories(ctx) + } + if repoName == "" { + return fmt.Errorf("either --all or --repo must be provided") + } + + ownerName, repoName, err := parseHeatmapRepoSelector(ownerName, repoName) + if err != nil { + return err + } + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, ownerName, repoName) + if err != nil { + return err + } + return reindexHeatmapRepository(ctx, repo) +} + +func parseHeatmapRepoSelector(ownerName, repoName string) (string, string, error) { + if ownerName != "" { + if strings.Contains(repoName, "/") { + return "", "", fmt.Errorf("--repo must be a repository name when --owner is set") + } + return ownerName, repoName, nil + } + + ownerName, repoName, ok := strings.Cut(repoName, "/") + if !ok || ownerName == "" || repoName == "" || strings.Contains(repoName, "/") { + return "", "", fmt.Errorf("--repo must be provided as owner/name unless --owner is set") + } + return ownerName, repoName, nil +} + +func reindexAllHeatmapRepositories(ctx context.Context) error { + for page := 1; ; page++ { + repos, count, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{ + ListOptions: db.ListOptions{ + PageSize: repo_model.RepositoryListDefaultPageSize, + Page: page, + }, + Private: true, + }) + if err != nil { + return fmt.Errorf("SearchRepositoryByName: %w", err) + } + if len(repos) == 0 { + break + } + log.Trace("Processing next %d repos of %d", len(repos), count) + for _, repo := range repos { + if err := reindexHeatmapRepository(ctx, repo); err != nil { + log.Warn("Reindexing heatmap contributions for repo %s failed: %v", repo.FullName(), err) + continue + } + } + } + return nil +} + +func reindexHeatmapRepository(ctx context.Context, repo *repo_model.Repository) error { + log.Trace("Reindexing heatmap contributions for repo %s", repo.FullName()) + if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil { + return fmt.Errorf("IndexDefaultBranchHeatmapContributions[%s]: %w", repo.FullName(), err) + } + return nil +} diff --git a/package/gitea/source/cmd/admin_heatmap_test.go b/package/gitea/source/cmd/admin_heatmap_test.go new file mode 100644 index 00000000..831fe10b --- /dev/null +++ b/package/gitea/source/cmd/admin_heatmap_test.go @@ -0,0 +1,154 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cmd + +import ( + "fmt" + "strings" + "testing" + "time" + + activities_model "code.gitea.io/gitea/models/activities" + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/gitcmd" + "code.gitea.io/gitea/modules/timeutil" + repo_service "code.gitea.io/gitea/services/repository" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHeatmapAdminReindex(t *testing.T) { + repo, commits := prepareHeatmapAdminRepo(t, "heatmap-admin-reindex", []heatmapAdminTestCommit{ + {Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"}, + }) + + timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC)) + defer timeutil.MockUnset() + + require.NoError(t, reindexHeatmapRepository(t.Context(), repo)) + require.NoError(t, reindexHeatmapRepository(t.Context(), repo)) + contributions := loadHeatmapAdminContributionsForRepo(t, repo.ID) + require.Len(t, contributions, 1) + assert.Equal(t, commits["initial"], contributions[0].CommitSHA) + + require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context())) + newCommits := runHeatmapAdminFastImport(t, repo, []heatmapAdminTestCommit{ + {Branch: "main", Mark: "forced", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"}, + }) + require.NoError(t, reindexHeatmapRepository(t.Context(), repo)) + contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID) + require.Len(t, contributions, 1) + assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA) + + require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{})) + require.NoError(t, reindexAllHeatmapRepositories(t.Context())) + contributions = loadHeatmapAdminContributionsForRepo(t, repo.ID) + require.Len(t, contributions, 1) + assert.Equal(t, newCommits["forced"], contributions[0].CommitSHA) +} + +func TestHeatmapAdminRepoSelector(t *testing.T) { + owner, repo, err := parseHeatmapRepoSelector("", "user/repo") + require.NoError(t, err) + assert.Equal(t, "user", owner) + assert.Equal(t, "repo", repo) + + owner, repo, err = parseHeatmapRepoSelector("user", "repo") + require.NoError(t, err) + assert.Equal(t, "user", owner) + assert.Equal(t, "repo", repo) + + _, _, err = parseHeatmapRepoSelector("", "repo") + assert.ErrorContains(t, err, "owner/name") + _, _, err = parseHeatmapRepoSelector("user", "owner/repo") + assert.ErrorContains(t, err, "when --owner is set") +} + +type heatmapAdminTestCommit struct { + Branch string + Mark string + Parent string + AuthorName string + AuthorEmail string + CommitterName string + CommitterEmail string + AuthorDate string +} + +func prepareHeatmapAdminRepo(t *testing.T, repoName string, commits []heatmapAdminTestCommit) (*repo_model.Repository, map[string]string) { + t.Helper() + require.NoError(t, unittest.PrepareTestDatabase()) + require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{})) + + owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo, err := repo_service.CreateRepositoryDirectly(t.Context(), owner, owner, repo_service.CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true) + require.NoError(t, err) + + marks := runHeatmapAdminFastImport(t, repo, commits) + repo.IsEmpty = false + require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty")) + return repo, marks +} + +func runHeatmapAdminFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapAdminTestCommit) map[string]string { + t.Helper() + + var stream strings.Builder + branchTips := make(map[string]string) + for i, commit := range commits { + mark := fmt.Sprintf(":%d", i+1) + branchRef := "refs/heads/" + commit.Branch + stream.WriteString("commit " + branchRef + "\n") + stream.WriteString("mark " + mark + "\n") + stream.WriteString(heatmapAdminSignatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate)) + stream.WriteString(heatmapAdminSignatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate)) + message := "heatmap admin " + commit.Mark + stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message)) + if parentMark := branchTips[commit.Branch]; parentMark != "" { + stream.WriteString("from " + parentMark + "\n") + } else if commit.Parent != "" { + stream.WriteString("from " + commit.Parent + "\n") + } + content := commit.Mark + "\n" + stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark)) + stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content)) + branchTips[commit.Branch] = mark + } + + require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-"). + WithDir(repo.RepoPath()). + WithStdinCopy(strings.NewReader(stream.String())). + Run(t.Context())) + + commitSHAs := make(map[string]string, len(commits)) + for _, commit := range commits { + stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings"). + AddOptionFormat("--grep=%s", "heatmap admin "+commit.Mark). + AddDynamicArguments("refs/heads/" + commit.Branch). + WithDir(repo.RepoPath()). + RunStdString(t.Context()) + require.NoError(t, err) + commitSHAs[commit.Mark] = strings.TrimSpace(stdout) + } + return commitSHAs +} + +func heatmapAdminSignatureLine(kind, name, email, date string) string { + parsed, err := time.Parse(time.RFC3339, date) + if err != nil { + panic(err) + } + return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix()) +} + +func loadHeatmapAdminContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution { + t.Helper() + contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID) + require.NoError(t, err) + return contributions +} diff --git a/package/gitea/source/services/mirror/mirror_pull.go b/package/gitea/source/services/mirror/mirror_pull.go index 9ce35f9e..c182a645 100644 --- a/package/gitea/source/services/mirror/mirror_pull.go +++ b/package/gitea/source/services/mirror/mirror_pull.go @@ -318,6 +318,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { defer gitRepo.Close() log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results)) + indexHeatmap := false if len(results) > 0 { if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok { log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err) @@ -330,6 +331,9 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { if result.RefName.IsPull() { continue } + if result.RefName.IsBranch() && result.RefName.BranchName() == m.Repo.DefaultBranch { + indexHeatmap = true + } // Create reference if result.OldCommitID == "" { @@ -391,6 +395,11 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { NewCommitID: newCommitID, }, theCommits) } + if indexHeatmap { + if err := repo_service.IndexDefaultBranchHeatmapContributions(ctx, m.Repo); err != nil { + log.Error("SyncMirrors [repo: %-v]: failed to index heatmap contributions: %v", m.Repo, err) + } + } log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo) isEmpty, err := gitRepo.IsEmpty() diff --git a/package/gitea/source/services/repository/branch.go b/package/gitea/source/services/repository/branch.go index dc17e4f2..1a5c2392 100644 --- a/package/gitea/source/services/repository/branch.go +++ b/package/gitea/source/services/repository/branch.go @@ -747,6 +747,9 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newB } notify_service.ChangeDefaultBranch(ctx, repo) + if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil { + log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err) + } return nil } diff --git a/package/gitea/source/services/repository/create.go b/package/gitea/source/services/repository/create.go index b0b1f4e7..20a46c9c 100644 --- a/package/gitea/source/services/repository/create.go +++ b/package/gitea/source/services/repository/create.go @@ -337,6 +337,10 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User, } } + if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil { + return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err) + } + return repo, nil } diff --git a/package/gitea/source/services/repository/heatmap_indexer_test.go b/package/gitea/source/services/repository/heatmap_indexer_test.go index 0835c45b..4993bf59 100644 --- a/package/gitea/source/services/repository/heatmap_indexer_test.go +++ b/package/gitea/source/services/repository/heatmap_indexer_test.go @@ -14,7 +14,9 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/gitcmd" + repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/timeutil" "github.com/stretchr/testify/assert" @@ -80,9 +82,56 @@ func TestHeatmapIndexIgnoresPusherAndNonDefaultBranch(t *testing.T) { assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID)) } +func TestHeatmapIndexOnPushDefaultBranch(t *testing.T) { + repo, commits := prepareHeatmapIndexRepo(t, "heatmap-push-default", []heatmapIndexTestCommit{ + {Branch: "main", Mark: "initial", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"}, + }) + + timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC)) + defer timeutil.MockUnset() + + require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo)) + assert.Len(t, loadHeatmapContributionsForRepo(t, repo.ID), 1) + + newCommits := runFastImport(t, repo, []heatmapIndexTestCommit{ + {Branch: "main", Mark: "pushed", Parent: commits["initial"], AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"}, + }) + require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{ + { + RefFullName: git.RefNameFromBranch("main"), + OldCommitID: commits["initial"], + NewCommitID: newCommits["pushed"], + PusherID: 1, + RepoUserName: repo.OwnerName, + RepoName: repo.Name, + }, + })) + + contributions := loadHeatmapContributionsForRepo(t, repo.ID) + require.Len(t, contributions, 2) + assert.Equal(t, newCommits["pushed"], contributions[1].CommitSHA) + + require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{})) + featureCommits := runFastImport(t, repo, []heatmapIndexTestCommit{ + {Branch: "feature", Mark: "feature-pushed", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-17T12:00:00Z"}, + }) + require.NoError(t, pushUpdates([]*repo_module.PushUpdateOptions{ + { + RefFullName: git.RefNameFromBranch("feature"), + OldCommitID: git.Sha1ObjectFormat.EmptyObjectID().String(), + NewCommitID: featureCommits["feature-pushed"], + PusherID: 1, + RepoUserName: repo.OwnerName, + RepoName: repo.Name, + }, + })) + assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID)) +} + type heatmapIndexTestCommit struct { Branch string Mark string + Parent string AuthorName string AuthorEmail string CommitterName string @@ -121,6 +170,8 @@ func runFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapI stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message)) if parentMark := branchTips[commit.Branch]; parentMark != "" { stream.WriteString("from " + parentMark + "\n") + } else if commit.Parent != "" { + stream.WriteString("from " + commit.Parent + "\n") } content := commit.Mark + "\n" stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark)) diff --git a/package/gitea/source/services/repository/migrate.go b/package/gitea/source/services/repository/migrate.go index a51791ed..0a27e7d7 100644 --- a/package/gitea/source/services/repository/migrate.go +++ b/package/gitea/source/services/repository/migrate.go @@ -173,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } } - return db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) { + repo, err = db.WithTx2(ctx, func(ctx context.Context) (*repo_model.Repository, error) { if opts.Mirror { remoteAddress, err := util.SanitizeURL(opts.CloneAddr) if err != nil { @@ -255,6 +255,13 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } return repo, nil }) + if err != nil { + return nil, err + } + if err = IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil { + return nil, fmt.Errorf("IndexDefaultBranchHeatmapContributions: %w", err) + } + return repo, nil } // CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. diff --git a/package/gitea/source/services/repository/push.go b/package/gitea/source/services/repository/push.go index 7addd4eb..46e5fdff 100644 --- a/package/gitea/source/services/repository/push.go +++ b/package/gitea/source/services/repository/push.go @@ -167,6 +167,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { } } + indexHeatmap := false if !opts.IsDelRef() { branch := opts.RefFullName.BranchName() @@ -193,6 +194,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil { log.Error("DelRepoDivergenceFromCache: %v", err) } + indexHeatmap = true } else { if err := DelDivergenceFromCache(repo.ID, branch); err != nil { log.Error("DelDivergenceFromCache: %v", err) @@ -222,6 +224,12 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { pushDeleteBranch(ctx, repo, pusher, opts) } + if indexHeatmap { + if err := IndexDefaultBranchHeatmapContributions(ctx, repo); err != nil { + log.Error("IndexDefaultBranchHeatmapContributions[%s]: %v", repo.FullName(), err) + } + } + // Even if user delete a branch on a repository which he didn't watch, he will be watch that. if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil { log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)