From 593c0d9abcce5fcb52ce5ab71f26c482882af5a0 Mon Sep 17 00:00:00 2001 From: yukkop Date: Sat, 6 Jun 2026 19:44:53 +0000 Subject: [PATCH] feat(`package`): `gitea`: index heatmap commits by author date --- .../models/activities/heatmap_contribution.go | 48 +++++ .../services/repository/heatmap_indexer.go | 109 ++++++++++++ .../repository/heatmap_indexer_test.go | 167 ++++++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 package/gitea/source/services/repository/heatmap_indexer.go create mode 100644 package/gitea/source/services/repository/heatmap_indexer_test.go diff --git a/package/gitea/source/models/activities/heatmap_contribution.go b/package/gitea/source/models/activities/heatmap_contribution.go index 0c91fafb..e552a8fb 100644 --- a/package/gitea/source/models/activities/heatmap_contribution.go +++ b/package/gitea/source/models/activities/heatmap_contribution.go @@ -24,6 +24,13 @@ type HeatmapContribution struct { UpdatedUnix timeutil.TimeStamp `xorm:"updated"` } +// HeatmapContributionIdentity identifies the counted contribution row that remains valid after reindexing. +type HeatmapContributionIdentity struct { + RepoID int64 + CommitSHA string + UserID int64 +} + func init() { db.RegisterModel(new(HeatmapContribution)) } @@ -71,3 +78,44 @@ func UpsertHeatmapContribution(ctx context.Context, contribution *HeatmapContrib func CountHeatmapContributions(ctx context.Context, userID int64) (int64, error) { return db.GetEngine(ctx).Where("user_id=?", userID).Count(new(HeatmapContribution)) } + +// DeleteStaleHeatmapContributions removes indexed rows not found in the current eligible contribution identities. +func DeleteStaleHeatmapContributions(ctx context.Context, repoID int64, identities []HeatmapContributionIdentity) error { + if len(identities) == 0 { + _, err := db.GetEngine(ctx).Where("repo_id=?", repoID).Delete(new(HeatmapContribution)) + return err + } + + current := make(map[HeatmapContributionIdentity]struct{}, len(identities)) + for _, identity := range identities { + current[identity] = struct{}{} + } + + contributions, err := FindHeatmapContributionsByRepo(ctx, repoID) + if err != nil { + return err + } + for _, contribution := range contributions { + identity := HeatmapContributionIdentity{ + RepoID: contribution.RepoID, + CommitSHA: contribution.CommitSHA, + UserID: contribution.UserID, + } + if _, ok := current[identity]; ok { + continue + } + if _, err := db.GetEngine(ctx).ID(contribution.ID).Delete(new(HeatmapContribution)); err != nil { + return err + } + } + return nil +} + +// FindHeatmapContributionsByRepo returns indexed contribution rows for a repository. +func FindHeatmapContributionsByRepo(ctx context.Context, repoID int64) ([]*HeatmapContribution, error) { + contributions := make([]*HeatmapContribution, 0) + return contributions, db.GetEngine(ctx). + Where("repo_id=?", repoID). + OrderBy("author_unix ASC, commit_sha ASC, user_id ASC"). + Find(&contributions) +} diff --git a/package/gitea/source/services/repository/heatmap_indexer.go b/package/gitea/source/services/repository/heatmap_indexer.go new file mode 100644 index 00000000..bc530b78 --- /dev/null +++ b/package/gitea/source/services/repository/heatmap_indexer.go @@ -0,0 +1,109 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "strings" + + activities_model "code.gitea.io/gitea/models/activities" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/timeutil" +) + +const heatmapIndexCommitsPageSize = 100 + +// IndexDefaultBranchHeatmapContributions indexes commit author-date heatmap contributions reachable from repo's default branch. +func IndexDefaultBranchHeatmapContributions(ctx context.Context, repo *repo_model.Repository) error { + if repo == nil || repo.ID == 0 { + return nil + } + if repo.DefaultBranch == "" || repo.IsEmpty { + return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil) + } + + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return err + } + defer closer.Close() + + head, err := gitRepo.GetBranchCommit(repo.DefaultBranch) + if git.IsErrNotExist(err) { + return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil) + } + if err != nil { + return err + } + + currentIdentities := make([]activities_model.HeatmapContributionIdentity, 0) + now := timeutil.TimeStampNow() + for page := 1; ; page++ { + commits, err := head.CommitsByRange(page, heatmapIndexCommitsPageSize, "", "", "") + if err != nil { + return err + } + if len(commits) == 0 { + break + } + + for _, commit := range commits { + commitSHA := commit.ID.String() + if commit.Author == nil || commit.Author.Email == "" { + continue + } + + authorUnix := timeutil.TimeStamp(commit.Author.When.Unix()) + if authorUnix > now { + continue + } + + userID, ok, err := getHeatmapAuthorUserIDByEmail(ctx, commit.Author.Email) + if err != nil { + return err + } + if !ok { + continue + } + + if err := activities_model.UpsertHeatmapContribution(ctx, &activities_model.HeatmapContribution{ + UserID: userID, + RepoID: repo.ID, + CommitSHA: commitSHA, + AuthorEmail: strings.ToLower(commit.Author.Email), + AuthorUnix: authorUnix, + }); err != nil { + return err + } + currentIdentities = append(currentIdentities, activities_model.HeatmapContributionIdentity{ + RepoID: repo.ID, + CommitSHA: commitSHA, + UserID: userID, + }) + } + + if len(commits) < heatmapIndexCommitsPageSize { + break + } + } + + return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, currentIdentities) +} + +func getHeatmapAuthorUserIDByEmail(ctx context.Context, email string) (int64, bool, error) { + address, err := user_model.GetEmailAddressByEmail(ctx, email) + if user_model.IsErrEmailAddressNotExist(err) { + return 0, false, nil + } + if err != nil { + return 0, false, err + } + if !address.IsActivated { + return 0, false, nil + } + return address.UID, true, nil +} diff --git a/package/gitea/source/services/repository/heatmap_indexer_test.go b/package/gitea/source/services/repository/heatmap_indexer_test.go new file mode 100644 index 00000000..0835c45b --- /dev/null +++ b/package/gitea/source/services/repository/heatmap_indexer_test.go @@ -0,0 +1,167 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "fmt" + "strings" + "testing" + "time" + + activities_model "code.gitea.io/gitea/models/activities" + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/gitcmd" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHeatmapIndexDefaultBranchAuthorDates(t *testing.T) { + repo, commits := prepareHeatmapIndexRepo(t, "heatmap-author-dates", []heatmapIndexTestCommit{ + {Branch: "main", Mark: "old", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"}, + {Branch: "main", Mark: "unknown", AuthorName: "Unknown", AuthorEmail: "unknown@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"}, + {Branch: "main", Mark: "future", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2030-01-15T12:00:00Z"}, + }) + + timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC)) + defer timeutil.MockUnset() + + require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo)) + require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo)) + + contributions := loadHeatmapContributionsForRepo(t, repo.ID) + require.Len(t, contributions, 1) + assert.Equal(t, int64(2), contributions[0].UserID) + assert.Equal(t, commits["old"], contributions[0].CommitSHA) + assert.Equal(t, "user2@example.com", contributions[0].AuthorEmail) + assert.Equal(t, timeutil.TimeStamp(1579089600), contributions[0].AuthorUnix) + + emailAddress, err := user_model.GetEmailAddressByEmail(t.Context(), "user2@example.com") + require.NoError(t, err) + emailAddress.UID = 1 + require.NoError(t, user_model.UpdateEmailAddress(t.Context(), emailAddress)) + require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo)) + + contributions = loadHeatmapContributionsForRepo(t, repo.ID) + require.Len(t, contributions, 1) + assert.Equal(t, int64(1), contributions[0].UserID) + assert.Equal(t, commits["old"], contributions[0].CommitSHA) +} + +func TestHeatmapIndexIgnoresPusherAndNonDefaultBranch(t *testing.T) { + repo, commits := prepareHeatmapIndexRepo(t, "heatmap-exclusions", []heatmapIndexTestCommit{ + {Branch: "main", Mark: "author-user2", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"}, + {Branch: "main", Mark: "author-user1", AuthorName: "User One", AuthorEmail: "user1@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-16T12:00:00Z"}, + {Branch: "feature", Mark: "feature-only", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-17T12:00:00Z"}, + }) + + timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC)) + defer timeutil.MockUnset() + + require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo)) + + contributions := loadHeatmapContributionsForRepo(t, repo.ID) + require.Len(t, contributions, 2) + assert.Equal(t, int64(2), contributions[0].UserID) + assert.Equal(t, commits["author-user2"], contributions[0].CommitSHA) + assert.Equal(t, int64(1), contributions[1].UserID) + assert.Equal(t, commits["author-user1"], contributions[1].CommitSHA) + for _, contribution := range contributions { + assert.NotEqual(t, commits["feature-only"], contribution.CommitSHA) + } + + deleteMainRef(t, repo) + require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo)) + assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID)) +} + +type heatmapIndexTestCommit struct { + Branch string + Mark string + AuthorName string + AuthorEmail string + CommitterName string + CommitterEmail string + AuthorDate string +} + +func prepareHeatmapIndexRepo(t *testing.T, repoName string, commits []heatmapIndexTestCommit) (*repo_model.Repository, map[string]string) { + t.Helper() + require.NoError(t, unittest.PrepareTestDatabase()) + require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{})) + + owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo, err := CreateRepositoryDirectly(t.Context(), owner, owner, CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true) + require.NoError(t, err) + + marks := runFastImport(t, repo, commits) + repo.IsEmpty = false + require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty")) + return repo, marks +} + +func runFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapIndexTestCommit) map[string]string { + t.Helper() + + var stream strings.Builder + branchTips := make(map[string]string) + for i, commit := range commits { + mark := fmt.Sprintf(":%d", i+1) + branchRef := "refs/heads/" + commit.Branch + stream.WriteString("commit " + branchRef + "\n") + stream.WriteString("mark " + mark + "\n") + stream.WriteString(signatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate)) + stream.WriteString(signatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate)) + message := "heatmap " + commit.Mark + stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message)) + if parentMark := branchTips[commit.Branch]; parentMark != "" { + stream.WriteString("from " + parentMark + "\n") + } + content := commit.Mark + "\n" + stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark)) + stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content)) + branchTips[commit.Branch] = mark + } + + require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-"). + WithDir(repo.RepoPath()). + WithStdinCopy(strings.NewReader(stream.String())). + Run(t.Context())) + + commitSHAs := make(map[string]string, len(commits)) + for _, commit := range commits { + stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings"). + AddOptionFormat("--grep=%s", "heatmap "+commit.Mark). + AddDynamicArguments("refs/heads/" + commit.Branch). + WithDir(repo.RepoPath()). + RunStdString(t.Context()) + require.NoError(t, err) + commitSHAs[commit.Mark] = strings.TrimSpace(stdout) + } + return commitSHAs +} + +func signatureLine(kind, name, email, date string) string { + parsed, err := time.Parse(time.RFC3339, date) + if err != nil { + panic(err) + } + return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix()) +} + +func loadHeatmapContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution { + t.Helper() + contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID) + require.NoError(t, err) + return contributions +} + +func deleteMainRef(t *testing.T, repo *repo_model.Repository) { + t.Helper() + require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context())) +}