feat(package): gitea: index heatmap commits by author date

2026-06-06 19:44:53 +00:00
parent 53dfd60b4c
commit 593c0d9abc
3 changed files with 324 additions and 0 deletions
@@ -24,6 +24,13 @@ type HeatmapContribution struct {
 	UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
 }

+// HeatmapContributionIdentity identifies the counted contribution row that remains valid after reindexing.
+type HeatmapContributionIdentity struct {
+	RepoID    int64
+	CommitSHA string
+	UserID    int64
+}
+
 func init() {
 	db.RegisterModel(new(HeatmapContribution))
 }
@@ -71,3 +78,44 @@ func UpsertHeatmapContribution(ctx context.Context, contribution *HeatmapContrib
 func CountHeatmapContributions(ctx context.Context, userID int64) (int64, error) {
 	return db.GetEngine(ctx).Where("user_id=?", userID).Count(new(HeatmapContribution))
 }
+
+// DeleteStaleHeatmapContributions removes indexed rows not found in the current eligible contribution identities.
+func DeleteStaleHeatmapContributions(ctx context.Context, repoID int64, identities []HeatmapContributionIdentity) error {
+	if len(identities) == 0 {
+		_, err := db.GetEngine(ctx).Where("repo_id=?", repoID).Delete(new(HeatmapContribution))
+		return err
+	}
+
+	current := make(map[HeatmapContributionIdentity]struct{}, len(identities))
+	for _, identity := range identities {
+		current[identity] = struct{}{}
+	}
+
+	contributions, err := FindHeatmapContributionsByRepo(ctx, repoID)
+	if err != nil {
+		return err
+	}
+	for _, contribution := range contributions {
+		identity := HeatmapContributionIdentity{
+			RepoID:    contribution.RepoID,
+			CommitSHA: contribution.CommitSHA,
+			UserID:    contribution.UserID,
+		}
+		if _, ok := current[identity]; ok {
+			continue
+		}
+		if _, err := db.GetEngine(ctx).ID(contribution.ID).Delete(new(HeatmapContribution)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// FindHeatmapContributionsByRepo returns indexed contribution rows for a repository.
+func FindHeatmapContributionsByRepo(ctx context.Context, repoID int64) ([]*HeatmapContribution, error) {
+	contributions := make([]*HeatmapContribution, 0)
+	return contributions, db.GetEngine(ctx).
+		Where("repo_id=?", repoID).
+		OrderBy("author_unix ASC, commit_sha ASC, user_id ASC").
+		Find(&contributions)
+}
@@ -0,0 +1,109 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package repository
+
+import (
+	"context"
+	"strings"
+
+	activities_model "code.gitea.io/gitea/models/activities"
+	repo_model "code.gitea.io/gitea/models/repo"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/gitrepo"
+	"code.gitea.io/gitea/modules/timeutil"
+)
+
+const heatmapIndexCommitsPageSize = 100
+
+// IndexDefaultBranchHeatmapContributions indexes commit author-date heatmap contributions reachable from repo's default branch.
+func IndexDefaultBranchHeatmapContributions(ctx context.Context, repo *repo_model.Repository) error {
+	if repo == nil || repo.ID == 0 {
+		return nil
+	}
+	if repo.DefaultBranch == "" || repo.IsEmpty {
+		return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
+	}
+
+	gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
+	if err != nil {
+		return err
+	}
+	defer closer.Close()
+
+	head, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
+	if git.IsErrNotExist(err) {
+		return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, nil)
+	}
+	if err != nil {
+		return err
+	}
+
+	currentIdentities := make([]activities_model.HeatmapContributionIdentity, 0)
+	now := timeutil.TimeStampNow()
+	for page := 1; ; page++ {
+		commits, err := head.CommitsByRange(page, heatmapIndexCommitsPageSize, "", "", "")
+		if err != nil {
+			return err
+		}
+		if len(commits) == 0 {
+			break
+		}
+
+		for _, commit := range commits {
+			commitSHA := commit.ID.String()
+			if commit.Author == nil || commit.Author.Email == "" {
+				continue
+			}
+
+			authorUnix := timeutil.TimeStamp(commit.Author.When.Unix())
+			if authorUnix > now {
+				continue
+			}
+
+			userID, ok, err := getHeatmapAuthorUserIDByEmail(ctx, commit.Author.Email)
+			if err != nil {
+				return err
+			}
+			if !ok {
+				continue
+			}
+
+			if err := activities_model.UpsertHeatmapContribution(ctx, &activities_model.HeatmapContribution{
+				UserID:      userID,
+				RepoID:      repo.ID,
+				CommitSHA:   commitSHA,
+				AuthorEmail: strings.ToLower(commit.Author.Email),
+				AuthorUnix:  authorUnix,
+			}); err != nil {
+				return err
+			}
+			currentIdentities = append(currentIdentities, activities_model.HeatmapContributionIdentity{
+				RepoID:    repo.ID,
+				CommitSHA: commitSHA,
+				UserID:    userID,
+			})
+		}
+
+		if len(commits) < heatmapIndexCommitsPageSize {
+			break
+		}
+	}
+
+	return activities_model.DeleteStaleHeatmapContributions(ctx, repo.ID, currentIdentities)
+}
+
+func getHeatmapAuthorUserIDByEmail(ctx context.Context, email string) (int64, bool, error) {
+	address, err := user_model.GetEmailAddressByEmail(ctx, email)
+	if user_model.IsErrEmailAddressNotExist(err) {
+		return 0, false, nil
+	}
+	if err != nil {
+		return 0, false, err
+	}
+	if !address.IsActivated {
+		return 0, false, nil
+	}
+	return address.UID, true, nil
+}
@@ -0,0 +1,167 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package repository
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	activities_model "code.gitea.io/gitea/models/activities"
+	"code.gitea.io/gitea/models/db"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/unittest"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git/gitcmd"
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHeatmapIndexDefaultBranchAuthorDates(t *testing.T) {
+	repo, commits := prepareHeatmapIndexRepo(t, "heatmap-author-dates", []heatmapIndexTestCommit{
+		{Branch: "main", Mark: "old", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
+		{Branch: "main", Mark: "unknown", AuthorName: "Unknown", AuthorEmail: "unknown@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
+		{Branch: "main", Mark: "future", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2030-01-15T12:00:00Z"},
+	})
+
+	timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
+	defer timeutil.MockUnset()
+
+	require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
+	require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
+
+	contributions := loadHeatmapContributionsForRepo(t, repo.ID)
+	require.Len(t, contributions, 1)
+	assert.Equal(t, int64(2), contributions[0].UserID)
+	assert.Equal(t, commits["old"], contributions[0].CommitSHA)
+	assert.Equal(t, "user2@example.com", contributions[0].AuthorEmail)
+	assert.Equal(t, timeutil.TimeStamp(1579089600), contributions[0].AuthorUnix)
+
+	emailAddress, err := user_model.GetEmailAddressByEmail(t.Context(), "user2@example.com")
+	require.NoError(t, err)
+	emailAddress.UID = 1
+	require.NoError(t, user_model.UpdateEmailAddress(t.Context(), emailAddress))
+	require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
+
+	contributions = loadHeatmapContributionsForRepo(t, repo.ID)
+	require.Len(t, contributions, 1)
+	assert.Equal(t, int64(1), contributions[0].UserID)
+	assert.Equal(t, commits["old"], contributions[0].CommitSHA)
+}
+
+func TestHeatmapIndexIgnoresPusherAndNonDefaultBranch(t *testing.T) {
+	repo, commits := prepareHeatmapIndexRepo(t, "heatmap-exclusions", []heatmapIndexTestCommit{
+		{Branch: "main", Mark: "author-user2", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User One", CommitterEmail: "user1@example.com", AuthorDate: "2020-01-15T12:00:00Z"},
+		{Branch: "main", Mark: "author-user1", AuthorName: "User One", AuthorEmail: "user1@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-16T12:00:00Z"},
+		{Branch: "feature", Mark: "feature-only", AuthorName: "User Two", AuthorEmail: "user2@example.com", CommitterName: "User Two", CommitterEmail: "user2@example.com", AuthorDate: "2020-01-17T12:00:00Z"},
+	})
+
+	timeutil.MockSet(time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC))
+	defer timeutil.MockUnset()
+
+	require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
+
+	contributions := loadHeatmapContributionsForRepo(t, repo.ID)
+	require.Len(t, contributions, 2)
+	assert.Equal(t, int64(2), contributions[0].UserID)
+	assert.Equal(t, commits["author-user2"], contributions[0].CommitSHA)
+	assert.Equal(t, int64(1), contributions[1].UserID)
+	assert.Equal(t, commits["author-user1"], contributions[1].CommitSHA)
+	for _, contribution := range contributions {
+		assert.NotEqual(t, commits["feature-only"], contribution.CommitSHA)
+	}
+
+	deleteMainRef(t, repo)
+	require.NoError(t, IndexDefaultBranchHeatmapContributions(t.Context(), repo))
+	assert.Empty(t, loadHeatmapContributionsForRepo(t, repo.ID))
+}
+
+type heatmapIndexTestCommit struct {
+	Branch         string
+	Mark           string
+	AuthorName     string
+	AuthorEmail    string
+	CommitterName  string
+	CommitterEmail string
+	AuthorDate     string
+}
+
+func prepareHeatmapIndexRepo(t *testing.T, repoName string, commits []heatmapIndexTestCommit) (*repo_model.Repository, map[string]string) {
+	t.Helper()
+	require.NoError(t, unittest.PrepareTestDatabase())
+	require.NoError(t, db.TruncateBeans(t.Context(), &activities_model.HeatmapContribution{}))
+
+	owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
+	repo, err := CreateRepositoryDirectly(t.Context(), owner, owner, CreateRepoOptions{Name: repoName, DefaultBranch: "main"}, true)
+	require.NoError(t, err)
+
+	marks := runFastImport(t, repo, commits)
+	repo.IsEmpty = false
+	require.NoError(t, repo_model.UpdateRepositoryColsWithAutoTime(t.Context(), repo, "is_empty"))
+	return repo, marks
+}
+
+func runFastImport(t *testing.T, repo *repo_model.Repository, commits []heatmapIndexTestCommit) map[string]string {
+	t.Helper()
+
+	var stream strings.Builder
+	branchTips := make(map[string]string)
+	for i, commit := range commits {
+		mark := fmt.Sprintf(":%d", i+1)
+		branchRef := "refs/heads/" + commit.Branch
+		stream.WriteString("commit " + branchRef + "\n")
+		stream.WriteString("mark " + mark + "\n")
+		stream.WriteString(signatureLine("author", commit.AuthorName, commit.AuthorEmail, commit.AuthorDate))
+		stream.WriteString(signatureLine("committer", commit.CommitterName, commit.CommitterEmail, commit.AuthorDate))
+		message := "heatmap " + commit.Mark
+		stream.WriteString(fmt.Sprintf("data %d\n%s\n", len(message), message))
+		if parentMark := branchTips[commit.Branch]; parentMark != "" {
+			stream.WriteString("from " + parentMark + "\n")
+		}
+		content := commit.Mark + "\n"
+		stream.WriteString(fmt.Sprintf("M 100644 inline %s.txt\n", commit.Mark))
+		stream.WriteString(fmt.Sprintf("data %d\n%s", len(content), content))
+		branchTips[commit.Branch] = mark
+	}
+
+	require.NoError(t, gitcmd.NewCommand("fast-import", "--export-marks=-").
+		WithDir(repo.RepoPath()).
+		WithStdinCopy(strings.NewReader(stream.String())).
+		Run(t.Context()))
+
+	commitSHAs := make(map[string]string, len(commits))
+	for _, commit := range commits {
+		stdout, _, err := gitcmd.NewCommand("log", "-1", "--format=%H", "--fixed-strings").
+			AddOptionFormat("--grep=%s", "heatmap "+commit.Mark).
+			AddDynamicArguments("refs/heads/" + commit.Branch).
+			WithDir(repo.RepoPath()).
+			RunStdString(t.Context())
+		require.NoError(t, err)
+		commitSHAs[commit.Mark] = strings.TrimSpace(stdout)
+	}
+	return commitSHAs
+}
+
+func signatureLine(kind, name, email, date string) string {
+	parsed, err := time.Parse(time.RFC3339, date)
+	if err != nil {
+		panic(err)
+	}
+	return fmt.Sprintf("%s %s <%s> %d +0000\n", kind, name, email, parsed.Unix())
+}
+
+func loadHeatmapContributionsForRepo(t *testing.T, repoID int64) []*activities_model.HeatmapContribution {
+	t.Helper()
+	contributions, err := activities_model.FindHeatmapContributionsByRepo(t.Context(), repoID)
+	require.NoError(t, err)
+	return contributions
+}
+
+func deleteMainRef(t *testing.T, repo *repo_model.Repository) {
+	t.Helper()
+	require.NoError(t, gitcmd.NewCommand("update-ref", "-d", "refs/heads/main").WithDir(repo.RepoPath()).Run(t.Context()))
+}