feat: vendor gitea 1.16.2

This commit is contained in:
2026-06-02 08:36:01 +00:00
parent 247cd60f69
commit 54798b4615
2145 changed files with 162965 additions and 126447 deletions
@@ -5,83 +5,34 @@ package pipeline
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"strconv"
"strings"
"sync"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/log"
)
// CatFileBatchCheck runs cat-file with --batch-check
func CatFileBatchCheck(ctx context.Context, shasToCheckReader *io.PipeReader, catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
defer wg.Done()
defer shasToCheckReader.Close()
defer catFileCheckWriter.Close()
stderr := new(bytes.Buffer)
var errbuf strings.Builder
cmd := gitcmd.NewCommand("cat-file", "--batch-check")
if err := cmd.Run(ctx, &gitcmd.RunOpts{
Dir: tmpBasePath,
Stdin: shasToCheckReader,
Stdout: catFileCheckWriter,
Stderr: stderr,
}); err != nil {
_ = catFileCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check [%s]: %w - %s", tmpBasePath, err, errbuf.String()))
}
func CatFileBatchCheck(ctx context.Context, cmd *gitcmd.Command, tmpBasePath string) error {
cmd.AddArguments("cat-file", "--batch-check")
return cmd.WithDir(tmpBasePath).RunWithStderr(ctx)
}
// CatFileBatchCheckAllObjects runs cat-file with --batch-check --batch-all
func CatFileBatchCheckAllObjects(ctx context.Context, catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string, errChan chan<- error) {
defer wg.Done()
defer catFileCheckWriter.Close()
stderr := new(bytes.Buffer)
var errbuf strings.Builder
cmd := gitcmd.NewCommand("cat-file", "--batch-check", "--batch-all-objects")
if err := cmd.Run(ctx, &gitcmd.RunOpts{
Dir: tmpBasePath,
Stdout: catFileCheckWriter,
Stderr: stderr,
}); err != nil {
log.Error("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String())
err = fmt.Errorf("git cat-file --batch-check --batch-all-object [%s]: %w - %s", tmpBasePath, err, errbuf.String())
_ = catFileCheckWriter.CloseWithError(err)
errChan <- err
}
func CatFileBatchCheckAllObjects(ctx context.Context, cmd *gitcmd.Command, tmpBasePath string) error {
return cmd.AddArguments("cat-file", "--batch-check", "--batch-all-objects").WithDir(tmpBasePath).RunWithStderr(ctx)
}
// CatFileBatch runs cat-file --batch
func CatFileBatch(ctx context.Context, shasToBatchReader *io.PipeReader, catFileBatchWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
defer wg.Done()
defer shasToBatchReader.Close()
defer catFileBatchWriter.Close()
stderr := new(bytes.Buffer)
var errbuf strings.Builder
if err := gitcmd.NewCommand("cat-file", "--batch").Run(ctx, &gitcmd.RunOpts{
Dir: tmpBasePath,
Stdout: catFileBatchWriter,
Stdin: shasToBatchReader,
Stderr: stderr,
}); err != nil {
_ = shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %w - %s", tmpBasePath, err, errbuf.String()))
}
func CatFileBatch(ctx context.Context, cmd *gitcmd.Command, tmpBasePath string) error {
return cmd.AddArguments("cat-file", "--batch").WithDir(tmpBasePath).RunWithStderr(ctx)
}
// BlobsLessThan1024FromCatFileBatchCheck reads a pipeline from cat-file --batch-check and returns the blobs <1024 in size
func BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) {
defer wg.Done()
defer catFileCheckReader.Close()
scanner := bufio.NewScanner(catFileCheckReader)
defer func() {
_ = shasToBatchWriter.CloseWithError(scanner.Err())
}()
func BlobsLessThan1024FromCatFileBatchCheck(in io.ReadCloser, out io.WriteCloser) error {
defer out.Close()
scanner := bufio.NewScanner(in)
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
@@ -97,12 +48,12 @@ func BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader *io.PipeReader, s
}
toWrite := []byte(fields[0] + "\n")
for len(toWrite) > 0 {
n, err := shasToBatchWriter.Write(toWrite)
n, err := out.Write(toWrite)
if err != nil {
_ = catFileCheckReader.CloseWithError(err)
break
return err
}
toWrite = toWrite[n:]
}
}
return scanner.Err()
}
@@ -4,7 +4,6 @@
package pipeline
import (
"fmt"
"time"
"code.gitea.io/gitea/modules/git"
@@ -26,7 +25,3 @@ type lfsResultSlice []*LFSResult
func (a lfsResultSlice) Len() int { return len(a) }
func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
func lfsError(msg string, err error) error {
return fmt.Errorf("LFS error occurred, %s: err: %w", msg, err)
}
@@ -6,11 +6,10 @@
package pipeline
import (
"bufio"
"fmt"
"io"
"sort"
"strings"
"sync"
"code.gitea.io/gitea/modules/git"
@@ -24,7 +23,6 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
resultsMap := map[string]*LFSResult{}
results := make([]*LFSResult, 0)
basePath := repo.Path
gogitRepo := repo.GoGitRepo()
commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
@@ -32,7 +30,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
All: true,
})
if err != nil {
return nil, lfsError("failed to get GoGit CommitsIter", err)
return nil, fmt.Errorf("LFS error occurred, failed to get GoGit CommitsIter: err: %w", err)
}
err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
@@ -66,7 +64,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
return nil
})
if err != nil && err != io.EOF {
return nil, lfsError("failure in CommitIter.ForEach", err)
return nil, fmt.Errorf("LFS error occurred, failure in CommitIter.ForEach: %w", err)
}
for _, result := range resultsMap {
@@ -82,65 +80,6 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
}
sort.Sort(lfsResultSlice(results))
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
shasToNameReader, shasToNameWriter := io.Pipe()
nameRevStdinReader, nameRevStdinWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(3)
go func() {
defer wg.Done()
scanner := bufio.NewScanner(nameRevStdinReader)
i := 0
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
result := results[i]
result.FullCommitName = line
result.BranchName = strings.Split(line, "~")[0]
i++
}
}()
go NameRevStdin(repo.Ctx, shasToNameReader, nameRevStdinWriter, &wg, basePath)
go func() {
defer wg.Done()
defer shasToNameWriter.Close()
for _, result := range results {
i := 0
if i < len(result.SHA) {
n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
if err != nil {
errChan <- err
break
}
i += n
}
n := 0
for n < 1 {
n, err = shasToNameWriter.Write([]byte{'\n'})
if err != nil {
errChan <- err
break
}
}
}
}()
wg.Wait()
select {
case err, has := <-errChan:
if has {
return nil, lfsError("unable to obtain name for LFS files", err)
}
default:
}
return results, nil
err = fillResultNameRev(repo.Ctx, repo.Path, results)
return results, err
}
@@ -8,46 +8,33 @@ package pipeline
import (
"bufio"
"bytes"
"encoding/hex"
"io"
"sort"
"strings"
"sync"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
)
// FindLFSFile finds commits that contain a provided pointer file hash
func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, error) {
func FindLFSFile(repo *git.Repository, objectID git.ObjectID) (results []*LFSResult, _ error) {
cmd := gitcmd.NewCommand("rev-list", "--all")
revListReader, revListReaderClose := cmd.MakeStdoutPipe()
defer revListReaderClose()
err := cmd.WithDir(repo.Path).
WithPipelineFunc(func(context gitcmd.Context) (err error) {
results, err = findLFSFileFunc(repo, objectID, revListReader)
return err
}).RunWithStderr(repo.Ctx)
return results, err
}
func findLFSFileFunc(repo *git.Repository, objectID git.ObjectID, revListReader io.Reader) ([]*LFSResult, error) {
resultsMap := map[string]*LFSResult{}
results := make([]*LFSResult, 0)
basePath := repo.Path
// Use rev-list to provide us with all commits in order
revListReader, revListWriter := io.Pipe()
defer func() {
_ = revListWriter.Close()
_ = revListReader.Close()
}()
go func() {
stderr := strings.Builder{}
err := gitcmd.NewCommand("rev-list", "--all").Run(repo.Ctx, &gitcmd.RunOpts{
Dir: repo.Path,
Stdout: revListWriter,
Stderr: &stderr,
})
if err != nil {
_ = revListWriter.CloseWithError(gitcmd.ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
// so let's create a batch stdin and stdout
batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx)
batch, cancel, err := repo.CatFileBatch(repo.Ctx)
if err != nil {
return nil, err
}
@@ -55,7 +42,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
// We'll use a scanner for the revList because it's simpler than a bufio.Reader
scan := bufio.NewScanner(revListReader)
trees := [][]byte{}
trees := []string{}
paths := []string{}
fnameBuf := make([]byte, 4096)
@@ -64,14 +51,10 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
for scan.Scan() {
// Get the next commit ID
commitID := scan.Bytes()
commitID := scan.Text()
// push the commit to the cat-file --batch process
_, err := batchStdinWriter.Write(commitID)
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte{'\n'})
info, batchReader, err := batch.QueryContent(commitID)
if err != nil {
return nil, err
}
@@ -81,26 +64,20 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
commitReadingLoop:
for {
_, typ, size, err := git.ReadBatchLine(batchReader)
if err != nil {
return nil, err
}
switch typ {
switch info.Type {
case "tag":
// This shouldn't happen but if it does well just get the commit and try again
id, err := git.ReadTagObjectID(batchReader, size)
id, err := git.ReadTagObjectID(batchReader, info.Size)
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte(id + "\n"))
if err != nil {
if info, batchReader, err = batch.QueryContent(id); err != nil {
return nil, err
}
continue
case "commit":
// Read in the commit to get its tree and in case this is one of the last used commits
curCommit, err = git.CommitFromReader(repo, git.MustIDFromString(string(commitID)), io.LimitReader(batchReader, size))
curCommit, err = git.CommitFromReader(repo, git.MustIDFromString(commitID), io.LimitReader(batchReader, info.Size))
if err != nil {
return nil, err
}
@@ -108,13 +85,13 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
return nil, err
}
if _, err := batchStdinWriter.Write([]byte(curCommit.Tree.ID.String() + "\n")); err != nil {
if info, _, err = batch.QueryContent(curCommit.Tree.ID.String()); err != nil {
return nil, err
}
curPath = ""
case "tree":
var n int64
for n < size {
for n < info.Size {
mode, fname, binObjectID, count, err := git.ParseCatFileTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
if err != nil {
return nil, err
@@ -124,15 +101,13 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
result := LFSResult{
Name: curPath + string(fname),
SHA: curCommit.ID.String(),
Summary: strings.Split(strings.TrimSpace(curCommit.CommitMessage), "\n")[0],
Summary: curCommit.Summary(),
When: curCommit.Author.When,
ParentHashes: curCommit.Parents,
}
resultsMap[curCommit.ID.String()+":"+curPath+string(fname)] = &result
} else if string(mode) == git.EntryModeTree.String() {
hexObjectID := make([]byte, objectID.Type().FullLength())
git.BinToHex(objectID.Type(), binObjectID, hexObjectID)
trees = append(trees, hexObjectID)
trees = append(trees, hex.EncodeToString(binObjectID))
paths = append(paths, curPath+string(fname)+"/")
}
}
@@ -140,11 +115,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
return nil, err
}
if len(trees) > 0 {
_, err := batchStdinWriter.Write(trees[len(trees)-1])
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte("\n"))
info, _, err = batch.QueryContent(trees[len(trees)-1])
if err != nil {
return nil, err
}
@@ -155,7 +126,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
break commitReadingLoop
}
default:
if err := git.DiscardFull(batchReader, size+1); err != nil {
if err := git.DiscardFull(batchReader, info.Size+1); err != nil {
return nil, err
}
}
@@ -179,56 +150,6 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
}
sort.Sort(lfsResultSlice(results))
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
shasToNameReader, shasToNameWriter := io.Pipe()
nameRevStdinReader, nameRevStdinWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(3)
go func() {
defer wg.Done()
scanner := bufio.NewScanner(nameRevStdinReader)
i := 0
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
result := results[i]
result.FullCommitName = line
result.BranchName = strings.Split(line, "~")[0]
i++
}
}()
go NameRevStdin(repo.Ctx, shasToNameReader, nameRevStdinWriter, &wg, basePath)
go func() {
defer wg.Done()
defer shasToNameWriter.Close()
for _, result := range results {
_, err := shasToNameWriter.Write([]byte(result.SHA))
if err != nil {
errChan <- err
break
}
_, err = shasToNameWriter.Write([]byte{'\n'})
if err != nil {
errChan <- err
break
}
}
}()
wg.Wait()
select {
case err, has := <-errChan:
if has {
return nil, lfsError("unable to obtain name for LFS files", err)
}
default:
}
return results, nil
err = fillResultNameRev(repo.Ctx, repo.Path, results)
return results, err
}
@@ -0,0 +1,38 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package pipeline
import (
"testing"
"time"
"code.gitea.io/gitea/modules/git"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFindLFSFile(t *testing.T) {
repoPath := "../../../tests/gitea-repositories-meta/user2/lfs.git"
gitRepo, err := git.OpenRepository(t.Context(), repoPath)
require.NoError(t, err)
defer gitRepo.Close()
objectID := git.MustIDFromString("2b6c6c4eaefa24b22f2092c3d54b263ff26feb58")
stats, err := FindLFSFile(gitRepo, objectID)
require.NoError(t, err)
tm, err := time.Parse(time.RFC3339, "2022-12-21T17:56:42-05:00")
require.NoError(t, err)
assert.Len(t, stats, 1)
assert.Equal(t, "CONTRIBUTING.md", stats[0].Name)
assert.Equal(t, "73cf03db6ece34e12bf91e8853dc58f678f2f82d", stats[0].SHA)
assert.Equal(t, "Initial commit", stats[0].Summary)
assert.Equal(t, tm, stats[0].When)
assert.Empty(t, stats[0].ParentHashes)
assert.Equal(t, "master", stats[0].BranchName)
assert.Equal(t, "master", stats[0].FullCommitName)
}
@@ -0,0 +1,14 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package pipeline
import (
"testing"
"code.gitea.io/gitea/modules/git"
)
func TestMain(m *testing.M) {
git.RunGitTests(m)
}
@@ -4,30 +4,54 @@
package pipeline
import (
"bytes"
"bufio"
"context"
"fmt"
"io"
"errors"
"strings"
"sync"
"code.gitea.io/gitea/modules/git/gitcmd"
"golang.org/x/sync/errgroup"
)
// NameRevStdin runs name-rev --stdin
func NameRevStdin(ctx context.Context, shasToNameReader *io.PipeReader, nameRevStdinWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
defer wg.Done()
defer shasToNameReader.Close()
defer nameRevStdinWriter.Close()
func fillResultNameRev(ctx context.Context, basePath string, results []*LFSResult) error {
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
wg := errgroup.Group{}
cmd := gitcmd.NewCommand("name-rev", "--stdin", "--name-only", "--always").WithDir(basePath)
stdin, stdinClose := cmd.MakeStdinPipe()
stdout, stdoutClose := cmd.MakeStdoutPipe()
defer stdinClose()
defer stdoutClose()
stderr := new(bytes.Buffer)
var errbuf strings.Builder
if err := gitcmd.NewCommand("name-rev", "--stdin", "--name-only", "--always").Run(ctx, &gitcmd.RunOpts{
Dir: tmpBasePath,
Stdout: nameRevStdinWriter,
Stdin: shasToNameReader,
Stderr: stderr,
}); err != nil {
_ = shasToNameReader.CloseWithError(fmt.Errorf("git name-rev [%s]: %w - %s", tmpBasePath, err, errbuf.String()))
}
wg.Go(func() error {
scanner := bufio.NewScanner(stdout)
i := 0
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
result := results[i]
result.FullCommitName = line
result.BranchName = strings.Split(line, "~")[0]
i++
}
return scanner.Err()
})
wg.Go(func() error {
defer stdinClose()
for _, result := range results {
_, err := stdin.Write([]byte(result.SHA))
if err != nil {
return err
}
_, err = stdin.Write([]byte{'\n'})
if err != nil {
return err
}
}
return nil
})
err := cmd.RunWithStderr(ctx)
return errors.Join(err, wg.Wait())
}
@@ -5,65 +5,26 @@ package pipeline
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"strings"
"sync"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/log"
)
// RevListAllObjects runs rev-list --objects --all and writes to a pipewriter
func RevListAllObjects(ctx context.Context, revListWriter *io.PipeWriter, wg *sync.WaitGroup, basePath string, errChan chan<- error) {
defer wg.Done()
defer revListWriter.Close()
stderr := new(bytes.Buffer)
var errbuf strings.Builder
cmd := gitcmd.NewCommand("rev-list", "--objects", "--all")
if err := cmd.Run(ctx, &gitcmd.RunOpts{
Dir: basePath,
Stdout: revListWriter,
Stderr: stderr,
}); err != nil {
log.Error("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String())
err = fmt.Errorf("git rev-list --objects --all [%s]: %w - %s", basePath, err, errbuf.String())
_ = revListWriter.CloseWithError(err)
errChan <- err
}
}
// RevListObjects run rev-list --objects from headSHA to baseSHA
func RevListObjects(ctx context.Context, revListWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath, headSHA, baseSHA string, errChan chan<- error) {
defer wg.Done()
defer revListWriter.Close()
stderr := new(bytes.Buffer)
var errbuf strings.Builder
cmd := gitcmd.NewCommand("rev-list", "--objects").AddDynamicArguments(headSHA)
func RevListObjects(ctx context.Context, cmd *gitcmd.Command, tmpBasePath, headSHA, baseSHA string) error {
cmd.AddArguments("rev-list", "--objects").AddDynamicArguments(headSHA)
if baseSHA != "" {
cmd = cmd.AddArguments("--not").AddDynamicArguments(baseSHA)
}
if err := cmd.Run(ctx, &gitcmd.RunOpts{
Dir: tmpBasePath,
Stdout: revListWriter,
Stderr: stderr,
}); err != nil {
log.Error("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())
errChan <- fmt.Errorf("git rev-list [%s]: %w - %s", tmpBasePath, err, errbuf.String())
}
return cmd.WithDir(tmpBasePath).RunWithStderr(ctx)
}
// BlobsFromRevListObjects reads a RevListAllObjects and only selects blobs
func BlobsFromRevListObjects(revListReader *io.PipeReader, shasToCheckWriter *io.PipeWriter, wg *sync.WaitGroup) {
defer wg.Done()
defer revListReader.Close()
scanner := bufio.NewScanner(revListReader)
defer func() {
_ = shasToCheckWriter.CloseWithError(scanner.Err())
}()
func BlobsFromRevListObjects(in io.ReadCloser, out io.WriteCloser) error {
defer out.Close()
scanner := bufio.NewScanner(in)
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
@@ -75,12 +36,12 @@ func BlobsFromRevListObjects(revListReader *io.PipeReader, shasToCheckWriter *io
}
toWrite := []byte(fields[0] + "\n")
for len(toWrite) > 0 {
n, err := shasToCheckWriter.Write(toWrite)
n, err := out.Write(toWrite)
if err != nil {
_ = revListReader.CloseWithError(err)
break
return err
}
toWrite = toWrite[n:]
}
}
return scanner.Err()
}