feat: vendor gitea 1.16.2

This commit is contained in:
2026-06-02 08:36:01 +00:00
parent 247cd60f69
commit 54798b4615
2145 changed files with 162965 additions and 126447 deletions
+1 -1
View File
@@ -193,7 +193,7 @@ func createCsvDiff(diffFile *DiffFile, baseReader, headReader *csv.Reader) ([]*T
}
if aRow == nil && bRow == nil {
// No content
return nil, nil
return nil, nil //nolint:nilnil // return nil to indicate that the row has no content
}
aIndex := 0 // tracks where we are in the a2bColMap
@@ -15,6 +15,7 @@ import (
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
type DiffTree struct {
@@ -56,12 +57,14 @@ func runGitDiffTree(ctx context.Context, gitRepo *git.Repository, useMergeBase b
return nil, err
}
cmd := gitcmd.NewCommand("diff-tree", "--raw", "-r", "--find-renames", "--root")
cmd := gitcmd.NewCommand("diff-tree", "--raw", "-r", "--root").
AddOptionFormat("--find-renames=%s", setting.Git.DiffRenameSimilarityThreshold)
if useMergeBase {
cmd.AddArguments("--merge-base")
}
cmd.AddDynamicArguments(baseCommitID, headCommitID)
stdout, _, runErr := cmd.RunStdString(ctx, &gitcmd.RunOpts{Dir: gitRepo.Path})
stdout, _, runErr := cmd.WithDir(gitRepo.Path).RunStdString(ctx)
if runErr != nil {
log.Warn("git diff-tree: %v", runErr)
return nil, runErr
@@ -163,16 +166,6 @@ func parseGitDiffTreeLine(line string) (*DiffTreeRecord, error) {
return nil, fmt.Errorf("unparsable output for diff-tree --raw: `%s`, expected 5 space delimited values got %d)", line, len(fields))
}
baseMode, err := git.ParseEntryMode(fields[0])
if err != nil {
return nil, err
}
headMode, err := git.ParseEntryMode(fields[1])
if err != nil {
return nil, err
}
baseBlobID := fields[2]
headBlobID := fields[3]
@@ -198,8 +191,8 @@ func parseGitDiffTreeLine(line string) (*DiffTreeRecord, error) {
return &DiffTreeRecord{
Status: status,
Score: score,
BaseMode: baseMode,
HeadMode: headMode,
BaseMode: git.ParseEntryMode(fields[0]),
HeadMode: git.ParseEntryMode(fields[1]),
BaseBlobID: baseBlobID,
HeadBlobID: headBlobID,
BasePath: basePath,
+372 -166
View File
@@ -13,6 +13,7 @@ import (
"html/template"
"io"
"net/url"
"path"
"sort"
"strings"
"time"
@@ -23,18 +24,23 @@ import (
pull_model "code.gitea.io/gitea/models/pull"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/analyze"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/attribute"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/highlight"
"code.gitea.io/gitea/modules/htmlutil"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/svg"
"code.gitea.io/gitea/modules/translation"
"code.gitea.io/gitea/modules/util"
"github.com/alecthomas/chroma/v2"
"github.com/sergi/go-diff/diffmatchpatch"
stdcharset "golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
@@ -64,18 +70,6 @@ const (
DiffFileCopy
)
// DiffLineExpandDirection represents the DiffLineSection expand direction
type DiffLineExpandDirection uint8
// DiffLineExpandDirection possible values.
const (
DiffLineExpandNone DiffLineExpandDirection = iota + 1
DiffLineExpandSingle
DiffLineExpandUpDown
DiffLineExpandUp
DiffLineExpandDown
)
// DiffLine represents a line difference in a DiffSection.
type DiffLine struct {
LeftIdx int // line number, 1-based
@@ -89,13 +83,37 @@ type DiffLine struct {
// DiffLineSectionInfo represents diff line section meta data
type DiffLineSectionInfo struct {
Path string
LastLeftIdx int
LastRightIdx int
LeftIdx int
RightIdx int
language *diffVarMutable[string]
Path string
// These line "idx" are 1-based line numbers
// Left/Right refer to the left/right side of the diff:
//
// LastLeftIdx | LastRightIdx
// [up/down expander] @@ hunk info @@
// LeftIdx | RightIdx
LastLeftIdx int
LastRightIdx int
LeftIdx int
RightIdx int
// Hunk sizes of the hidden lines
LeftHunkSize int
RightHunkSize int
// For example:
// 17 | 31
// [up/down] @@ -40,23 +54,9 @@ ....
// 40 | 54
//
// In this case:
// LastLeftIdx = 17, LastRightIdx = 31
// LeftHunkSize = 23, RightHunkSize = 9
// LeftIdx = 40, RightIdx = 54
HiddenCommentIDs []int64 // IDs of hidden comments in this section
}
// DiffHTMLOperation is the HTML version of diffmatchpatch.Diff
@@ -107,8 +125,14 @@ type DiffHTMLOperation struct {
// BlobExcerptChunkSize represent max lines of excerpt
const BlobExcerptChunkSize = 20
// MaxDiffHighlightEntireFileSize is the maximum file size that will be highlighted with "entire file diff"
const MaxDiffHighlightEntireFileSize = 1 * 1024 * 1024
// Chroma seems extremely slow when highlighting large files, it might take dozens or hundreds of milliseconds.
// When fully highlighting a diff with a lot of large files, it would take many seconds or even dozens of seconds.
// So, don't highlight the entire file if it's too large, or highlighting takes too long.
// When there is no full-file highlighting, the legacy "line-by-line" highlighting is still applied as the fallback.
const (
MaxFullFileHighlightSizeLimit = 256 * 1024
MaxFullFileHighlightTimeLimit = 2 * time.Second
)
// GetType returns the type of DiffLine.
func (d *DiffLine) GetType() int {
@@ -150,40 +174,117 @@ func (d *DiffLine) GetLineTypeMarker() string {
return ""
}
// GetBlobExcerptQuery builds query string to get blob excerpt
func (d *DiffLine) GetBlobExcerptQuery() string {
query := fmt.Sprintf(
func (d *DiffLine) getBlobExcerptQuery() string {
language := ""
if d.SectionInfo.language != nil { // for normal cases, it can't be nil, this check is only for some tests
language = d.SectionInfo.language.value
}
return fmt.Sprintf(
"last_left=%d&last_right=%d&"+
"left=%d&right=%d&"+
"left_hunk_size=%d&right_hunk_size=%d&"+
"path=%s",
"path=%s&filelang=%s",
d.SectionInfo.LastLeftIdx, d.SectionInfo.LastRightIdx,
d.SectionInfo.LeftIdx, d.SectionInfo.RightIdx,
d.SectionInfo.LeftHunkSize, d.SectionInfo.RightHunkSize,
url.QueryEscape(d.SectionInfo.Path))
return query
url.QueryEscape(d.SectionInfo.Path), url.QueryEscape(language))
}
// GetExpandDirection gets DiffLineExpandDirection
func (d *DiffLine) GetExpandDirection() DiffLineExpandDirection {
func (d *DiffLine) GetExpandDirection() string {
if d.Type != DiffLineSection || d.SectionInfo == nil || d.SectionInfo.LeftIdx-d.SectionInfo.LastLeftIdx <= 1 || d.SectionInfo.RightIdx-d.SectionInfo.LastRightIdx <= 1 {
return DiffLineExpandNone
return ""
}
if d.SectionInfo.LastLeftIdx <= 0 && d.SectionInfo.LastRightIdx <= 0 {
return DiffLineExpandUp
} else if d.SectionInfo.RightIdx-d.SectionInfo.LastRightIdx > BlobExcerptChunkSize && d.SectionInfo.RightHunkSize > 0 {
return DiffLineExpandUpDown
return "up"
} else if d.SectionInfo.RightIdx-d.SectionInfo.LastRightIdx-1 > BlobExcerptChunkSize && d.SectionInfo.RightHunkSize > 0 {
return "updown"
} else if d.SectionInfo.LeftHunkSize <= 0 && d.SectionInfo.RightHunkSize <= 0 {
return DiffLineExpandDown
return "down"
}
return DiffLineExpandSingle
return "single"
}
func getDiffLineSectionInfo(treePath, line string, lastLeftIdx, lastRightIdx int) *DiffLineSectionInfo {
type DiffBlobExcerptData struct {
BaseLink string
IsWikiRepo bool
PullIssueIndex int64
DiffStyle string
AfterCommitID string
}
const (
DiffStyleSplit = "split"
DiffStyleUnified = "unified"
)
func (d *DiffLine) RenderBlobExcerptButtons(fileNameHash string, data *DiffBlobExcerptData) template.HTML {
dataHiddenCommentIDs := strings.Join(base.Int64sToStrings(d.SectionInfo.HiddenCommentIDs), ",")
anchor := fmt.Sprintf("diff-%sK%d", fileNameHash, d.SectionInfo.RightIdx)
makeButton := func(direction, svgName string) template.HTML {
style := util.IfZero(data.DiffStyle, "unified")
link := data.BaseLink + "/" + data.AfterCommitID + fmt.Sprintf("?style=%s&direction=%s&anchor=%s", url.QueryEscape(style), direction, url.QueryEscape(anchor)) + "&" + d.getBlobExcerptQuery()
if data.PullIssueIndex > 0 {
link += fmt.Sprintf("&pull_issue_index=%d", data.PullIssueIndex)
}
return htmlutil.HTMLFormat(
`<button class="code-expander-button" hx-target="closest tr" hx-get="%s" data-hidden-comment-ids=",%s,">%s</button>`,
link, dataHiddenCommentIDs, svg.RenderHTML(svgName),
)
}
var content template.HTML
if len(d.SectionInfo.HiddenCommentIDs) > 0 {
tooltip := fmt.Sprintf("%d hidden comment(s)", len(d.SectionInfo.HiddenCommentIDs))
content += htmlutil.HTMLFormat(`<span class="code-comment-more" data-tooltip-content="%s">%d</span>`, tooltip, len(d.SectionInfo.HiddenCommentIDs))
}
expandDirection := d.GetExpandDirection()
if expandDirection == "updown" || expandDirection == "down" {
content += makeButton("down", "octicon-fold-down")
}
if expandDirection == "up" || expandDirection == "updown" {
content += makeButton("up", "octicon-fold-up")
}
if expandDirection == "single" {
content += makeButton("single", "octicon-fold")
}
return htmlutil.HTMLFormat(`<div class="code-expander-buttons" data-expand-direction="%s">%s</div>`, expandDirection, content)
}
// FillHiddenCommentIDsForDiffLine finds comment IDs that are in the hidden range of an expand button
func FillHiddenCommentIDsForDiffLine(line *DiffLine, lineComments map[int64][]*issues_model.Comment) {
if line.Type != DiffLineSection {
return
}
var hiddenCommentIDs []int64
for commentLineNum, comments := range lineComments {
if commentLineNum < 0 {
// ATTENTION: BLOB-EXCERPT-COMMENT-RIGHT: skip left-side, unchanged lines always use "right (proposed)" side for comments
continue
}
lineNum := int(commentLineNum)
isEndOfFileExpansion := line.SectionInfo.RightHunkSize == 0
inRange := lineNum > line.SectionInfo.LastRightIdx &&
(isEndOfFileExpansion && lineNum <= line.SectionInfo.RightIdx ||
!isEndOfFileExpansion && lineNum < line.SectionInfo.RightIdx)
if inRange {
for _, comment := range comments {
hiddenCommentIDs = append(hiddenCommentIDs, comment.ID)
}
}
}
line.SectionInfo.HiddenCommentIDs = hiddenCommentIDs
}
func newDiffLineSectionInfo(curFile *DiffFile, line string, lastLeftIdx, lastRightIdx int) *DiffLineSectionInfo {
leftLine, leftHunk, rightLine, rightHunk := git.ParseDiffHunkString(line)
return &DiffLineSectionInfo{
Path: treePath,
Path: curFile.Name,
language: &curFile.language,
LastLeftIdx: lastLeftIdx,
LastRightIdx: lastRightIdx,
LeftIdx: leftLine,
@@ -203,7 +304,11 @@ func getLineContent(content string, locale translation.Locale) DiffInline {
// DiffSection represents a section of a DiffFile.
type DiffSection struct {
file *DiffFile
language *diffVarMutable[string]
highlightedLeftLines *diffVarMutable[map[int]template.HTML]
highlightedRightLines *diffVarMutable[map[int]template.HTML]
highlightLexer *diffVarMutable[chroma.Lexer]
FileName string
Lines []*DiffLine
}
@@ -244,17 +349,19 @@ func (diffSection *DiffSection) getLineContentForRender(lineIdx int, diffLine *D
if setting.Git.DisableDiffHighlight {
return template.HTML(html.EscapeString(diffLine.Content[1:]))
}
h, _ = highlight.Code(diffSection.FileName, fileLanguage, diffLine.Content[1:])
return h
if diffSection.highlightLexer.value == nil {
diffSection.highlightLexer.value = highlight.DetectChromaLexerByFileName(diffSection.FileName, fileLanguage)
}
return highlight.RenderCodeByLexer(diffSection.highlightLexer.value, diffLine.Content[1:])
}
func (diffSection *DiffSection) getDiffLineForRender(diffLineType DiffLineType, leftLine, rightLine *DiffLine, locale translation.Locale) DiffInline {
var fileLanguage string
var highlightedLeftLines, highlightedRightLines map[int]template.HTML
// when a "diff section" is manually prepared by ExcerptBlob, it doesn't have "file" information
if diffSection.file != nil {
fileLanguage = diffSection.file.Language
highlightedLeftLines, highlightedRightLines = diffSection.file.highlightedLeftLines, diffSection.file.highlightedRightLines
if diffSection.language != nil {
fileLanguage = diffSection.language.value
highlightedLeftLines, highlightedRightLines = diffSection.highlightedLeftLines.value, diffSection.highlightedRightLines.value
}
var lineHTML template.HTML
@@ -288,6 +395,12 @@ func (diffSection *DiffSection) getDiffLineForRender(diffLineType DiffLineType,
// GetComputedInlineDiffFor computes inline diff for the given line.
func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine, locale translation.Locale) DiffInline {
defer func() {
if err := recover(); err != nil {
// the logic is too complex in this function, help to catch any panic because Golang template doesn't print the stack
log.Error("panic in GetComputedInlineDiffFor: %v\nStack: %s", err, log.Stack(2))
}
}()
// try to find equivalent diff line. ignore, otherwise
switch diffLine.Type {
case DiffLineSection:
@@ -305,33 +418,37 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine, loc
}
}
// diffVarMutable is a wrapper to make a variable mutable to be shared across structs
type diffVarMutable[T any] struct {
value T
}
// DiffFile represents a file diff.
type DiffFile struct {
// only used internally to parse Ambiguous filenames
isAmbiguous bool
// basic fields (parsed from diff result)
Name string
NameHash string
OldName string
Addition int
Deletion int
Type DiffFileType
Mode string
OldMode string
IsCreated bool
IsDeleted bool
IsBin bool
IsLFSFile bool
IsRenamed bool
IsSubmodule bool
Name string
NameHash string
OldName string
Addition int
Deletion int
Type DiffFileType
EntryMode string
OldEntryMode string
IsCreated bool
IsDeleted bool
IsBin bool
IsLFSFile bool
IsRenamed bool
IsSubmodule bool
// basic fields but for render purpose only
Sections []*DiffSection
IsIncomplete bool
IsIncompleteLineTooLong bool
// will be filled by the extra loop in GitDiffForRender
Language string
IsGenerated bool
IsVendored bool
SubmoduleDiffInfo *SubmoduleDiffInfo // IsSubmodule==true, then there must be a SubmoduleDiffInfo
@@ -343,9 +460,11 @@ type DiffFile struct {
IsViewed bool // User specific
HasChangedSinceLastReview bool // User specific
// for render purpose only, will be filled by the extra loop in GitDiffForRender
highlightedLeftLines map[int]template.HTML
highlightedRightLines map[int]template.HTML
// for render purpose only, will be filled by the extra loop in GitDiffForRender, the maps of lines are 0-based
language diffVarMutable[string]
highlightRender diffVarMutable[chroma.Lexer] // cache render (atm: lexer) for current file, only detect once for line-by-line mode
highlightedLeftLines diffVarMutable[map[int]template.HTML]
highlightedRightLines diffVarMutable[map[int]template.HTML]
}
// GetType returns type of diff file.
@@ -382,6 +501,7 @@ func (diffFile *DiffFile) GetTailSectionAndLimitedContent(leftCommit, rightCommi
Type: DiffLineSection,
Content: " ",
SectionInfo: &DiffLineSectionInfo{
language: &diffFile.language,
Path: diffFile.Name,
LastLeftIdx: lastLine.LeftIdx,
LastRightIdx: lastLine.RightIdx,
@@ -401,25 +521,48 @@ func (diffFile *DiffFile) GetDiffFileName() string {
return diffFile.Name
}
// GetDiffFileBaseName returns the short name of the diff file, or its short old name in case it was deleted
func (diffFile *DiffFile) GetDiffFileBaseName() string {
if diffFile.Name == "" {
return path.Base(diffFile.OldName)
}
return path.Base(diffFile.Name)
}
func (diffFile *DiffFile) ShouldBeHidden() bool {
return diffFile.IsGenerated || diffFile.IsViewed
}
func (diffFile *DiffFile) ModeTranslationKey(mode string) string {
switch mode {
case "040000":
return "git.filemode.directory"
case "100644":
return "git.filemode.normal_file"
case "100755":
return "git.filemode.executable_file"
case "120000":
return "git.filemode.symbolic_link"
case "160000":
return "git.filemode.submodule"
default:
return mode
func (diffFile *DiffFile) TranslateDiffEntryMode(locale translation.Locale) string {
entryModeTr := func(mode string) string {
entryMode := git.ParseEntryMode(mode)
switch {
case entryMode.IsDir():
return locale.TrString("git.filemode.directory")
case entryMode.IsRegular():
return locale.TrString("git.filemode.normal_file")
case entryMode.IsExecutable():
return locale.TrString("git.filemode.executable_file")
case entryMode.IsLink():
return locale.TrString("git.filemode.symbolic_link")
case entryMode.IsSubModule():
return locale.TrString("git.filemode.submodule")
default:
return mode
}
}
if diffFile.EntryMode != "" && diffFile.OldEntryMode != "" {
oldMode := entryModeTr(diffFile.OldEntryMode)
newMode := entryModeTr(diffFile.EntryMode)
return locale.TrString("git.filemode.changed_filemode", oldMode, newMode)
}
if diffFile.EntryMode != "" {
if entryMode := git.ParseEntryMode(diffFile.EntryMode); !entryMode.IsRegular() {
return entryModeTr(diffFile.EntryMode)
}
}
return ""
}
type limitByteWriter struct {
@@ -439,7 +582,7 @@ func getCommitFileLineCountAndLimitedContent(commit *git.Commit, filePath string
if err != nil {
return 0, nil
}
w := &limitByteWriter{limit: MaxDiffHighlightEntireFileSize + 1}
w := &limitByteWriter{limit: MaxFullFileHighlightSizeLimit + 1}
lineCount, err = blob.GetBlobLineCount(w)
if err != nil {
return 0, nil
@@ -473,6 +616,8 @@ func (diff *Diff) LoadComments(ctx context.Context, issue *issues_model.Issue, c
sort.SliceStable(line.Comments, func(i, j int) bool {
return line.Comments[i].CreatedUnix < line.Comments[j].CreatedUnix
})
// Mark expand buttons that have comments in hidden lines
FillHiddenCommentIDsForDiffLine(line, lineCommits)
}
}
}
@@ -597,10 +742,10 @@ parsingLoop:
strings.HasPrefix(line, "new mode "):
if strings.HasPrefix(line, "old mode ") {
curFile.OldMode = prepareValue(line, "old mode ")
curFile.OldEntryMode = prepareValue(line, "old mode ")
}
if strings.HasPrefix(line, "new mode ") {
curFile.Mode = prepareValue(line, "new mode ")
curFile.EntryMode = prepareValue(line, "new mode ")
}
if strings.HasSuffix(line, " 160000\n") {
curFile.IsSubmodule, curFile.SubmoduleDiffInfo = true, &SubmoduleDiffInfo{}
@@ -635,7 +780,7 @@ parsingLoop:
curFile.Type = DiffFileAdd
curFile.IsCreated = true
if strings.HasPrefix(line, "new file mode ") {
curFile.Mode = prepareValue(line, "new file mode ")
curFile.EntryMode = prepareValue(line, "new file mode ")
}
if strings.HasSuffix(line, " 160000\n") {
curFile.IsSubmodule, curFile.SubmoduleDiffInfo = true, &SubmoduleDiffInfo{}
@@ -742,11 +887,11 @@ parsingLoop:
if buffer.Len() == 0 {
continue
}
charsetLabel, err := charset.DetectEncoding(buffer.Bytes())
if charsetLabel != "UTF-8" && err == nil {
encoding, _ := stdcharset.Lookup(charsetLabel)
if encoding != nil {
diffLineTypeDecoders[lineType] = encoding.NewDecoder()
charsetLabel, _ := charset.DetectEncoding(buffer.Bytes())
if charsetLabel != "UTF-8" {
charsetEncoding, _ := stdcharset.Lookup(charsetLabel)
if charsetEncoding != nil {
diffLineTypeDecoders[lineType] = charsetEncoding.NewDecoder()
}
}
}
@@ -795,6 +940,15 @@ func skipToNextDiffHead(input *bufio.Reader) (line string, err error) {
return line, err
}
func newDiffSectionForDiffFile(curFile *DiffFile) *DiffSection {
return &DiffSection{
language: &curFile.language,
highlightLexer: &curFile.highlightRender,
highlightedLeftLines: &curFile.highlightedLeftLines,
highlightedRightLines: &curFile.highlightedRightLines,
}
}
func parseHunks(ctx context.Context, curFile *DiffFile, maxLines, maxLineCharacters int, input *bufio.Reader) (lineBytes []byte, isFragment bool, err error) {
sb := strings.Builder{}
@@ -852,12 +1006,12 @@ func parseHunks(ctx context.Context, curFile *DiffFile, maxLines, maxLineCharact
line := sb.String()
// Create a new section to represent this hunk
curSection = &DiffSection{file: curFile}
curSection = newDiffSectionForDiffFile(curFile)
lastLeftIdx = -1
curFile.Sections = append(curFile.Sections, curSection)
// FIXME: the "-1" can't be right, these "line idx" are all 1-based, maybe there are other bugs that covers this bug.
lineSectionInfo := getDiffLineSectionInfo(curFile.Name, line, leftLine-1, rightLine-1)
lineSectionInfo := newDiffLineSectionInfo(curFile, line, leftLine-1, rightLine-1)
diffLine := &DiffLine{
Type: DiffLineSection,
Content: line,
@@ -892,7 +1046,7 @@ func parseHunks(ctx context.Context, curFile *DiffFile, maxLines, maxLineCharact
rightLine++
if curSection == nil {
// Create a new section to represent this hunk
curSection = &DiffSection{file: curFile}
curSection = newDiffSectionForDiffFile(curFile)
curFile.Sections = append(curFile.Sections, curSection)
lastLeftIdx = -1
}
@@ -925,7 +1079,7 @@ func parseHunks(ctx context.Context, curFile *DiffFile, maxLines, maxLineCharact
}
if curSection == nil {
// Create a new section to represent this hunk
curSection = &DiffSection{file: curFile}
curSection = newDiffSectionForDiffFile(curFile)
curFile.Sections = append(curFile.Sections, curSection)
lastLeftIdx = -1
}
@@ -952,7 +1106,7 @@ func parseHunks(ctx context.Context, curFile *DiffFile, maxLines, maxLineCharact
lastLeftIdx = -1
if curSection == nil {
// Create a new section to represent this hunk
curSection = &DiffSection{file: curFile}
curSection = newDiffSectionForDiffFile(curFile)
curFile.Sections = append(curFile.Sections, curSection)
}
curSection.Lines = append(curSection.Lines, diffLine)
@@ -1132,8 +1286,9 @@ func getDiffBasic(ctx context.Context, gitRepo *git.Repository, opts *DiffOption
}
cmdDiff := gitcmd.NewCommand().
AddArguments("diff", "--src-prefix=\\a/", "--dst-prefix=\\b/", "-M").
AddArguments(opts.WhitespaceBehavior...)
AddArguments("diff", "--src-prefix=\\a/", "--dst-prefix=\\b/").
AddArguments(opts.WhitespaceBehavior...).
AddOptionFormat("--find-renames=%s", setting.Git.DiffRenameSimilarityThreshold)
// In git 2.31, git diff learned --skip-to which we can use to shortcut skip to file
// so if we are using at least this version of git we don't have to tell ParsePatch to do
@@ -1150,24 +1305,14 @@ func getDiffBasic(ctx context.Context, gitRepo *git.Repository, opts *DiffOption
cmdCtx, cmdCancel := context.WithCancel(ctx)
defer cmdCancel()
reader, writer := io.Pipe()
defer func() {
_ = reader.Close()
_ = writer.Close()
}()
reader, readerClose := cmdDiff.MakeStdoutPipe()
defer readerClose()
go func() {
stderr := &bytes.Buffer{}
if err := cmdDiff.Run(cmdCtx, &gitcmd.RunOpts{
Timeout: time.Duration(setting.Git.Timeout.Default) * time.Second,
Dir: repoPath,
Stdout: writer,
Stderr: stderr,
}); err != nil && !git.IsErrCanceledOrKilled(err) {
log.Error("error during GetDiff(git diff dir: %s): %v, stderr: %s", repoPath, err, stderr.String())
if err := cmdDiff.
WithDir(repoPath).
RunWithStderr(cmdCtx); err != nil && !gitcmd.IsErrorCanceledOrKilled(err) {
log.Error("error during GetDiff(git diff dir: %s): %v", repoPath, err)
}
_ = writer.Close()
}()
diff, err := ParsePatch(cmdCtx, opts.MaxLines, opts.MaxLineCharacters, opts.MaxFiles, reader, parsePatchSkipToFile)
@@ -1191,6 +1336,8 @@ func GetDiffForRender(ctx context.Context, repoLink string, gitRepo *git.Reposit
return nil, err
}
startTime := time.Now()
checker, err := attribute.NewBatchChecker(gitRepo, opts.AfterCommitID, []string{attribute.LinguistVendored, attribute.LinguistGenerated, attribute.LinguistLanguage, attribute.GitlabLanguage, attribute.Diff})
if err != nil {
return nil, err
@@ -1206,7 +1353,7 @@ func GetDiffForRender(ctx context.Context, repoLink string, gitRepo *git.Reposit
isVendored, isGenerated = attrs.GetVendored(), attrs.GetGenerated()
language := attrs.GetLanguage()
if language.Has() {
diffFile.Language = language.Value()
diffFile.language.value = language.Value()
}
attrDiff = attrs.Get(attribute.Diff).ToString()
}
@@ -1230,13 +1377,14 @@ func GetDiffForRender(ctx context.Context, repoLink string, gitRepo *git.Reposit
diffFile.Sections = append(diffFile.Sections, tailSection)
}
shouldFullFileHighlight := !setting.Git.DisableDiffHighlight && attrDiff.Value() == ""
shouldFullFileHighlight := attrDiff.Value() == "" // only do highlight if no custom diff command
shouldFullFileHighlight = shouldFullFileHighlight && time.Since(startTime) < MaxFullFileHighlightTimeLimit
if shouldFullFileHighlight {
if limitedContent.LeftContent != nil && limitedContent.LeftContent.buf.Len() < MaxDiffHighlightEntireFileSize {
diffFile.highlightedLeftLines = highlightCodeLines(diffFile, true /* left */, limitedContent.LeftContent.buf.Bytes())
if limitedContent.LeftContent != nil {
diffFile.highlightedLeftLines.value = highlightCodeLinesForDiffFile(diffFile, true /* left */, limitedContent.LeftContent.buf.Bytes())
}
if limitedContent.RightContent != nil && limitedContent.RightContent.buf.Len() < MaxDiffHighlightEntireFileSize {
diffFile.highlightedRightLines = highlightCodeLines(diffFile, false /* right */, limitedContent.RightContent.buf.Bytes())
if limitedContent.RightContent != nil {
diffFile.highlightedRightLines.value = highlightCodeLinesForDiffFile(diffFile, false /* right */, limitedContent.RightContent.buf.Bytes())
}
}
}
@@ -1244,38 +1392,27 @@ func GetDiffForRender(ctx context.Context, repoLink string, gitRepo *git.Reposit
return diff, nil
}
func splitHighlightLines(buf []byte) (ret [][]byte) {
lineCount := bytes.Count(buf, []byte("\n")) + 1
ret = make([][]byte, 0, lineCount)
nlTagClose := []byte("\n</")
for {
pos := bytes.IndexByte(buf, '\n')
if pos == -1 {
ret = append(ret, buf)
return ret
}
// Chroma highlighting output sometimes have "</span>" right after \n, sometimes before.
// * "<span>text\n</span>"
// * "<span>text</span>\n"
if bytes.HasPrefix(buf[pos:], nlTagClose) {
pos1 := bytes.IndexByte(buf[pos:], '>')
if pos1 != -1 {
pos += pos1
}
}
ret = append(ret, buf[:pos+1])
buf = buf[pos+1:]
}
func FillDiffFileHighlightLinesByContent(diffFile *DiffFile, left, right []byte) {
diffFile.highlightedLeftLines.value = highlightCodeLinesForDiffFile(diffFile, true /* left */, left)
diffFile.highlightedRightLines.value = highlightCodeLinesForDiffFile(diffFile, false /* right */, right)
}
func highlightCodeLines(diffFile *DiffFile, isLeft bool, rawContent []byte) map[int]template.HTML {
contentUTF8, _ := charset.ToUTF8(rawContent, charset.ConvertOpts{})
content := util.UnsafeBytesToString(contentUTF8)
highlightedNewContent, _ := highlight.Code(diffFile.Name, diffFile.Language, content)
splitLines := splitHighlightLines([]byte(highlightedNewContent))
lines := make(map[int]template.HTML, len(splitLines))
func highlightCodeLinesForDiffFile(diffFile *DiffFile, isLeft bool, rawContent []byte) map[int]template.HTML {
return highlightCodeLines(diffFile.Name, diffFile.language.value, diffFile.Sections, isLeft, rawContent)
}
func highlightCodeLines(name, lang string, sections []*DiffSection, isLeft bool, rawContent []byte) map[int]template.HTML {
if setting.Git.DisableDiffHighlight || len(rawContent) > MaxFullFileHighlightSizeLimit {
return nil
}
content := util.UnsafeBytesToString(charset.ToUTF8(rawContent, charset.ConvertOpts{}))
lexer := highlight.DetectChromaLexerByFileName(name, lang)
highlightedNewContent := highlight.RenderCodeByLexer(lexer, content)
unsafeLines := highlight.UnsafeSplitHighlightedLines(highlightedNewContent)
lines := make(map[int]template.HTML, len(unsafeLines))
// only save the highlighted lines we need, but not the whole file, to save memory
for _, sec := range diffFile.Sections {
for _, sec := range sections {
for _, ln := range sec.Lines {
lineIdx := ln.LeftIdx
if !isLeft {
@@ -1283,8 +1420,8 @@ func highlightCodeLines(diffFile *DiffFile, isLeft bool, rawContent []byte) map[
}
if lineIdx >= 1 {
idx := lineIdx - 1
if idx < len(splitLines) {
lines[idx] = template.HTML(splitLines[idx])
if idx < len(unsafeLines) {
lines[idx] = template.HTML(util.UnsafeBytesToString(unsafeLines[idx]))
}
}
}
@@ -1296,9 +1433,7 @@ type DiffShortStat struct {
NumFiles, TotalAddition, TotalDeletion int
}
func GetDiffShortStat(gitRepo *git.Repository, beforeCommitID, afterCommitID string) (*DiffShortStat, error) {
repoPath := gitRepo.Path
func GetDiffShortStat(ctx context.Context, repoStorage gitrepo.Repository, gitRepo *git.Repository, beforeCommitID, afterCommitID string) (*DiffShortStat, error) {
afterCommit, err := gitRepo.GetCommit(afterCommitID)
if err != nil {
return nil, err
@@ -1310,7 +1445,7 @@ func GetDiffShortStat(gitRepo *git.Repository, beforeCommitID, afterCommitID str
}
diff := &DiffShortStat{}
diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStatByCmdArgs(gitRepo.Ctx, repoPath, nil, actualBeforeCommitID.String(), afterCommitID)
diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = gitrepo.GetDiffShortStatByCmdArgs(ctx, repoStorage, nil, actualBeforeCommitID.String(), afterCommitID)
if err != nil {
return nil, err
}
@@ -1342,38 +1477,40 @@ func SyncUserSpecificDiff(ctx context.Context, userID int64, pull *issues_model.
if errIgnored != nil {
log.Error("Could not get changed files between %s and %s for pull request %d in repo with path %s. Assuming no changes. Error: %w", review.CommitSHA, latestCommit, pull.Index, gitRepo.Path, err)
}
changedFilesSet := make(map[string]struct{}, len(changedFiles))
for _, changedFile := range changedFiles {
changedFilesSet[changedFile] = struct{}{}
}
filesChangedSinceLastDiff := make(map[string]pull_model.ViewedState)
outer:
for _, diffFile := range diff.Files {
fileViewedState := review.UpdatedFiles[diffFile.GetDiffFileName()]
// Check whether it was previously detected that the file has changed since the last review
if fileViewedState == pull_model.HasChanged {
diffFile.HasChangedSinceLastReview = true
continue
}
filename := diffFile.GetDiffFileName()
fileViewedState := review.UpdatedFiles[filename]
// Check explicitly whether the file has changed since the last review
for _, changedFile := range changedFiles {
diffFile.HasChangedSinceLastReview = filename == changedFile
if diffFile.HasChangedSinceLastReview {
filesChangedSinceLastDiff[filename] = pull_model.HasChanged
continue outer // We don't want to check if the file is viewed here as that would fold the file, which is in this case unwanted
}
}
// Check whether the file has already been viewed
if fileViewedState == pull_model.Viewed {
if fileViewedState == pull_model.HasChanged { // Check whether it was previously detected that the file has changed since the last review
diffFile.HasChangedSinceLastReview = true
delete(changedFilesSet, filename)
} else if _, ok := changedFilesSet[filename]; ok { // Check explicitly whether the file has changed since the last review
diffFile.HasChangedSinceLastReview = true
filesChangedSinceLastDiff[filename] = pull_model.HasChanged
delete(changedFilesSet, filename)
} else if fileViewedState == pull_model.Viewed { // Check whether the file has already been viewed
diffFile.IsViewed = true
}
}
// All changed files still present at this point aren't part of the diff anymore, this occurs
// when a file was modified in a previous commit of the diff and the modification got reverted afterwards.
// Marking the files as unviewed to prevent errors where a non-existing file has a view state
for changedFile := range changedFilesSet {
if _, ok := review.UpdatedFiles[changedFile]; ok {
filesChangedSinceLastDiff[changedFile] = pull_model.Unviewed
}
}
if len(filesChangedSinceLastDiff) > 0 {
// Explicitly store files that have changed in the database, if any is present at all.
// This has the benefit that the "Has Changed" attribute will be present as long as the user does not explicitly mark this file as viewed, so it will even survive a page reload after marking another file as viewed.
// On the other hand, this means that even if a commit reverting an unseen change is committed, the file will still be seen as changed.
updatedReview, err := pull_model.UpdateReviewState(ctx, review.UserID, review.PullID, review.CommitSHA, filesChangedSinceLastDiff)
if err != nil {
log.Warn("Could not update review for user %d, pull %d, commit %s and the changed files %v: %v", review.UserID, review.PullID, review.CommitSHA, filesChangedSinceLastDiff, err)
@@ -1404,6 +1541,75 @@ func CommentAsDiff(ctx context.Context, c *issues_model.Comment) (*Diff, error)
return diff, nil
}
// GeneratePatchForUnchangedLine creates a patch showing code context for an unchanged line
func GeneratePatchForUnchangedLine(gitRepo *git.Repository, commitID, treePath string, line int64, contextLines int) (string, error) {
commit, err := gitRepo.GetCommit(commitID)
if err != nil {
return "", fmt.Errorf("GetCommit: %w", err)
}
entry, err := commit.GetTreeEntryByPath(treePath)
if err != nil {
return "", fmt.Errorf("GetTreeEntryByPath: %w", err)
}
blob := entry.Blob()
dataRc, err := blob.DataAsync()
if err != nil {
return "", fmt.Errorf("DataAsync: %w", err)
}
defer dataRc.Close()
return generatePatchForUnchangedLineFromReader(dataRc, treePath, line, contextLines)
}
// generatePatchForUnchangedLineFromReader is the testable core logic that generates a patch from a reader
func generatePatchForUnchangedLineFromReader(reader io.Reader, treePath string, line int64, contextLines int) (string, error) {
// Calculate line range (commented line + lines above it)
commentLine := int(line)
if line < 0 {
commentLine = int(-line)
}
startLine := max(commentLine-contextLines, 1)
endLine := commentLine
// Read only the needed lines efficiently
scanner := bufio.NewScanner(reader)
currentLine := 0
var lines []string
for scanner.Scan() {
currentLine++
if currentLine >= startLine && currentLine <= endLine {
lines = append(lines, scanner.Text())
}
if currentLine > endLine {
break
}
}
if err := scanner.Err(); err != nil {
return "", fmt.Errorf("scanner error: %w", err)
}
if len(lines) == 0 {
return "", fmt.Errorf("no lines found in range %d-%d", startLine, endLine)
}
// Generate synthetic patch
var patchBuilder strings.Builder
fmt.Fprintf(&patchBuilder, "diff --git a/%s b/%s\n", treePath, treePath)
fmt.Fprintf(&patchBuilder, "--- a/%s\n", treePath)
fmt.Fprintf(&patchBuilder, "+++ b/%s\n", treePath)
fmt.Fprintf(&patchBuilder, "@@ -%d,%d +%d,%d @@\n", startLine, len(lines), startLine, len(lines))
for _, lineContent := range lines {
patchBuilder.WriteString(" ")
patchBuilder.WriteString(lineContent)
patchBuilder.WriteString("\n")
}
return patchBuilder.String(), nil
}
// CommentMustAsDiff executes AsDiff and logs the error instead of returning
func CommentMustAsDiff(ctx context.Context, c *issues_model.Comment) *Diff {
if c == nil {
@@ -0,0 +1,124 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package gitdiff
import (
"bufio"
"bytes"
"fmt"
"html/template"
"io"
"code.gitea.io/gitea/modules/setting"
"github.com/alecthomas/chroma/v2"
)
type BlobExcerptOptions struct {
LastLeft int
LastRight int
LeftIndex int
RightIndex int
LeftHunkSize int
RightHunkSize int
Direction string
Language string
}
func fillExcerptLines(section *DiffSection, filePath string, reader io.Reader, lang string, idxLeft, idxRight, chunkSize int) error {
buf := &bytes.Buffer{}
scanner := bufio.NewScanner(reader)
var diffLines []*DiffLine
for line := 0; line < idxRight+chunkSize; line++ {
if ok := scanner.Scan(); !ok {
break
}
lineText := scanner.Text()
if buf.Len()+len(lineText) < int(setting.UI.MaxDisplayFileSize) {
buf.WriteString(lineText)
buf.WriteByte('\n')
}
if line < idxRight {
continue
}
diffLine := &DiffLine{
LeftIdx: idxLeft + (line - idxRight) + 1,
RightIdx: line + 1,
Type: DiffLinePlain,
Content: " " + lineText,
}
diffLines = append(diffLines, diffLine)
}
if err := scanner.Err(); err != nil {
return fmt.Errorf("fillExcerptLines scan: %w", err)
}
section.Lines = diffLines
// DiffLinePlain always uses right lines
section.highlightedRightLines.value = highlightCodeLines(filePath, lang, []*DiffSection{section}, false /* right */, buf.Bytes())
return nil
}
func BuildBlobExcerptDiffSection(filePath string, reader io.Reader, opts BlobExcerptOptions) (*DiffSection, error) {
lastLeft, lastRight, idxLeft, idxRight := opts.LastLeft, opts.LastRight, opts.LeftIndex, opts.RightIndex
leftHunkSize, rightHunkSize, direction := opts.LeftHunkSize, opts.RightHunkSize, opts.Direction
language := opts.Language
chunkSize := BlobExcerptChunkSize
section := &DiffSection{
language: &diffVarMutable[string]{value: language},
highlightLexer: &diffVarMutable[chroma.Lexer]{},
highlightedLeftLines: &diffVarMutable[map[int]template.HTML]{},
highlightedRightLines: &diffVarMutable[map[int]template.HTML]{},
FileName: filePath,
}
var err error
if direction == "up" && (idxLeft-lastLeft) > chunkSize {
idxLeft -= chunkSize
idxRight -= chunkSize
leftHunkSize += chunkSize
rightHunkSize += chunkSize
err = fillExcerptLines(section, filePath, reader, language, idxLeft-1, idxRight-1, chunkSize)
} else if direction == "down" && (idxLeft-lastLeft) > chunkSize {
err = fillExcerptLines(section, filePath, reader, language, lastLeft, lastRight, chunkSize)
lastLeft += chunkSize
lastRight += chunkSize
} else {
offset := -1
if direction == "down" {
offset = 0
}
err = fillExcerptLines(section, filePath, reader, language, lastLeft, lastRight, idxRight-lastRight+offset)
leftHunkSize = 0
rightHunkSize = 0
idxLeft = lastLeft
idxRight = lastRight
}
if err != nil {
return nil, err
}
newLineSection := &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
language: &diffVarMutable[string]{value: opts.Language},
Path: filePath,
LastLeftIdx: lastLeft,
LastRightIdx: lastRight,
LeftIdx: idxLeft,
RightIdx: idxRight,
LeftHunkSize: leftHunkSize,
RightHunkSize: rightHunkSize,
},
}
if newLineSection.GetExpandDirection() != "" {
newLineSection.Content = fmt.Sprintf("@@ -%d,%d +%d,%d @@\n", idxLeft, leftHunkSize, idxRight, rightHunkSize)
switch direction {
case "up":
section.Lines = append([]*DiffLine{newLineSection}, section.Lines...)
case "down":
section.Lines = append(section.Lines, newLineSection)
}
}
return section, nil
}
@@ -0,0 +1,39 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package gitdiff
import (
"bytes"
"strconv"
"testing"
"code.gitea.io/gitea/modules/translation"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBuildBlobExcerptDiffSection(t *testing.T) {
data := &bytes.Buffer{}
for i := range 100 {
data.WriteString("a = " + strconv.Itoa(i+1) + "\n")
}
locale := translation.MockLocale{}
lineMiddle := 50
diffSection, err := BuildBlobExcerptDiffSection("a.py", bytes.NewReader(data.Bytes()), BlobExcerptOptions{
LeftIndex: lineMiddle,
RightIndex: lineMiddle,
LeftHunkSize: 10,
RightHunkSize: 10,
Direction: "up",
})
require.NoError(t, err)
assert.Len(t, diffSection.highlightedRightLines.value, BlobExcerptChunkSize)
assert.NotEmpty(t, diffSection.highlightedRightLines.value[lineMiddle-BlobExcerptChunkSize-1])
assert.NotEmpty(t, diffSection.highlightedRightLines.value[lineMiddle-2]) // 0-based
diffInline := diffSection.GetComputedInlineDiffFor(diffSection.Lines[1], locale)
assert.Equal(t, `<span class="n">a</span> <span class="o">=</span> <span class="mi">30</span>`+"\n", string(diffInline.Content))
}
@@ -11,6 +11,7 @@ import (
"testing"
issues_model "code.gitea.io/gitea/models/issues"
pull_model "code.gitea.io/gitea/models/pull"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
@@ -642,25 +643,489 @@ func TestNoCrashes(t *testing.T) {
}
}
func TestGeneratePatchForUnchangedLineFromReader(t *testing.T) {
tests := []struct {
name string
content string
treePath string
line int64
contextLines int
want string
wantErr bool
}{
{
name: "single line with context",
content: "line1\nline2\nline3\nline4\nline5\n",
treePath: "test.txt",
line: 3,
contextLines: 1,
want: `diff --git a/test.txt b/test.txt
--- a/test.txt
+++ b/test.txt
@@ -2,2 +2,2 @@
line2
line3
`,
},
{
name: "negative line number (left side)",
content: "line1\nline2\nline3\nline4\nline5\n",
treePath: "test.txt",
line: -3,
contextLines: 1,
want: `diff --git a/test.txt b/test.txt
--- a/test.txt
+++ b/test.txt
@@ -2,2 +2,2 @@
line2
line3
`,
},
{
name: "line near start of file",
content: "line1\nline2\nline3\n",
treePath: "test.txt",
line: 2,
contextLines: 5,
want: `diff --git a/test.txt b/test.txt
--- a/test.txt
+++ b/test.txt
@@ -1,2 +1,2 @@
line1
line2
`,
},
{
name: "first line with context",
content: "line1\nline2\nline3\n",
treePath: "test.txt",
line: 1,
contextLines: 3,
want: `diff --git a/test.txt b/test.txt
--- a/test.txt
+++ b/test.txt
@@ -1,1 +1,1 @@
line1
`,
},
{
name: "zero context lines",
content: "line1\nline2\nline3\n",
treePath: "test.txt",
line: 2,
contextLines: 0,
want: `diff --git a/test.txt b/test.txt
--- a/test.txt
+++ b/test.txt
@@ -2,1 +2,1 @@
line2
`,
},
{
name: "multi-line context",
content: "package main\n\nfunc main() {\n fmt.Println(\"Hello\")\n}\n",
treePath: "main.go",
line: 4,
contextLines: 2,
want: `diff --git a/main.go b/main.go
--- a/main.go
+++ b/main.go
@@ -2,3 +2,3 @@
<SP>
func main() {
fmt.Println("Hello")
`,
},
{
name: "empty file",
content: "",
treePath: "empty.txt",
line: 1,
contextLines: 1,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
reader := strings.NewReader(tt.content)
got, err := generatePatchForUnchangedLineFromReader(reader, tt.treePath, tt.line, tt.contextLines)
if tt.wantErr {
assert.Error(t, err)
} else {
require.NoError(t, err)
assert.Equal(t, strings.ReplaceAll(tt.want, "<SP>", " "), got)
}
})
}
}
func TestCalculateHiddenCommentIDsForLine(t *testing.T) {
tests := []struct {
name string
line *DiffLine
lineComments map[int64][]*issues_model.Comment
expected []int64
}{
{
name: "comments in hidden range",
line: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 10,
RightIdx: 20,
},
},
lineComments: map[int64][]*issues_model.Comment{
15: {{ID: 100}, {ID: 101}},
12: {{ID: 102}},
},
expected: []int64{100, 101, 102},
},
{
name: "comments outside hidden range",
line: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 10,
RightIdx: 20,
},
},
lineComments: map[int64][]*issues_model.Comment{
5: {{ID: 100}},
25: {{ID: 101}},
},
expected: nil,
},
{
name: "negative line numbers (left side)",
line: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 10,
RightIdx: 20,
},
},
lineComments: map[int64][]*issues_model.Comment{
-15: {{ID: 100}}, // Left-side comment, should NOT be counted
15: {{ID: 101}}, // Right-side comment, should be counted
},
expected: []int64{101}, // Only right-side comment
},
{
name: "boundary conditions - normal expansion (both boundaries exclusive)",
line: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 10,
RightIdx: 20,
RightHunkSize: 5, // Normal case: next section has content
},
},
lineComments: map[int64][]*issues_model.Comment{
10: {{ID: 100}}, // at LastRightIdx (visible line), should NOT be included
20: {{ID: 101}}, // at RightIdx (visible line), should NOT be included
11: {{ID: 102}}, // just inside range, should be included
19: {{ID: 103}}, // just inside range, should be included
},
expected: []int64{102, 103},
},
{
name: "boundary conditions - end of file expansion (RightIdx inclusive)",
line: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 54,
RightIdx: 70,
RightHunkSize: 0, // End of file: no more content after
},
},
lineComments: map[int64][]*issues_model.Comment{
54: {{ID: 54}}, // at LastRightIdx (visible line), should NOT be included
70: {{ID: 70}}, // at RightIdx (last hidden line), SHOULD be included
60: {{ID: 60}}, // inside range, should be included
},
expected: []int64{60, 70}, // Lines 60 and 70 are hidden
},
{
name: "real-world scenario - start of file with hunk",
line: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 0, // No previous visible section
RightIdx: 26, // Line 26 is first visible line of hunk
RightHunkSize: 9, // Normal hunk with content
},
},
lineComments: map[int64][]*issues_model.Comment{
1: {{ID: 1}}, // Line 1 is hidden
26: {{ID: 26}}, // Line 26 is visible (hunk start) - should NOT be hidden
10: {{ID: 10}}, // Line 10 is hidden
15: {{ID: 15}}, // Line 15 is hidden
},
expected: []int64{1, 10, 15}, // Lines 1, 10, 15 are hidden; line 26 is visible
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
FillHiddenCommentIDsForDiffLine(tt.line, tt.lineComments)
assert.ElementsMatch(t, tt.expected, tt.line.SectionInfo.HiddenCommentIDs)
})
}
}
func TestDiffLine_RenderBlobExcerptButtons(t *testing.T) {
tests := []struct {
name string
line *DiffLine
fileNameHash string
data *DiffBlobExcerptData
expectContains []string
expectNotContain []string
}{
{
name: "expand up button with hidden comments",
line: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 0,
RightIdx: 26,
LeftIdx: 26,
LastLeftIdx: 0,
LeftHunkSize: 0,
RightHunkSize: 0,
HiddenCommentIDs: []int64{100},
},
},
fileNameHash: "abc123",
data: &DiffBlobExcerptData{
BaseLink: "/repo/blob_excerpt",
AfterCommitID: "commit123",
DiffStyle: "unified",
},
expectContains: []string{
"octicon-fold-up",
"direction=up",
"code-comment-more",
"1 hidden comment(s)",
},
},
{
name: "expand up and down buttons with pull request",
line: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 10,
RightIdx: 50,
LeftIdx: 10,
LastLeftIdx: 5,
LeftHunkSize: 5,
RightHunkSize: 5,
HiddenCommentIDs: []int64{200, 201},
},
},
fileNameHash: "def456",
data: &DiffBlobExcerptData{
BaseLink: "/repo/blob_excerpt",
AfterCommitID: "commit456",
DiffStyle: "split",
PullIssueIndex: 42,
},
expectContains: []string{
"octicon-fold-down",
"octicon-fold-up",
"direction=down",
"direction=up",
`data-hidden-comment-ids=",200,201,"`, // use leading and trailing commas to ensure exact match by CSS selector `attr*=",id,"`
"pull_issue_index=42",
"2 hidden comment(s)",
},
},
{
name: "no hidden comments",
line: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 10,
RightIdx: 20,
LeftIdx: 10,
LastLeftIdx: 5,
LeftHunkSize: 5,
RightHunkSize: 5,
HiddenCommentIDs: nil,
},
},
fileNameHash: "ghi789",
data: &DiffBlobExcerptData{
BaseLink: "/repo/blob_excerpt",
AfterCommitID: "commit789",
},
expectContains: []string{
"code-expander-button",
},
expectNotContain: []string{
"code-comment-more",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.line.RenderBlobExcerptButtons(tt.fileNameHash, tt.data)
resultStr := string(result)
for _, expected := range tt.expectContains {
assert.Contains(t, resultStr, expected, "Expected to contain: %s", expected)
}
for _, notExpected := range tt.expectNotContain {
assert.NotContains(t, resultStr, notExpected, "Expected NOT to contain: %s", notExpected)
}
})
}
}
func TestDiffLine_GetExpandDirection(t *testing.T) {
cases := []struct {
name string
diffLine *DiffLine
direction string
}{
{
name: "NotSectionLine",
diffLine: &DiffLine{Type: DiffLineAdd, SectionInfo: &DiffLineSectionInfo{}},
direction: "",
},
{
name: "NilSectionInfo",
diffLine: &DiffLine{Type: DiffLineSection, SectionInfo: nil},
direction: "",
},
{
name: "NoHiddenLines",
// last block stops at line 100, next block starts at line 101, so no hidden lines, no expansion.
diffLine: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 100,
LastLeftIdx: 100,
RightIdx: 101,
LeftIdx: 101,
},
},
direction: "",
},
{
name: "FileHead",
diffLine: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 0, // LastXxxIdx = 0 means this is the first section in the file.
LastLeftIdx: 0,
RightIdx: 1,
LeftIdx: 1,
},
},
direction: "",
},
{
name: "FileHeadHiddenLines",
diffLine: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 0,
LastLeftIdx: 0,
RightIdx: 101,
LeftIdx: 101,
},
},
direction: "up",
},
{
name: "HiddenSingleHunk",
diffLine: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 100,
LastLeftIdx: 100,
RightIdx: 102,
LeftIdx: 102,
RightHunkSize: 1234, // non-zero dummy value
LeftHunkSize: 5678, // non-zero dummy value
},
},
direction: "single",
},
{
name: "HiddenSingleFullHunk",
// the hidden lines can exactly fit into one hunk
diffLine: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 100,
LastLeftIdx: 100,
RightIdx: 100 + BlobExcerptChunkSize + 1,
LeftIdx: 100 + BlobExcerptChunkSize + 1,
RightHunkSize: 1234, // non-zero dummy value
LeftHunkSize: 5678, // non-zero dummy value
},
},
direction: "single",
},
{
name: "HiddenUpDownHunks",
diffLine: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 100,
LastLeftIdx: 100,
RightIdx: 100 + BlobExcerptChunkSize + 2,
LeftIdx: 100 + BlobExcerptChunkSize + 2,
RightHunkSize: 1234, // non-zero dummy value
LeftHunkSize: 5678, // non-zero dummy value
},
},
direction: "updown",
},
{
name: "FileTail",
diffLine: &DiffLine{
Type: DiffLineSection,
SectionInfo: &DiffLineSectionInfo{
LastRightIdx: 100,
LastLeftIdx: 100,
RightIdx: 102,
LeftIdx: 102,
RightHunkSize: 0,
LeftHunkSize: 0,
},
},
direction: "down",
},
}
for _, c := range cases {
assert.Equal(t, c.direction, c.diffLine.GetExpandDirection(), "case %s expected direction: %s", c.name, c.direction)
}
}
func TestHighlightCodeLines(t *testing.T) {
t.Run("CharsetDetecting", func(t *testing.T) {
diffFile := &DiffFile{
Name: "a.c",
Language: "c",
Name: "a.c",
Sections: []*DiffSection{
{
Lines: []*DiffLine{{LeftIdx: 1}},
},
},
}
ret := highlightCodeLines(diffFile, true, []byte("// abc\xcc def\xcd")) // ISO-8859-1 bytes
ret := highlightCodeLinesForDiffFile(diffFile, true, []byte("// abc\xcc def\xcd")) // ISO-8859-1 bytes
assert.Equal(t, "<span class=\"c1\">// abcÌ defÍ\n</span>", string(ret[0]))
})
t.Run("LeftLines", func(t *testing.T) {
diffFile := &DiffFile{
Name: "a.c",
Language: "c",
Name: "a.c",
Sections: []*DiffSection{
{
Lines: []*DiffLine{
@@ -672,10 +1137,103 @@ func TestHighlightCodeLines(t *testing.T) {
},
}
const nl = "\n"
ret := highlightCodeLines(diffFile, true, []byte("a\nb\n"))
ret := highlightCodeLinesForDiffFile(diffFile, true, []byte("a\nb\n"))
assert.Equal(t, map[int]template.HTML{
0: `<span class="n">a</span>` + nl,
1: `<span class="n">b</span>`,
1: `<span class="n">b</span>` + nl,
}, ret)
})
}
func TestSyncUserSpecificDiff_UpdatedFiles(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
pull := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{ID: 7})
assert.NoError(t, pull.LoadBaseRepo(t.Context()))
stdin := `blob
mark :1
data 7
change
commit refs/heads/branch1
mark :2
committer test <test@example.com> 1772749114 +0000
data 7
change
from 1978192d98bb1b65e11c2cf37da854fbf94bffd6
M 100644 :1 test2.txt
M 100644 :1 test3.txt
commit refs/heads/branch1
committer test <test@example.com> 1772749114 +0000
data 7
revert
from :2
D test2.txt
D test10.txt`
require.NoError(t, gitcmd.NewCommand("fast-import").WithDir(pull.BaseRepo.RepoPath()).WithStdinBytes([]byte(stdin)).Run(t.Context()))
gitRepo, err := git.OpenRepository(t.Context(), pull.BaseRepo.RepoPath())
assert.NoError(t, err)
defer gitRepo.Close()
firstReviewCommit := "1978192d98bb1b65e11c2cf37da854fbf94bffd6"
firstReviewUpdatedFiles := map[string]pull_model.ViewedState{
"test1.txt": pull_model.Viewed,
"test2.txt": pull_model.Viewed,
"test10.txt": pull_model.Viewed,
}
_, err = pull_model.UpdateReviewState(t.Context(), user.ID, pull.ID, firstReviewCommit, firstReviewUpdatedFiles)
assert.NoError(t, err)
firstReview, err := pull_model.GetNewestReviewState(t.Context(), user.ID, pull.ID)
assert.NoError(t, err)
assert.NotNil(t, firstReview)
assert.Equal(t, firstReviewUpdatedFiles, firstReview.UpdatedFiles)
assert.Equal(t, 3, firstReview.GetViewedFileCount())
secondReviewCommit := "f80737c7dc9de0a9c1e051e83cb6897f950c6bb8"
secondReviewUpdatedFiles := map[string]pull_model.ViewedState{
"test1.txt": pull_model.Viewed,
"test2.txt": pull_model.HasChanged,
"test3.txt": pull_model.HasChanged,
"test10.txt": pull_model.Viewed,
}
secondReviewDiffOpts := &DiffOptions{
AfterCommitID: secondReviewCommit,
BeforeCommitID: pull.MergeBase,
MaxLines: setting.Git.MaxGitDiffLines,
MaxLineCharacters: setting.Git.MaxGitDiffLineCharacters,
MaxFiles: setting.Git.MaxGitDiffFiles,
}
secondReviewDiff, err := GetDiffForAPI(t.Context(), gitRepo, secondReviewDiffOpts)
assert.NoError(t, err)
secondReview, err := SyncUserSpecificDiff(t.Context(), user.ID, pull, gitRepo, secondReviewDiff, secondReviewDiffOpts)
assert.NoError(t, err)
assert.NotNil(t, secondReview)
assert.Equal(t, secondReviewUpdatedFiles, secondReview.UpdatedFiles)
assert.Equal(t, 2, secondReview.GetViewedFileCount())
thirdReviewCommit := "73424f3a99e140f6399c73a1712654e122d2a74b"
thirdReviewUpdatedFiles := map[string]pull_model.ViewedState{
"test1.txt": pull_model.Viewed,
"test2.txt": pull_model.Unviewed,
"test3.txt": pull_model.HasChanged,
"test10.txt": pull_model.Unviewed,
}
thirdReviewDiffOpts := &DiffOptions{
AfterCommitID: thirdReviewCommit,
BeforeCommitID: pull.MergeBase,
MaxLines: setting.Git.MaxGitDiffLines,
MaxLineCharacters: setting.Git.MaxGitDiffLineCharacters,
MaxFiles: setting.Git.MaxGitDiffFiles,
}
thirdReviewDiff, err := GetDiffForAPI(t.Context(), gitRepo, thirdReviewDiffOpts)
assert.NoError(t, err)
thirdReview, err := SyncUserSpecificDiff(t.Context(), user.ID, pull, gitRepo, thirdReviewDiff, thirdReviewDiffOpts)
assert.NoError(t, err)
assert.NotNil(t, thirdReview)
assert.Equal(t, thirdReviewUpdatedFiles, thirdReview.UpdatedFiles)
assert.Equal(t, 1, thirdReview.GetViewedFileCount())
}
@@ -7,12 +7,45 @@ import (
"bytes"
"html/template"
"strings"
"unicode/utf8"
"code.gitea.io/gitea/modules/util"
"github.com/sergi/go-diff/diffmatchpatch"
)
// token is a html tag or entity, eg: "<span ...>", "</span>", "&lt;"
func extractHTMLToken(s string) (before, token, after string, valid bool) {
// extractDiffTokenRemainingFullTag tries to extract full tag with content from the remaining string
// e.g. for input: "content</span>the-rest...", it returns "content</span>", "the-rest...", true
func extractDiffTokenRemainingFullTag(s string) (token, after string, valid bool) {
pos := 0
for ; pos < len(s); pos++ {
c := s[pos]
if c == '<' {
break
}
// keep in mind: even if we'd like to relax this check,
// we should never ignore "&" because it is for HTML entity and can't be safely used in the diff algorithm,
// because diff between "&lt;" and "&gt;" will generate broken result.
isSymbolChar := 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' || c == '_' || c == '-' || c == '.'
if !isSymbolChar {
return "", s, false
}
}
if pos+1 >= len(s) || s[pos+1] != '/' {
return "", s, false
}
pos2 := strings.IndexByte(s[pos:], '>')
if pos2 == -1 {
return "", s, false
}
return s[:pos+pos2+1], s[pos+pos2+1:], true
}
// Returned token:
// * full tag with content: "<<span>content</span>>", it is used to optimize diff results to highlight the whole changed symbol
// * opening/closing tag: "<span ...>" or "</span>"
// * HTML entity: "&lt;"
func extractDiffToken(s string) (before, token, after string, valid bool) {
for pos1 := 0; pos1 < len(s); pos1++ {
switch s[pos1] {
case '<':
@@ -20,7 +53,15 @@ func extractHTMLToken(s string) (before, token, after string, valid bool) {
if pos2 == -1 {
return "", "", s, false
}
return s[:pos1], s[pos1 : pos1+pos2+1], s[pos1+pos2+1:], true
before, token, after = s[:pos1], s[pos1:pos1+pos2+1], s[pos1+pos2+1:]
if !strings.HasPrefix(token, "</") {
// try to extract full tag with content, e.g. `<<span>content</span>>`, to optimize diff results
if fullTokenRemaining, fullTokenAfter, ok := extractDiffTokenRemainingFullTag(after); ok {
return before, "<" + token + fullTokenRemaining + ">", fullTokenAfter, true
}
}
return before, token, after, true
case '&':
pos2 := strings.IndexByte(s[pos1:], ';')
if pos2 == -1 {
@@ -47,7 +88,9 @@ type highlightCodeDiff struct {
placeholderOverflowCount int
lineWrapperTags []string
diffCodeAddedOpen rune
diffCodeRemovedOpen rune
diffCodeClose rune
}
func newHighlightCodeDiff() *highlightCodeDiff {
@@ -87,11 +130,26 @@ func (hcd *highlightCodeDiff) collectUsedRunes(code template.HTML) {
}
}
func (hcd *highlightCodeDiff) diffLineWithHighlight(lineType DiffLineType, codeA, codeB template.HTML) template.HTML {
return hcd.diffLineWithHighlightWrapper(nil, lineType, codeA, codeB)
func (hcd *highlightCodeDiff) diffEqualPartIsSpaceOnly(s string) bool {
for _, r := range s {
if r >= hcd.placeholderBegin {
recovered := hcd.placeholderTokenMap[r]
if strings.HasPrefix(recovered, "<<") {
return false // a full tag with content, it can't be space-only
} else if strings.HasPrefix(recovered, "<") {
continue // a single opening/closing tag, skip the tag and continue to check the content
}
return false // otherwise, it must be an HTML entity, it can't be space-only
}
isSpace := r == ' ' || r == '\t' || r == '\n' || r == '\r'
if !isSpace {
return false
}
}
return true
}
func (hcd *highlightCodeDiff) diffLineWithHighlightWrapper(lineWrapperTags []string, lineType DiffLineType, codeA, codeB template.HTML) template.HTML {
func (hcd *highlightCodeDiff) diffLineWithHighlight(lineType DiffLineType, codeA, codeB template.HTML) template.HTML {
hcd.collectUsedRunes(codeA)
hcd.collectUsedRunes(codeB)
@@ -104,32 +162,44 @@ func (hcd *highlightCodeDiff) diffLineWithHighlightWrapper(lineWrapperTags []str
buf := bytes.NewBuffer(nil)
// restore the line wrapper tags <span class="line"> and <span class="cl">, if necessary
for _, tag := range lineWrapperTags {
buf.WriteString(tag)
if hcd.diffCodeClose == 0 {
// tests can pre-set the placeholders
hcd.diffCodeAddedOpen = hcd.registerTokenAsPlaceholder(`<span class="added-code">`)
hcd.diffCodeRemovedOpen = hcd.registerTokenAsPlaceholder(`<span class="removed-code">`)
hcd.diffCodeClose = hcd.registerTokenAsPlaceholder(`</span><!-- diff-code-close -->`)
}
addedCodePrefix := hcd.registerTokenAsPlaceholder(`<span class="added-code">`)
removedCodePrefix := hcd.registerTokenAsPlaceholder(`<span class="removed-code">`)
codeTagSuffix := hcd.registerTokenAsPlaceholder(`</span>`)
equalPartSpaceOnly := true
for _, diff := range diffs {
if diff.Type != diffmatchpatch.DiffEqual {
continue
}
if equalPartSpaceOnly = hcd.diffEqualPartIsSpaceOnly(diff.Text); !equalPartSpaceOnly {
break
}
}
if codeTagSuffix != 0 {
// only add "added"/"removed" tags when needed:
// * non-space contents appear in the DiffEqual parts (not a full-line add/del)
// * placeholder map still works (not exhausted, can get the closing tag placeholder)
addDiffTags := !equalPartSpaceOnly && hcd.diffCodeClose != 0
if addDiffTags {
for _, diff := range diffs {
switch {
case diff.Type == diffmatchpatch.DiffEqual:
buf.WriteString(diff.Text)
case diff.Type == diffmatchpatch.DiffInsert && lineType == DiffLineAdd:
buf.WriteRune(addedCodePrefix)
buf.WriteRune(hcd.diffCodeAddedOpen)
buf.WriteString(diff.Text)
buf.WriteRune(codeTagSuffix)
buf.WriteRune(hcd.diffCodeClose)
case diff.Type == diffmatchpatch.DiffDelete && lineType == DiffLineDel:
buf.WriteRune(removedCodePrefix)
buf.WriteRune(hcd.diffCodeRemovedOpen)
buf.WriteString(diff.Text)
buf.WriteRune(codeTagSuffix)
buf.WriteRune(hcd.diffCodeClose)
}
}
} else {
// placeholder map space is exhausted
// the caller will still add added/removed backgrounds for the whole line
for _, diff := range diffs {
take := diff.Type == diffmatchpatch.DiffEqual || (diff.Type == diffmatchpatch.DiffInsert && lineType == DiffLineAdd) || (diff.Type == diffmatchpatch.DiffDelete && lineType == DiffLineDel)
if take {
@@ -137,19 +207,21 @@ func (hcd *highlightCodeDiff) diffLineWithHighlightWrapper(lineWrapperTags []str
}
}
}
for range lineWrapperTags {
buf.WriteString("</span>")
}
return hcd.recoverOneDiff(buf.String())
}
func (hcd *highlightCodeDiff) registerTokenAsPlaceholder(token string) rune {
recovered := token
if token[0] == '<' && token[1] != '<' {
// when recovering a single tag, only use the tag itself, ignore the trailing comment (for how the comment is generated, see the code in `convert` function)
recovered = token[:strings.IndexByte(token, '>')+1]
}
placeholder, ok := hcd.tokenPlaceholderMap[token]
if !ok {
placeholder = hcd.nextPlaceholder()
if placeholder != 0 {
hcd.tokenPlaceholderMap[token] = placeholder
hcd.placeholderTokenMap[placeholder] = token
hcd.placeholderTokenMap[placeholder] = recovered
}
}
return placeholder
@@ -160,44 +232,42 @@ func (hcd *highlightCodeDiff) convertToPlaceholders(htmlContent template.HTML) s
var tagStack []string
res := strings.Builder{}
firstRunForLineTags := hcd.lineWrapperTags == nil
htmlCode := string(htmlContent)
var beforeToken, token string
var valid bool
htmlCode := string(htmlContent)
// the standard chroma highlight HTML is "<span class="line [hl]"><span class="cl"> ... </span></span>"
for {
beforeToken, token, htmlCode, valid = extractHTMLToken(htmlCode)
beforeToken, token, htmlCode, valid = extractDiffToken(htmlCode)
if !valid || token == "" {
break
}
// write the content before the token into result string, and consume the token in the string
res.WriteString(beforeToken)
// the standard chroma highlight HTML is `<span class="line [hl]"><span class="cl"> ... </span></span>`
// the line wrapper tags should be removed before diff
if strings.HasPrefix(token, `<span class="line`) || strings.HasPrefix(token, `<span class="cl"`) {
if firstRunForLineTags {
// if this is the first run for converting, save the line wrapper tags for later use, they should be added back
hcd.lineWrapperTags = append(hcd.lineWrapperTags, token)
}
htmlCode = strings.TrimSuffix(htmlCode, "</span>")
continue
}
var tokenInMap string
if strings.HasSuffix(token, "</") { // for closing tag
if strings.HasPrefix(token, "</") { // for closing tag
if len(tagStack) == 0 {
break // invalid diff result, no opening tag but see closing tag
continue // no opening tag but see closing tag, skip it
}
// make sure the closing tag in map is related to the open tag, to make the diff algorithm can match the opening/closing tags
// the closing tag will be recorded in the map by key "</span><!-- <span the-opening> -->" for "<span the-opening>"
tokenInMap = token + "<!-- " + tagStack[len(tagStack)-1] + "-->"
tagStack = tagStack[:len(tagStack)-1]
} else if token[0] == '<' { // for opening tag
tokenInMap = token
tagStack = append(tagStack, token)
} else if token[0] == '&' { // for html entity
} else if token[0] == '<' {
if token[1] == '<' {
// full tag `<<span>content</span>>`, recover to `<span>content</span>`
tokenInMap = token
} else {
// opening tag
tokenInMap = token
tagStack = append(tagStack, token)
}
} else if token[0] == '&' { // for HTML entity
tokenInMap = token
} // else: impossible
@@ -210,8 +280,13 @@ func (hcd *highlightCodeDiff) convertToPlaceholders(htmlContent template.HTML) s
// unfortunately, all private use runes has been exhausted, no more placeholder could be used, no more converting
// usually, the exhausting won't occur in real cases, the magnitude of used placeholders is not larger than that of the CSS classes outputted by chroma.
hcd.placeholderOverflowCount++
if strings.HasPrefix(token, "<<") {
pos1 := strings.IndexByte(token, '>')
pos2 := strings.LastIndexByte(token, '<')
res.WriteString(token[pos1+1 : pos2]) // recover to `content` from "<<span>content</span>>"
}
if strings.HasPrefix(token, "&") {
// when the token is a html entity, something must be outputted even if there is no placeholder.
// when the token is an HTML entity, something must be outputted even if there is no placeholder.
res.WriteRune(0xFFFD) // replacement character TODO: how to handle this case more gracefully?
res.WriteString(token[1:]) // still output the entity code part, otherwise there will be no diff result.
}
@@ -223,43 +298,99 @@ func (hcd *highlightCodeDiff) convertToPlaceholders(htmlContent template.HTML) s
return res.String()
}
// recoverOneRune tries to recover one rune
// * if the rune is a placeholder, it will be recovered to the corresponding content
// * otherwise it will be returned as is
func (hcd *highlightCodeDiff) recoverOneRune(buf []byte) (r rune, runeLen int, isSingleTag bool, recovered string) {
r, runeLen = utf8.DecodeRune(buf)
token := hcd.placeholderTokenMap[r]
if token == "" {
return r, runeLen, false, "" // rune itself, not a placeholder
} else if token[0] == '<' {
if token[1] == '<' {
return 0, runeLen, false, token[1 : len(token)-1] // full tag `<<span>content</span>>`, recover to `<span>content</span>`
}
return r, runeLen, true, token // single tag
}
return 0, runeLen, false, token // HTML entity
}
func (hcd *highlightCodeDiff) recoverOneDiff(str string) template.HTML {
sb := strings.Builder{}
var tagStack []string
var diffCodeOpenTag string
diffCodeCloseTag := hcd.placeholderTokenMap[hcd.diffCodeClose]
strBytes := util.UnsafeStringToBytes(str)
for _, r := range str {
token, ok := hcd.placeholderTokenMap[r]
if !ok || token == "" {
sb.WriteRune(r) // if the rune is not a placeholder, write it as it is
continue
}
var tokenToRecover string
if strings.HasPrefix(token, "</") { // for closing tag
// only get the tag itself, ignore the trailing comment (for how the comment is generated, see the code in `convert` function)
tokenToRecover = token[:strings.IndexByte(token, '>')+1]
if len(tagStack) == 0 {
continue // if no opening tag in stack yet, skip the closing tag
// this loop is slightly longer than expected, for performance consideration
for idx := 0; idx < len(strBytes); {
// take a look at the next rune
r, runeLen, isSingleTag, recovered := hcd.recoverOneRune(strBytes[idx:])
idx += runeLen
// loop section 1: if it isn't a single tag, then try to find the following runes until the next single tag, and recover them together
if !isSingleTag {
if diffCodeOpenTag != "" {
// start the "added/removed diff tag" if the current token is in the diff part
sb.WriteString(diffCodeOpenTag)
}
tagStack = tagStack[:len(tagStack)-1]
} else if token[0] == '<' { // for opening tag
tokenToRecover = token
tagStack = append(tagStack, token)
} else if token[0] == '&' { // for html entity
tokenToRecover = token
} // else: impossible
sb.WriteString(tokenToRecover)
if recovered != "" {
sb.WriteString(recovered)
} else {
sb.WriteRune(r)
}
// inner loop to recover following runes until the next single tag
for idx < len(strBytes) {
r, runeLen, isSingleTag, recovered = hcd.recoverOneRune(strBytes[idx:])
idx += runeLen
if isSingleTag {
break
}
if recovered != "" {
sb.WriteString(recovered)
} else {
sb.WriteRune(r)
}
}
if diffCodeOpenTag != "" {
// end the "added/removed diff tag" if the current token is in the diff part
sb.WriteString(diffCodeCloseTag)
}
}
if !isSingleTag {
break // the inner loop has already consumed all remaining runes, no more single tag found
}
// loop section 2: for opening/closing HTML tags
placeholder := r
if recovered[1] != '/' { // opening tag
if placeholder == hcd.diffCodeAddedOpen || placeholder == hcd.diffCodeRemovedOpen {
diffCodeOpenTag = recovered
recovered = ""
} else {
tagStack = append(tagStack, recovered)
}
} else { // closing tag
if placeholder == hcd.diffCodeClose {
diffCodeOpenTag = "" // the highlighted diff is closed, no more diff
recovered = ""
} else if len(tagStack) != 0 {
tagStack = tagStack[:len(tagStack)-1]
} else {
recovered = ""
}
}
sb.WriteString(recovered)
}
if len(tagStack) > 0 {
// close all opening tags
for i := len(tagStack) - 1; i >= 0; i-- {
tagToClose := tagStack[i]
// get the closing tag "</span>" from "<span class=...>" or "<span>"
pos := strings.IndexAny(tagToClose, " >")
if pos != -1 {
sb.WriteString("</" + tagToClose[1:pos] + ">")
} // else: impossible. every tag was pushed into the stack by the code above and is valid HTML opening tag
}
// close all opening tags
for i := len(tagStack) - 1; i >= 0; i-- {
tagToClose := tagStack[i]
// get the closing tag "</span>" from "<span class=...>" or "<span>"
pos := strings.IndexAny(tagToClose, " >")
// pos must be positive, because the tags were pushed by us
sb.WriteString("</" + tagToClose[1:pos] + ">")
}
return template.HTML(sb.String())
}
@@ -9,28 +9,62 @@ import (
"strings"
"testing"
"code.gitea.io/gitea/modules/highlight"
"github.com/stretchr/testify/assert"
)
func TestDiffWithHighlight(t *testing.T) {
t.Run("DiffLineAddDel", func(t *testing.T) {
func BenchmarkHighlightDiff(b *testing.B) {
for b.Loop() {
// still fast enough: BenchmarkHighlightDiff-12 1000000 1027 ns/op
// TODO: the real bottleneck is that "diffLineWithHighlight" is called twice when rendering "added" and "removed" lines by the caller
// Ideally the caller should cache the diff result, and then use the diff result to render "added" and "removed" lines separately
hcd := newHighlightCodeDiff()
codeA := template.HTML(`x <span class="k">foo</span> y`)
codeB := template.HTML(`x <span class="k">bar</span> y`)
outDel := hcd.diffLineWithHighlight(DiffLineDel, codeA, codeB)
assert.Equal(t, `x <span class="k"><span class="removed-code">foo</span></span> y`, string(outDel))
outAdd := hcd.diffLineWithHighlight(DiffLineAdd, codeA, codeB)
assert.Equal(t, `x <span class="k"><span class="added-code">bar</span></span> y`, string(outAdd))
hcd.diffLineWithHighlight(DiffLineDel, codeA, codeB)
}
}
func TestDiffWithHighlight(t *testing.T) {
t.Run("DiffLineAddDel", func(t *testing.T) {
t.Run("WithDiffTags", func(t *testing.T) {
hcd := newHighlightCodeDiff()
codeA := template.HTML(`x <span class="k">foo</span> y`)
codeB := template.HTML(`x <span class="k">bar</span> y`)
outDel := hcd.diffLineWithHighlight(DiffLineDel, codeA, codeB)
assert.Equal(t, `x <span class="removed-code"><span class="k">foo</span></span> y`, string(outDel))
outAdd := hcd.diffLineWithHighlight(DiffLineAdd, codeA, codeB)
assert.Equal(t, `x <span class="added-code"><span class="k">bar</span></span> y`, string(outAdd))
})
t.Run("NoRedundantTags", func(t *testing.T) {
// the equal parts only contain spaces, in this case, don't use "added/removed" tags
// because the diff lines already have a background color to indicate the change
hcd := newHighlightCodeDiff()
codeA := template.HTML("<span> </span> \t<span>foo</span> ")
codeB := template.HTML(" <span>bar</span> \n")
outDel := hcd.diffLineWithHighlight(DiffLineDel, codeA, codeB)
assert.Equal(t, string(codeA), string(outDel))
outAdd := hcd.diffLineWithHighlight(DiffLineAdd, codeA, codeB)
assert.Equal(t, string(codeB), string(outAdd))
})
})
t.Run("CleanUp", func(t *testing.T) {
hcd := newHighlightCodeDiff()
codeA := template.HTML(`<span class="cm>this is a comment</span>`)
codeB := template.HTML(`<span class="cm>this is updated comment</span>`)
codeA := template.HTML(` <span class="cm">this is a comment</span>`)
codeB := template.HTML(` <span class="cm">this is updated comment</span>`)
outDel := hcd.diffLineWithHighlight(DiffLineDel, codeA, codeB)
assert.Equal(t, `<span class="cm>this is <span class="removed-code">a</span> comment</span>`, string(outDel))
assert.Equal(t, ` <span class="cm">this is <span class="removed-code">a</span> comment</span>`, string(outDel))
outAdd := hcd.diffLineWithHighlight(DiffLineAdd, codeA, codeB)
assert.Equal(t, `<span class="cm>this is <span class="added-code">updated</span> comment</span>`, string(outAdd))
assert.Equal(t, ` <span class="cm">this is <span class="added-code">updated</span> comment</span>`, string(outAdd))
codeA = `<span class="line"><span>line1</span></span>` + "\n" + `<span class="cl"><span>line2</span></span>`
codeB = `<span class="cl"><span>line1</span></span>` + "\n" + `<span class="line"><span>line!</span></span>`
outDel = hcd.diffLineWithHighlight(DiffLineDel, codeA, codeB)
assert.Equal(t, `<span>line1</span>`+"\n"+`<span class="removed-code"><span>line2</span></span>`, string(outDel))
outAdd = hcd.diffLineWithHighlight(DiffLineAdd, codeA, codeB)
assert.Equal(t, `<span>line1</span>`+"\n"+`<span><span class="added-code">line!</span></span>`, string(outAdd))
})
t.Run("OpenCloseTags", func(t *testing.T) {
@@ -40,6 +74,55 @@ func TestDiffWithHighlight(t *testing.T) {
assert.Equal(t, "<span></span>", string(hcd.recoverOneDiff("O")))
assert.Empty(t, string(hcd.recoverOneDiff("C")))
})
t.Run("ComplexDiff1", func(t *testing.T) {
oldCode, _, _ := highlight.RenderCodeSlowGuess("a.go", "Go", `xxx || yyy`)
newCode, _, _ := highlight.RenderCodeSlowGuess("a.go", "Go", `bot&xxx || bot&yyy`)
hcd := newHighlightCodeDiff()
out := hcd.diffLineWithHighlight(DiffLineAdd, oldCode, newCode)
assert.Equal(t, strings.ReplaceAll(`
<span class="added-code"><span class="nx">bot</span></span><span class="o"><span class="added-code">&amp;</span></span>
<span class="nx">xxx</span><span class="w"> </span><span class="o">||</span><span class="w"> </span>
<span class="added-code"><span class="nx">bot</span></span><span class="o"><span class="added-code">&amp;</span></span>
<span class="nx">yyy</span>`, "\n", ""), string(out))
})
forceTokenAsPlaceholder := func(hcd *highlightCodeDiff, r rune, token string) rune {
// for testing purpose only
hcd.tokenPlaceholderMap[token] = r
hcd.placeholderTokenMap[r] = token
return r
}
t.Run("ComplexDiff2", func(t *testing.T) {
// When running "diffLineWithHighlight", the newly inserted "added-code", and "removed-code" tags may break the original layout.
// The newly inserted tags can appear in any position, because the "diff" algorithm can make outputs like:
// * Equal: <span>
// * Insert: xx</span><span>yy
// * Equal: zz</span>
// Then the newly inserted tags will make this output, the tags mismatch.
// * <span> <added>xx</span><span>yy</added> zz</span>
// So we need to fix it to:
// * <span> <added>xx</added></span> <span><added>yy</added> zz</span>
hcd := newHighlightCodeDiff()
hcd.diffCodeAddedOpen = forceTokenAsPlaceholder(hcd, '[', "<add>")
hcd.diffCodeClose = forceTokenAsPlaceholder(hcd, ']', "</add>")
forceTokenAsPlaceholder(hcd, '{', "<T>")
forceTokenAsPlaceholder(hcd, '}', "</T>")
assert.Equal(t, `aa<T>xx<add>yy</add>zz</T>bb`, string(hcd.recoverOneDiff("aa{xx[yy]zz}bb")))
assert.Equal(t, `aa<add>xx</add><T><add>yy</add></T><add>zz</add>bb`, string(hcd.recoverOneDiff("aa[xx{yy}zz]bb")))
assert.Equal(t, `aa<T>xx<add>yy</add></T><add>zz</add>bb`, string(hcd.recoverOneDiff("aa{xx[yy}zz]bb")))
assert.Equal(t, `aa<add>xx</add><T><add>yy</add>zz</T>bb`, string(hcd.recoverOneDiff("aa[xx{yy]zz}bb")))
assert.Equal(t, `aa<add>xx</add><T><add>yy</add><add>zz</add></T><add>bb</add>cc`, string(hcd.recoverOneDiff("aa[xx{yy][zz}bb]cc")))
// And do a simple test for "diffCodeRemovedOpen", it shares the same logic as "diffCodeAddedOpen"
hcd = newHighlightCodeDiff()
hcd.diffCodeRemovedOpen = forceTokenAsPlaceholder(hcd, '[', "<del>")
hcd.diffCodeClose = forceTokenAsPlaceholder(hcd, ']', "</del>")
forceTokenAsPlaceholder(hcd, '{', "<T>")
forceTokenAsPlaceholder(hcd, '}', "</T>")
assert.Equal(t, `aa<del>xx</del><T><del>yy</del><del>zz</del></T><del>bb</del>cc`, string(hcd.recoverOneDiff("aa[xx{yy][zz}bb]cc")))
})
}
func TestDiffWithHighlightPlaceholder(t *testing.T) {
@@ -64,6 +147,11 @@ func TestDiffWithHighlightPlaceholderExhausted(t *testing.T) {
assert.Equal(t, placeHolderAmp+"lt;", string(output))
output = hcd.diffLineWithHighlight(DiffLineAdd, `<span class="k">&lt;</span>`, `<span class="k">&gt;</span>`)
assert.Equal(t, placeHolderAmp+"gt;", string(output))
output = hcd.diffLineWithHighlight(DiffLineDel, `<span class="k">foo</span>`, `<span class="k">bar</span>`)
assert.Equal(t, "foo", string(output))
output = hcd.diffLineWithHighlight(DiffLineAdd, `<span class="k">foo</span>`, `<span class="k">bar</span>`)
assert.Equal(t, "bar", string(output))
}
func TestDiffWithHighlightTagMatch(t *testing.T) {