feat: vendor gitea 1.16.2

This commit is contained in:
2026-06-02 08:36:01 +00:00
parent 247cd60f69
commit 54798b4615
2145 changed files with 162965 additions and 126447 deletions
@@ -1,115 +0,0 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build ignore
package main
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/setting"
)
func main() {
if len(os.Args) != 2 {
println("usage: backport-locales <to-ref>")
println("eg: backport-locales release/v1.19")
os.Exit(1)
}
mustNoErr := func(err error) {
if err != nil {
panic(err)
}
}
collectInis := func(ref string) map[string]setting.ConfigProvider {
inis := map[string]setting.ConfigProvider{}
err := filepath.WalkDir("options/locale", func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.HasSuffix(d.Name(), ".ini") {
return nil
}
cfg, err := setting.NewConfigProviderForLocale(path)
mustNoErr(err)
inis[path] = cfg
fmt.Printf("collecting: %s @ %s\n", path, ref)
return nil
})
mustNoErr(err)
return inis
}
// collect new locales from current working directory
inisNew := collectInis("HEAD")
// switch to the target ref, and collect the old locales
cmd := exec.Command("git", "checkout", os.Args[1])
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
mustNoErr(cmd.Run())
inisOld := collectInis(os.Args[1])
// use old en-US as the base, and copy the new translations to the old locales
enUsOld := inisOld["options/locale/locale_en-US.ini"]
brokenWarned := make(container.Set[string])
for path, iniOld := range inisOld {
if iniOld == enUsOld {
continue
}
iniNew := inisNew[path]
if iniNew == nil {
continue
}
for _, secEnUS := range enUsOld.Sections() {
secOld := iniOld.Section(secEnUS.Name())
secNew := iniNew.Section(secEnUS.Name())
for _, keyEnUs := range secEnUS.Keys() {
if secNew.HasKey(keyEnUs.Name()) {
oldStr := secOld.Key(keyEnUs.Name()).String()
newStr := secNew.Key(keyEnUs.Name()).String()
broken := oldStr != "" && strings.Count(oldStr, "%") != strings.Count(newStr, "%")
broken = broken || strings.Contains(oldStr, "\n") || strings.Contains(oldStr, "\n")
if broken {
brokenWarned.Add(secOld.Name() + "." + keyEnUs.Name())
fmt.Println("----")
fmt.Printf("WARNING: skip broken locale: %s , [%s] %s\n", path, secEnUS.Name(), keyEnUs.Name())
fmt.Printf("\told: %s\n", strings.ReplaceAll(oldStr, "\n", "\\n"))
fmt.Printf("\tnew: %s\n", strings.ReplaceAll(newStr, "\n", "\\n"))
continue
}
secOld.Key(keyEnUs.Name()).SetValue(newStr)
}
}
}
mustNoErr(iniOld.SaveTo(path))
}
fmt.Println("========")
for path, iniNew := range inisNew {
for _, sec := range iniNew.Sections() {
for _, key := range sec.Keys() {
str := sec.Key(key.Name()).String()
broken := strings.Contains(str, "\n")
broken = broken || strings.HasPrefix(str, "`") != strings.HasSuffix(str, "`")
broken = broken || strings.HasPrefix(str, "\"`")
broken = broken || strings.HasPrefix(str, "`\"")
broken = broken || strings.Count(str, `"`)%2 == 1
broken = broken || strings.Count(str, "`")%2 == 1
if broken && !brokenWarned.Contains(sec.Name()+"."+key.Name()) {
fmt.Printf("WARNING: found broken locale: %s , [%s] %s\n", path, sec.Name(), key.Name())
fmt.Printf("\tstr: %s\n", strings.ReplaceAll(str, "\n", "\\n"))
fmt.Println("----")
}
}
}
}
}
@@ -1,281 +0,0 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build ignore
package main
import (
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"code.gitea.io/gitea/build/codeformat"
)
// Windows has a limitation for command line arguments, the size can not exceed 32KB.
// So we have to feed the files to some tools (like gofmt) batch by batch
// We also introduce a `gitea-fmt` command, it does better import formatting than gofmt/goimports. `gitea-fmt` calls `gofmt` internally.
var optionLogVerbose bool
func logVerbose(msg string, args ...any) {
if optionLogVerbose {
log.Printf(msg, args...)
}
}
func passThroughCmd(cmd string, args []string) error {
foundCmd, err := exec.LookPath(cmd)
if err != nil {
log.Fatalf("can not find cmd: %s", cmd)
}
c := exec.Cmd{
Path: foundCmd,
Args: append([]string{cmd}, args...),
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
return c.Run()
}
type fileCollector struct {
dirs []string
includePatterns []*regexp.Regexp
excludePatterns []*regexp.Regexp
batchSize int
}
func newFileCollector(fileFilter string, batchSize int) (*fileCollector, error) {
co := &fileCollector{batchSize: batchSize}
if fileFilter == "go-own" {
co.dirs = []string{
"build",
"cmd",
"contrib",
"tests",
"models",
"modules",
"routers",
"services",
}
co.includePatterns = append(co.includePatterns, regexp.MustCompile(`.*\.go$`))
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`.*\bbindata\.go$`))
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`\.pb\.go$`))
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`tests/gitea-repositories-meta`))
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`tests/integration/migration-test`))
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`modules/git/tests`))
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`models/fixtures`))
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`models/migrations/fixtures`))
co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`services/gitdiff/testdata`))
}
if co.dirs == nil {
return nil, fmt.Errorf("unknown file-filter: %s", fileFilter)
}
return co, nil
}
func (fc *fileCollector) matchPatterns(path string, regexps []*regexp.Regexp) bool {
path = strings.ReplaceAll(path, "\\", "/")
for _, re := range regexps {
if re.MatchString(path) {
return true
}
}
return false
}
func (fc *fileCollector) collectFiles() (res [][]string, err error) {
var batch []string
for _, dir := range fc.dirs {
err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
include := len(fc.includePatterns) == 0 || fc.matchPatterns(path, fc.includePatterns)
exclude := fc.matchPatterns(path, fc.excludePatterns)
process := include && !exclude
if !process {
if d.IsDir() {
if exclude {
logVerbose("exclude dir %s", path)
return filepath.SkipDir
}
// for a directory, if it is not excluded explicitly, we should walk into
return nil
}
// for a file, we skip it if it shouldn't be processed
logVerbose("skip process %s", path)
return nil
}
if d.IsDir() {
// skip dir, we don't add dirs to the file list now
return nil
}
if len(batch) >= fc.batchSize {
res = append(res, batch)
batch = nil
}
batch = append(batch, path)
return nil
})
if err != nil {
return nil, err
}
}
res = append(res, batch)
return res, nil
}
// substArgFiles expands the {file-list} to a real file list for commands
func substArgFiles(args, files []string) []string {
for i, s := range args {
if s == "{file-list}" {
newArgs := append(args[:i], files...)
newArgs = append(newArgs, args[i+1:]...)
return newArgs
}
}
return args
}
func exitWithCmdErrors(subCmd string, subArgs []string, cmdErrors []error) {
for _, err := range cmdErrors {
if err != nil {
if exitError, ok := err.(*exec.ExitError); ok {
exitCode := exitError.ExitCode()
log.Printf("run command failed (code=%d): %s %v", exitCode, subCmd, subArgs)
os.Exit(exitCode)
} else {
log.Fatalf("run command failed (err=%s) %s %v", err, subCmd, subArgs)
}
}
}
}
func parseArgs() (mainOptions map[string]string, subCmd string, subArgs []string) {
mainOptions = map[string]string{}
for i := 1; i < len(os.Args); i++ {
arg := os.Args[i]
if arg == "" {
break
}
if arg[0] == '-' {
arg = strings.TrimPrefix(arg, "-")
arg = strings.TrimPrefix(arg, "-")
fields := strings.SplitN(arg, "=", 2)
if len(fields) == 1 {
mainOptions[fields[0]] = "1"
} else {
mainOptions[fields[0]] = fields[1]
}
} else {
subCmd = arg
subArgs = os.Args[i+1:]
break
}
}
return mainOptions, subCmd, subArgs
}
func showUsage() {
fmt.Printf(`Usage: %[1]s [options] {command} [arguments]
Options:
--verbose
--file-filter=go-own
--batch-size=100
Commands:
%[1]s gofmt ...
Arguments:
{file-list} the file list
Example:
%[1]s gofmt -s -d {file-list}
`, "file-batch-exec")
}
func newFileCollectorFromMainOptions(mainOptions map[string]string) (fc *fileCollector, err error) {
fileFilter := mainOptions["file-filter"]
if fileFilter == "" {
fileFilter = "go-own"
}
batchSize, _ := strconv.Atoi(mainOptions["batch-size"])
if batchSize == 0 {
batchSize = 100
}
return newFileCollector(fileFilter, batchSize)
}
func containsString(a []string, s string) bool {
for _, v := range a {
if v == s {
return true
}
}
return false
}
func giteaFormatGoImports(files []string, doWriteFile bool) error {
for _, file := range files {
if err := codeformat.FormatGoImports(file, doWriteFile); err != nil {
log.Printf("failed to format go imports: %s, err=%v", file, err)
return err
}
}
return nil
}
func main() {
mainOptions, subCmd, subArgs := parseArgs()
if subCmd == "" {
showUsage()
os.Exit(1)
}
optionLogVerbose = mainOptions["verbose"] != ""
fc, err := newFileCollectorFromMainOptions(mainOptions)
if err != nil {
log.Fatalf("can not create file collector: %s", err.Error())
}
fileBatches, err := fc.collectFiles()
if err != nil {
log.Fatalf("can not collect files: %s", err.Error())
}
processed := 0
var cmdErrors []error
for _, files := range fileBatches {
if len(files) == 0 {
break
}
substArgs := substArgFiles(subArgs, files)
logVerbose("batch cmd: %s %v", subCmd, substArgs)
switch subCmd {
case "gitea-fmt":
if containsString(subArgs, "-d") {
log.Print("the -d option is not supported by gitea-fmt")
}
cmdErrors = append(cmdErrors, giteaFormatGoImports(files, containsString(subArgs, "-w")))
cmdErrors = append(cmdErrors, passThroughCmd("gofmt", append([]string{"-w", "-r", "interface{} -> any"}, substArgs...)))
cmdErrors = append(cmdErrors, passThroughCmd("go", append([]string{"run", os.Getenv("GOFUMPT_PACKAGE"), "-extra"}, substArgs...)))
default:
log.Fatalf("unknown cmd: %s %v", subCmd, subArgs)
}
processed += len(files)
}
logVerbose("processed %d files", processed)
exitWithCmdErrors(subCmd, subArgs, cmdErrors)
}
@@ -1,195 +0,0 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package codeformat
import (
"bytes"
"errors"
"io"
"os"
"sort"
"strings"
)
var importPackageGroupOrders = map[string]int{
"": 1, // internal
"code.gitea.io/gitea/": 2,
}
var errInvalidCommentBetweenImports = errors.New("comments between imported packages are invalid, please move comments to the end of the package line")
var (
importBlockBegin = []byte("\nimport (\n")
importBlockEnd = []byte("\n)")
)
type importLineParsed struct {
group string
pkg string
content string
}
func parseImportLine(line string) (*importLineParsed, error) {
il := &importLineParsed{content: line}
p1 := strings.IndexRune(line, '"')
if p1 == -1 {
return nil, errors.New("invalid import line: " + line)
}
p1++
p := strings.IndexRune(line[p1:], '"')
if p == -1 {
return nil, errors.New("invalid import line: " + line)
}
p2 := p1 + p
il.pkg = line[p1:p2]
pDot := strings.IndexRune(il.pkg, '.')
pSlash := strings.IndexRune(il.pkg, '/')
if pDot != -1 && pDot < pSlash {
il.group = "domain-package"
}
for groupName := range importPackageGroupOrders {
if groupName == "" {
continue // skip internal
}
if strings.HasPrefix(il.pkg, groupName) {
il.group = groupName
}
}
return il, nil
}
type (
importLineGroup []*importLineParsed
importLineGroupMap map[string]importLineGroup
)
func formatGoImports(contentBytes []byte) ([]byte, error) {
p1 := bytes.Index(contentBytes, importBlockBegin)
if p1 == -1 {
return nil, nil
}
p1 += len(importBlockBegin)
p := bytes.Index(contentBytes[p1:], importBlockEnd)
if p == -1 {
return nil, nil
}
p2 := p1 + p
importGroups := importLineGroupMap{}
r := bytes.NewBuffer(contentBytes[p1:p2])
eof := false
for !eof {
line, err := r.ReadString('\n')
eof = err == io.EOF
if err != nil && !eof {
return nil, err
}
line = strings.TrimSpace(line)
if line != "" {
if strings.HasPrefix(line, "//") || strings.HasPrefix(line, "/*") {
return nil, errInvalidCommentBetweenImports
}
importLine, err := parseImportLine(line)
if err != nil {
return nil, err
}
importGroups[importLine.group] = append(importGroups[importLine.group], importLine)
}
}
var groupNames []string
for groupName, importLines := range importGroups {
groupNames = append(groupNames, groupName)
sort.Slice(importLines, func(i, j int) bool {
return strings.Compare(importLines[i].pkg, importLines[j].pkg) < 0
})
}
sort.Slice(groupNames, func(i, j int) bool {
n1 := groupNames[i]
n2 := groupNames[j]
o1 := importPackageGroupOrders[n1]
o2 := importPackageGroupOrders[n2]
if o1 != 0 && o2 != 0 {
return o1 < o2
}
if o1 == 0 && o2 == 0 {
return strings.Compare(n1, n2) < 0
}
return o1 != 0
})
formattedBlock := bytes.Buffer{}
for _, groupName := range groupNames {
hasNormalImports := false
hasDummyImports := false
// non-dummy import comes first
for _, importLine := range importGroups[groupName] {
if strings.HasPrefix(importLine.content, "_") {
hasDummyImports = true
} else {
formattedBlock.WriteString("\t" + importLine.content + "\n")
hasNormalImports = true
}
}
// dummy (_ "pkg") comes later
if hasDummyImports {
if hasNormalImports {
formattedBlock.WriteString("\n")
}
for _, importLine := range importGroups[groupName] {
if strings.HasPrefix(importLine.content, "_") {
formattedBlock.WriteString("\t" + importLine.content + "\n")
}
}
}
formattedBlock.WriteString("\n")
}
formattedBlockBytes := bytes.TrimRight(formattedBlock.Bytes(), "\n")
var formattedBytes []byte
formattedBytes = append(formattedBytes, contentBytes[:p1]...)
formattedBytes = append(formattedBytes, formattedBlockBytes...)
formattedBytes = append(formattedBytes, contentBytes[p2:]...)
return formattedBytes, nil
}
// FormatGoImports format the imports by our rules (see unit tests)
func FormatGoImports(file string, doWriteFile bool) error {
f, err := os.Open(file)
if err != nil {
return err
}
var contentBytes []byte
{
defer f.Close()
contentBytes, err = io.ReadAll(f)
if err != nil {
return err
}
}
formattedBytes, err := formatGoImports(contentBytes)
if err != nil {
return err
}
if formattedBytes == nil {
return nil
}
if bytes.Equal(contentBytes, formattedBytes) {
return nil
}
if doWriteFile {
f, err = os.OpenFile(file, os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
return err
}
defer f.Close()
_, err = f.Write(formattedBytes)
return err
}
return err
}
@@ -1,124 +0,0 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package codeformat
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestFormatImportsSimple(t *testing.T) {
formatted, err := formatGoImports([]byte(`
package codeformat
import (
"github.com/stretchr/testify/assert"
"testing"
)
`))
expected := `
package codeformat
import (
"testing"
"github.com/stretchr/testify/assert"
)
`
assert.NoError(t, err)
assert.Equal(t, expected, string(formatted))
}
func TestFormatImportsGroup(t *testing.T) {
// gofmt/goimports won't group the packages, for example, they produce such code:
// "bytes"
// "image"
// (a blank line)
// "fmt"
// "image/color/palette"
// our formatter does better, and these packages are grouped into one.
formatted, err := formatGoImports([]byte(`
package test
import (
"bytes"
"fmt"
"image"
"image/color"
_ "image/gif" // for processing gif images
_ "image/jpeg" // for processing jpeg images
_ "image/png" // for processing png images
"code.gitea.io/other/package"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"xorm.io/the/package"
"github.com/issue9/identicon"
"github.com/nfnt/resize"
"github.com/oliamb/cutter"
)
`))
expected := `
package test
import (
"bytes"
"fmt"
"image"
"image/color"
_ "image/gif" // for processing gif images
_ "image/jpeg" // for processing jpeg images
_ "image/png" // for processing png images
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/other/package"
"github.com/issue9/identicon"
"github.com/nfnt/resize"
"github.com/oliamb/cutter"
"xorm.io/the/package"
)
`
assert.NoError(t, err)
assert.Equal(t, expected, string(formatted))
}
func TestFormatImportsInvalidComment(t *testing.T) {
// why we shouldn't write comments between imports: it breaks the grouping of imports
// for example:
// "pkg1"
// "pkg2"
// // a comment
// "pkgA"
// "pkgB"
// the comment splits the packages into two groups, pkg1/2 are sorted separately, pkgA/B are sorted separately
// we don't want such code, so the code should be:
// "pkg1"
// "pkg2"
// "pkgA" // a comment
// "pkgB"
_, err := formatGoImports([]byte(`
package test
import (
"image/jpeg"
// for processing gif images
"image/gif"
)
`))
assert.ErrorIs(t, err, errInvalidCommentBetweenImports)
}
+4 -4
View File
@@ -24,8 +24,8 @@ import (
)
const (
gemojiURL = "https://raw.githubusercontent.com/github/gemoji/master/db/emoji.json"
maxUnicodeVersion = 15
gemojiURL = "https://raw.githubusercontent.com/rhysd/gemoji/537ff2d7e0496e9964824f7f73ec7ece88c9765a/db/emoji.json"
maxUnicodeVersion = 16
)
var flagOut = flag.String("o", "modules/emoji/emoji_data.go", "out")
@@ -149,8 +149,8 @@ func generate() ([]byte, error) {
}
// write a JSON file to use with tribute (write before adding skin tones since we can't support them there yet)
file, _ := json.Marshal(data)
_ = os.WriteFile("assets/emoji.json", file, 0o644)
file, _ := json.MarshalIndent(data, "", " ")
_ = os.WriteFile("assets/emoji.json", append(file, '\n'), 0o644)
// Add skin tones to emoji that support it
var (
@@ -8,99 +8,220 @@ package main
import (
"encoding/json"
"fmt"
"io/fs"
"os"
"path"
"os/exec"
"path/filepath"
"regexp"
"slices"
"sort"
"strings"
"code.gitea.io/gitea/modules/container"
)
// regexp is based on go-license, excluding README and NOTICE
// https://github.com/google/go-licenses/blob/master/licenses/find.go
// also defined in vite.config.ts
var licenseRe = regexp.MustCompile(`^(?i)((UN)?LICEN(S|C)E|COPYING).*$`)
// primaryLicenseRe matches exact primary license filenames without suffixes.
// When a directory has both primary and variant files (e.g. LICENSE and
// LICENSE.docs), only the primary files are kept.
var primaryLicenseRe = regexp.MustCompile(`^(?i)(LICEN[SC]E|COPYING)$`)
// ignoredNames are LicenseEntry.Name values to exclude from the output.
var ignoredNames = map[string]bool{
"code.gitea.io/gitea": true,
"code.gitea.io/gitea/options/license": true,
}
var excludedExt = map[string]bool{
".gitignore": true,
".go": true,
".mod": true,
".sum": true,
".toml": true,
".yaml": true,
".yml": true,
}
type ModuleInfo struct {
Path string
Dir string
PkgDirs []string // directories of packages imported from this module
}
type LicenseEntry struct {
Name string `json:"name"`
Path string `json:"path"`
LicenseText string `json:"licenseText"`
}
func main() {
if len(os.Args) != 3 {
fmt.Println("usage: go run generate-go-licenses.go <base-dir> <out-json-file>")
// getModules returns all dependency modules with their local directory paths
// and the package directories used from each module.
func getModules(goCmd string) []ModuleInfo {
cmd := exec.Command(goCmd, "list", "-deps", "-f",
"{{if .Module}}{{.Module.Path}}\t{{.Module.Dir}}\t{{.Dir}}{{end}}", "./...")
cmd.Stderr = os.Stderr
// Use GOOS=linux with CGO to ensure we capture all platform-specific
// dependencies, matching the CI environment.
cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=1")
output, err := cmd.Output()
if err != nil {
fmt.Fprintf(os.Stderr, "failed to run 'go list -deps': %v\n", err)
os.Exit(1)
}
base, out := os.Args[1], os.Args[2]
// Add ext for excluded files because license_test.go will be included for some reason.
// And there are more files that should be excluded, check with:
//
// go run github.com/google/go-licenses@v1.6.0 save . --force --save_path=.go-licenses 2>/dev/null
// find .go-licenses -type f | while read FILE; do echo "${$(basename $FILE)##*.}"; done | sort -u
// AUTHORS
// COPYING
// LICENSE
// Makefile
// NOTICE
// gitignore
// go
// md
// mod
// sum
// toml
// txt
// yml
//
// It could be removed once we have a better regex.
excludedExt := container.SetOf(".gitignore", ".go", ".mod", ".sum", ".toml", ".yml")
var paths []string
err := filepath.WalkDir(base, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return err
var modules []ModuleInfo
seen := make(map[string]int) // module path -> index in modules
for _, line := range strings.Split(string(output), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if entry.IsDir() || !licenseRe.MatchString(entry.Name()) || excludedExt.Contains(filepath.Ext(entry.Name())) {
return nil
parts := strings.Split(line, "\t")
if len(parts) != 3 {
continue
}
paths = append(paths, path)
return nil
})
if err != nil {
panic(err)
modPath, modDir, pkgDir := parts[0], parts[1], parts[2]
if idx, ok := seen[modPath]; ok {
modules[idx].PkgDirs = append(modules[idx].PkgDirs, pkgDir)
} else {
seen[modPath] = len(modules)
modules = append(modules, ModuleInfo{
Path: modPath,
Dir: modDir,
PkgDirs: []string{pkgDir},
})
}
}
return modules
}
// findLicenseFiles scans a module's root directory and its used package
// directories for license files. It also walks up from each package directory
// to the module root, scanning intermediate directories. Subdirectory licenses
// are only included if their text differs from the root license(s).
func findLicenseFiles(mod ModuleInfo) []LicenseEntry {
var entries []LicenseEntry
seenTexts := make(map[string]bool)
// First, collect root-level license files.
entries = append(entries, scanDirForLicenses(mod.Dir, mod.Path, "")...)
for _, e := range entries {
seenTexts[e.LicenseText] = true
}
sort.Strings(paths)
// Then check each package directory and all intermediate parent directories
// up to the module root for license files with unique text.
seenDirs := map[string]bool{mod.Dir: true}
for _, pkgDir := range mod.PkgDirs {
for dir := pkgDir; dir != mod.Dir && strings.HasPrefix(dir, mod.Dir); dir = filepath.Dir(dir) {
if seenDirs[dir] {
continue
}
seenDirs[dir] = true
for _, e := range scanDirForLicenses(dir, mod.Path, mod.Dir) {
if !seenTexts[e.LicenseText] {
seenTexts[e.LicenseText] = true
entries = append(entries, e)
}
}
}
}
return entries
}
// scanDirForLicenses reads a single directory for license files and returns entries.
// If moduleRoot is non-empty, paths are made relative to it.
func scanDirForLicenses(dir, modulePath, moduleRoot string) []LicenseEntry {
dirEntries, err := os.ReadDir(dir)
if err != nil {
return nil
}
var entries []LicenseEntry
for _, filePath := range paths {
licenseText, err := os.ReadFile(filePath)
if err != nil {
panic(err)
for _, entry := range dirEntries {
if entry.IsDir() {
continue
}
pkgPath := filepath.ToSlash(filePath)
pkgPath = strings.TrimPrefix(pkgPath, base+"/")
pkgName := path.Dir(pkgPath)
// There might be a bug somewhere in go-licenses that sometimes interprets the
// root package as "." and sometimes as "code.gitea.io/gitea". Workaround by
// removing both of them for the sake of stable output.
if pkgName == "." || pkgName == "code.gitea.io/gitea" {
name := entry.Name()
if !licenseRe.MatchString(name) {
continue
}
if excludedExt[strings.ToLower(filepath.Ext(name))] {
continue
}
content, err := os.ReadFile(filepath.Join(dir, name))
if err != nil {
continue
}
entryName := modulePath
entryPath := modulePath + "/" + name
if moduleRoot != "" {
rel, _ := filepath.Rel(moduleRoot, dir)
if rel != "." {
relSlash := filepath.ToSlash(rel)
entryName = modulePath + "/" + relSlash
entryPath = modulePath + "/" + relSlash + "/" + name
}
}
entries = append(entries, LicenseEntry{
Name: pkgName,
Path: pkgPath,
LicenseText: string(licenseText),
Name: entryName,
Path: entryPath,
LicenseText: string(content),
})
}
// When multiple license files exist, prefer primary files (e.g. LICENSE)
// over variants with suffixes (e.g. LICENSE.docs, LICENSE-2.0.txt).
// If no primary file exists, keep only the first variant.
if len(entries) > 1 {
var primary []LicenseEntry
for _, e := range entries {
fileName := e.Path[strings.LastIndex(e.Path, "/")+1:]
if primaryLicenseRe.MatchString(fileName) {
primary = append(primary, e)
}
}
if len(primary) > 0 {
return primary
}
return entries[:1]
}
return entries
}
func main() {
if len(os.Args) != 2 {
fmt.Println("usage: go run generate-go-licenses.go <out-json-file>")
os.Exit(1)
}
out := os.Args[1]
goCmd := "go"
if env := os.Getenv("GO"); env != "" {
goCmd = env
}
modules := getModules(goCmd)
var entries []LicenseEntry
for _, mod := range modules {
entries = append(entries, findLicenseFiles(mod)...)
}
entries = slices.DeleteFunc(entries, func(e LicenseEntry) bool {
return ignoredNames[e.Name]
})
sort.Slice(entries, func(i, j int) bool {
return entries[i].Path < entries[j].Path
})
jsonBytes, err := json.MarshalIndent(entries, "", " ")
if err != nil {
panic(err)
-118
View File
@@ -1,118 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Copyright (c) 2015, Wade Simmons
// SPDX-License-Identifier: MIT
// gocovmerge takes the results from multiple `go test -coverprofile` runs and
// merges them into one profile
//go:build ignore
package main
import (
"flag"
"fmt"
"io"
"log"
"os"
"sort"
"golang.org/x/tools/cover"
)
func mergeProfiles(p, merge *cover.Profile) {
if p.Mode != merge.Mode {
log.Fatalf("cannot merge profiles with different modes")
}
// Since the blocks are sorted, we can keep track of where the last block
// was inserted and only look at the blocks after that as targets for merge
startIndex := 0
for _, b := range merge.Blocks {
startIndex = mergeProfileBlock(p, b, startIndex)
}
}
func mergeProfileBlock(p *cover.Profile, pb cover.ProfileBlock, startIndex int) int {
sortFunc := func(i int) bool {
pi := p.Blocks[i+startIndex]
return pi.StartLine >= pb.StartLine && (pi.StartLine != pb.StartLine || pi.StartCol >= pb.StartCol)
}
i := 0
if sortFunc(i) != true {
i = sort.Search(len(p.Blocks)-startIndex, sortFunc)
}
i += startIndex
if i < len(p.Blocks) && p.Blocks[i].StartLine == pb.StartLine && p.Blocks[i].StartCol == pb.StartCol {
if p.Blocks[i].EndLine != pb.EndLine || p.Blocks[i].EndCol != pb.EndCol {
log.Fatalf("OVERLAP MERGE: %v %v %v", p.FileName, p.Blocks[i], pb)
}
switch p.Mode {
case "set":
p.Blocks[i].Count |= pb.Count
case "count", "atomic":
p.Blocks[i].Count += pb.Count
default:
log.Fatalf("unsupported covermode: '%s'", p.Mode)
}
} else {
if i > 0 {
pa := p.Blocks[i-1]
if pa.EndLine >= pb.EndLine && (pa.EndLine != pb.EndLine || pa.EndCol > pb.EndCol) {
log.Fatalf("OVERLAP BEFORE: %v %v %v", p.FileName, pa, pb)
}
}
if i < len(p.Blocks)-1 {
pa := p.Blocks[i+1]
if pa.StartLine <= pb.StartLine && (pa.StartLine != pb.StartLine || pa.StartCol < pb.StartCol) {
log.Fatalf("OVERLAP AFTER: %v %v %v", p.FileName, pa, pb)
}
}
p.Blocks = append(p.Blocks, cover.ProfileBlock{})
copy(p.Blocks[i+1:], p.Blocks[i:])
p.Blocks[i] = pb
}
return i + 1
}
func addProfile(profiles []*cover.Profile, p *cover.Profile) []*cover.Profile {
i := sort.Search(len(profiles), func(i int) bool { return profiles[i].FileName >= p.FileName })
if i < len(profiles) && profiles[i].FileName == p.FileName {
mergeProfiles(profiles[i], p)
} else {
profiles = append(profiles, nil)
copy(profiles[i+1:], profiles[i:])
profiles[i] = p
}
return profiles
}
func dumpProfiles(profiles []*cover.Profile, out io.Writer) {
if len(profiles) == 0 {
return
}
fmt.Fprintf(out, "mode: %s\n", profiles[0].Mode)
for _, p := range profiles {
for _, b := range p.Blocks {
fmt.Fprintf(out, "%s:%d.%d,%d.%d %d %d\n", p.FileName, b.StartLine, b.StartCol, b.EndLine, b.EndCol, b.NumStmt, b.Count)
}
}
}
func main() {
flag.Parse()
var merged []*cover.Profile
for _, file := range flag.Args() {
profiles, err := cover.ParseProfiles(file)
if err != nil {
log.Fatalf("failed to parse profile '%s': %v", file, err)
}
for _, p := range profiles {
merged = addProfile(merged, p)
}
}
dumpProfiles(merged, os.Stdout)
}
-20
View File
@@ -1,20 +0,0 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build ignore
package main
import (
"fmt"
"io"
"os"
)
func main() {
_, err := io.Copy(os.Stdout, os.Stdin)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v", err)
os.Exit(1)
}
}
+7 -37
View File
@@ -1,52 +1,22 @@
#!/bin/sh
# this script runs in alpine image which only has `sh` shell
set +e
if sed --version 2>/dev/null | grep -q GNU; then
SED_INPLACE="sed -i"
else
SED_INPLACE="sed -i ''"
fi
set -e
if [ ! -f ./options/locale/locale_en-US.ini ]; then
if [ ! -f ./options/locale/locale_en-US.json ]; then
echo "please run this script in the root directory of the project"
exit 1
fi
mv ./options/locale/locale_en-US.ini ./options/
# the "ini" library for locale has many quirks, its behavior is different from Crowdin.
# see i18n_test.go for more details
# this script helps to unquote the Crowdin outputs for the quirky ini library
# * find all `key="...\"..."` lines
# * remove the leading quote
# * remove the trailing quote
# * unescape the quotes
# * eg: key="...\"..." => key=..."...
$SED_INPLACE -r -e '/^[-.A-Za-z0-9_]+[ ]*=[ ]*".*"$/ {
s/^([-.A-Za-z0-9_]+)[ ]*=[ ]*"/\1=/
s/"$//
s/\\"/"/g
}' ./options/locale/*.ini
# * if the escaped line is incomplete like `key="...` or `key=..."`, quote it with backticks
# * eg: key="... => key=`"...`
# * eg: key=..." => key=`..."`
$SED_INPLACE -r -e 's/^([-.A-Za-z0-9_]+)[ ]*=[ ]*(".*[^"])$/\1=`\2`/' ./options/locale/*.ini
$SED_INPLACE -r -e 's/^([-.A-Za-z0-9_]+)[ ]*=[ ]*([^"].*")$/\1=`\2`/' ./options/locale/*.ini
mv ./options/locale/locale_en-US.json ./options/
# Remove translation under 25% of en_us
baselines=$(wc -l "./options/locale_en-US.ini" | cut -d" " -f1)
baselines=$(cat "./options/locale_en-US.json" | wc -l)
baselines=$((baselines / 4))
for filename in ./options/locale/*.ini; do
lines=$(wc -l "$filename" | cut -d" " -f1)
if [ $lines -lt $baselines ]; then
for filename in ./options/locale/*.json; do
lines=$(cat "$filename" | wc -l)
if [ "$lines" -lt "$baselines" ]; then
echo "Removing $filename: $lines/$baselines"
rm "$filename"
fi
done
mv ./options/locale_en-US.ini ./options/locale/
mv ./options/locale_en-US.json ./options/locale/