feat: update gitea vendor
runner nix smoke / nix label and flake smoke (push) Failing after 1m28s

This commit is contained in:
2026-09-26 21:18:24 +00:00
parent c439c1b948
commit d9b2a4e787
3538 changed files with 116131 additions and 44340 deletions
@@ -13,14 +13,15 @@ import (
"strconv"
"strings"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
)
var (
ErrMissingPKGINFOFile = util.NewInvalidArgumentErrorf("PKGINFO file is missing")
ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid")
ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid")
ErrMissingPKGINFOFile = util.NewInvalidArgumentErrorf("PKGINFO file is missing")
ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid")
ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid")
ErrPackageInfoTooLarge = util.NewInvalidArgumentErrorf("PKGINFO contains too many entries")
)
const (
@@ -36,6 +37,8 @@ const (
RepositoryVersion = "_repository"
NoArch = "noarch"
maxPackageInfoEntries = 1024
)
// https://wiki.alpinelinux.org/wiki/Apk_spec
@@ -185,10 +188,16 @@ func ParsePackageInfo(r io.Reader) (*Package, error) {
p.FileMetadata.InstallIf = value
case "provides":
if value != "" {
if len(p.FileMetadata.Provides)+len(p.FileMetadata.Dependencies) >= maxPackageInfoEntries {
return nil, ErrPackageInfoTooLarge
}
p.FileMetadata.Provides = append(p.FileMetadata.Provides, value)
}
case "depend":
if value != "" {
if len(p.FileMetadata.Provides)+len(p.FileMetadata.Dependencies) >= maxPackageInfoEntries {
return nil, ErrPackageInfoTooLarge
}
p.FileMetadata.Dependencies = append(p.FileMetadata.Dependencies, value)
}
case "provider_priority":
@@ -8,6 +8,7 @@ import (
"bytes"
"compress/gzip"
"io"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -97,6 +98,15 @@ func TestParsePackage(t *testing.T) {
assert.Equal(t, "Q1SRYURM5+uQDqfHSwTnNIOIuuDVQ=", p.FileMetadata.Checksum)
})
t.Run("TooManyDependencyEntries", func(t *testing.T) {
data := append(createPKGINFOContent(packageName, packageVersion), []byte("\ndepend = item")...)
data = append(data, []byte(strings.Repeat("\ndepend = item", maxPackageInfoEntries))...)
p, err := ParsePackageInfo(bytes.NewReader(data))
assert.Nil(t, p)
assert.ErrorIs(t, err, ErrPackageInfoTooLarge)
})
}
func TestParsePackageInfo(t *testing.T) {
@@ -13,8 +13,9 @@ import (
"strconv"
"strings"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/packages"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
@@ -46,6 +47,11 @@ var (
namePattern = regexp.MustCompile(`\A[a-zA-Z0-9@._+-]+\z`)
// (epoch:pkgver-pkgrel)
versionPattern = regexp.MustCompile(`\A(?:\d:)?[\w.+~]+(?:-[-\w.+~]+)?\z`)
// caps on the accumulated package file list (vars so tests can lower them); far above
// any legitimate package, but low enough to stop metadata amplification
maxFileEntries = 100000
maxFileNameBytes = 16 * 1024 * 1024
)
type Package struct {
@@ -124,7 +130,7 @@ func ParsePackage(r io.Reader) (*Package, error) {
}
var p *Package
files := make([]string, 0, 10)
files := packages.NewBoundedFileList(maxFileEntries, maxFileNameBytes)
tr := tar.NewReader(inner)
for {
@@ -147,7 +153,12 @@ func ParsePackage(r io.Reader) (*Package, error) {
return nil, err
}
} else if !strings.HasPrefix(filename, ".") {
files = append(files, hd.Name)
if strings.ContainsAny(hd.Name, "\n\r") {
continue // a newline would forge extra lines in the pacman index
}
if err := files.Add(hd.Name); err != nil {
return nil, err
}
}
}
@@ -155,7 +166,7 @@ func ParsePackage(r io.Reader) (*Package, error) {
return nil, ErrMissingPKGINFOFile
}
p.FileMetadata.Files = files
p.FileMetadata.Files = files.Files()
p.FileCompressionExtension = compressionType
return p, nil
@@ -10,6 +10,9 @@ import (
"io"
"testing"
"gitea.dev/modules/test"
"gitea.dev/modules/util"
"github.com/klauspost/compress/zstd"
"github.com/stretchr/testify/assert"
"github.com/ulikunitz/xz"
@@ -101,6 +104,7 @@ func TestParsePackage(t *testing.T) {
data := createPackage(c, map[string][]byte{
".PKGINFO": createPKGINFOContent(packageName, packageVersion),
"/test/dummy.txt": {},
"usr/lib/legit\n\n%FILES%\n/etc/cron.d/x": {}, // must not reach the file list
})
p, err := ParsePackage(data)
@@ -167,3 +171,25 @@ func TestParsePackageInfo(t *testing.T) {
assert.ElementsMatch(t, []string{"usr/bin/paket1"}, p.FileMetadata.Backup)
})
}
// TestParsePackageTooManyFiles ensures the accumulated file list is bounded to prevent
// metadata amplification from a package with a huge number of (tiny) file entries.
func TestParsePackageTooManyFiles(t *testing.T) {
defer test.MockVariableValue(&maxFileEntries, 3)()
buf := test.WriteTarCompression(func(w io.Writer) io.WriteCloser { return gzip.NewWriter(w) }, map[string]string{
"file1": "content1",
".PKGINFO": string(createPKGINFOContent(packageName, packageVersion)),
})
_, err := ParsePackage(buf)
assert.NoError(t, err)
buf = test.WriteTarCompression(func(w io.Writer) io.WriteCloser { return gzip.NewWriter(w) }, map[string]string{
"file1": "content1",
"file2": "content2",
"file3": "content3",
"file4": "content4",
".PKGINFO": string(createPKGINFOContent(packageName, packageVersion)),
})
_, err = ParsePackage(buf)
assert.ErrorIs(t, err, util.ErrInvalidArgument)
}
@@ -9,8 +9,8 @@ import (
"io"
"regexp"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/json"
"gitea.dev/modules/validation"
"github.com/hashicorp/go-version"
)
@@ -10,9 +10,9 @@ import (
"regexp"
"strings"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/json"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
)
const (
@@ -15,9 +15,9 @@ import (
"regexp"
"strings"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/json"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"github.com/hashicorp/go-version"
)
@@ -12,7 +12,7 @@ import (
"strings"
"testing"
"code.gitea.io/gitea/modules/json"
"gitea.dev/modules/json"
"github.com/dsnet/compress/bzip2"
"github.com/stretchr/testify/assert"
@@ -8,7 +8,7 @@ import (
"io"
"strings"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/util"
)
// Conaninfo represents infos of a Conan package
@@ -8,8 +8,8 @@ import (
"regexp"
"strings"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/log"
"gitea.dev/modules/util"
)
const (
@@ -10,10 +10,10 @@ import (
"io"
"strings"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"code.gitea.io/gitea/modules/zstd"
"gitea.dev/modules/json"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"gitea.dev/modules/zstd"
)
var (
@@ -10,7 +10,7 @@ import (
"io"
"testing"
"code.gitea.io/gitea/modules/zstd"
"gitea.dev/modules/zstd"
"github.com/dsnet/compress/bzip2"
"github.com/stretchr/testify/assert"
@@ -8,9 +8,9 @@ import (
"io"
"strings"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/packages/container/helm"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/json"
"gitea.dev/modules/packages/container/helm"
"gitea.dev/modules/validation"
oci "github.com/opencontainers/image-spec/specs-go/v1"
)
@@ -7,7 +7,7 @@ import (
"strings"
"testing"
"code.gitea.io/gitea/modules/packages/container/helm"
"gitea.dev/modules/packages/container/helm"
oci "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/assert"
@@ -9,9 +9,9 @@ import (
"path"
"strings"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/setting"
"gitea.dev/modules/storage"
"gitea.dev/modules/util"
)
// BlobHash256Key is the key to address a blob content
@@ -13,7 +13,7 @@ import (
"regexp"
"strings"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/util"
)
const (
@@ -11,10 +11,11 @@ import (
"net/mail"
"regexp"
"strings"
"sync"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"code.gitea.io/gitea/modules/zstd"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"gitea.dev/modules/zstd"
"github.com/blakesmith/ar"
"github.com/ulikunitz/xz"
@@ -36,18 +37,36 @@ const (
controlTar = "control.tar"
)
var (
ErrMissingControlFile = util.NewInvalidArgumentErrorf("control file is missing")
ErrUnsupportedCompression = util.NewInvalidArgumentErrorf("unsupported compression algorithm")
ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid")
ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid")
ErrInvalidArchitecture = util.NewInvalidArgumentErrorf("package architecture is invalid")
var GlobalVars = sync.OnceValue(func() (ret struct {
ErrMissingControlFile error
ErrUnsupportedCompression error
ErrInvalidName error
ErrInvalidVersion error
ErrInvalidArchitecture error
namePattern *regexp.Regexp
versionPattern *regexp.Regexp
symbolPattern *regexp.Regexp
},
) {
ret.ErrMissingControlFile = util.NewInvalidArgumentErrorf("control file is missing")
ret.ErrUnsupportedCompression = util.NewInvalidArgumentErrorf("unsupported compression algorithm")
ret.ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid")
ret.ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid")
ret.ErrInvalidArchitecture = util.NewInvalidArgumentErrorf("package architecture is invalid")
// https://www.debian.org/doc/debian-policy/ch-controlfields.html#source
namePattern = regexp.MustCompile(`\A[a-z0-9][a-z0-9+-.]+\z`)
ret.namePattern = regexp.MustCompile(`\A[a-z0-9][a-z0-9+-.]+\z`)
// https://www.debian.org/doc/debian-policy/ch-controlfields.html#version
versionPattern = regexp.MustCompile(`\A(?:(0|[1-9][0-9]*):)?[a-zA-Z0-9.+~]+(?:-[a-zA-Z0-9.+-~]+)?\z`)
)
ret.versionPattern = regexp.MustCompile(`\A(?:(0|[1-9][0-9]*):)?[a-zA-Z0-9.+~]+(?:-[a-zA-Z0-9.+-~]+)?\z`)
// distribution and component are taken from the request path and written
// verbatim into the generated line-based Release and Packages indices (and
// into the pool/<distribution>/<component> paths referenced from them), so
// they must be restricted to a character set that cannot break that format.
ret.symbolPattern = regexp.MustCompile(`\A[a-zA-Z0-9][a-zA-Z0-9.~+_-]*\z`)
return ret
})
type Package struct {
Name string
@@ -64,6 +83,10 @@ type Metadata struct {
Dependencies []string `json:"dependencies,omitempty"`
}
func IsValidDistributionOrComponent(s string) bool {
return GlobalVars().symbolPattern.MatchString(s)
}
// ParsePackage parses the Debian package file
// https://manpages.debian.org/bullseye/dpkg-dev/deb.5.en.html
func ParsePackage(r io.Reader) (*Package, error) {
@@ -109,10 +132,13 @@ func ParsePackage(r io.Reader) (*Package, error) {
inner = zr
default:
return nil, ErrUnsupportedCompression
return nil, GlobalVars().ErrUnsupportedCompression
}
tr := tar.NewReader(inner)
// bound the decompressed control archive: it holds only the small control file
// and maintainer scripts, so a much larger stream is a decompression bomb
const maxControlTarSize = 32 * 1024 * 1024
tr := tar.NewReader(io.LimitReader(inner, maxControlTarSize))
for {
hd, err := tr.Next()
if err == io.EOF {
@@ -133,7 +159,7 @@ func ParsePackage(r io.Reader) (*Package, error) {
}
}
return nil, ErrMissingControlFile
return nil, GlobalVars().ErrMissingControlFile
}
// ParseControlFile parses a Debian control file to retrieve the metadata
@@ -145,20 +171,35 @@ func ParseControlFile(r io.Reader) (*Package, error) {
key := ""
var depends strings.Builder
var control strings.Builder
var description strings.Builder
s := bufio.NewScanner(io.TeeReader(r, &control))
// https://www.debian.org/doc/debian-policy/ch-controlfields.html#syntax-of-control-files
s := bufio.NewScanner(r)
for s.Scan() {
line := s.Text()
trimmed := strings.TrimSpace(line)
if trimmed == "" {
continue
// A binary package control file holds exactly one stanza. Stop at the
// blank line that terminates it, otherwise a crafted control file could
// smuggle additional stanzas (with attacker-chosen Filename/Package
// fields) into the generated repository "Packages" index.
if control.Len() == 0 {
continue
}
break
}
control.WriteString(line)
control.WriteByte('\n')
// a leading space or tab marks a folded continuation line that belongs to the previous field
// (identified by key), not a new "Key: value" pair; only the multi-line fields append here.
// Continuation lines may themselves contain a colon, so they must not be re-split on ":".
if line[0] == ' ' || line[0] == '\t' {
switch key {
case "Description":
p.Metadata.Description += line
description.WriteString(line)
case "Depends":
depends.WriteString(trimmed)
}
@@ -185,7 +226,8 @@ func ParseControlFile(r io.Reader) (*Package, error) {
p.Metadata.Maintainer = a.Name
}
case "Description":
p.Metadata.Description = value
description.Reset()
description.WriteString(value)
case "Depends":
depends.WriteString(value)
case "Homepage":
@@ -199,16 +241,18 @@ func ParseControlFile(r io.Reader) (*Package, error) {
return nil, err
}
if !namePattern.MatchString(p.Name) {
return nil, ErrInvalidName
if !GlobalVars().namePattern.MatchString(p.Name) {
return nil, GlobalVars().ErrInvalidName
}
if !versionPattern.MatchString(p.Version) {
return nil, ErrInvalidVersion
if !GlobalVars().versionPattern.MatchString(p.Version) {
return nil, GlobalVars().ErrInvalidVersion
}
if p.Architecture == "" {
return nil, ErrInvalidArchitecture
return nil, GlobalVars().ErrInvalidArchitecture
}
p.Metadata.Description = description.String()
dependencies := strings.Split(depends.String(), ",")
for i := range dependencies {
dependencies[i] = strings.TrimSpace(dependencies[i])
@@ -10,8 +10,8 @@ import (
"io"
"testing"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/zstd"
"gitea.dev/modules/util"
"gitea.dev/modules/zstd"
"github.com/blakesmith/ar"
"github.com/stretchr/testify/assert"
@@ -49,7 +49,7 @@ func TestParsePackage(t *testing.T) {
p, err := ParsePackage(data)
assert.Nil(t, p)
assert.ErrorIs(t, err, ErrMissingControlFile)
assert.ErrorIs(t, err, GlobalVars().ErrMissingControlFile)
})
t.Run("Compression", func(t *testing.T) {
@@ -58,7 +58,7 @@ func TestParsePackage(t *testing.T) {
p, err := ParsePackage(data)
assert.Nil(t, p)
assert.ErrorIs(t, err, ErrUnsupportedCompression)
assert.ErrorIs(t, err, GlobalVars().ErrUnsupportedCompression)
})
var buf bytes.Buffer
@@ -141,7 +141,7 @@ func TestParseControlFile(t *testing.T) {
for _, name := range []string{"", "-cd"} {
p, err := ParseControlFile(buildContent(name, packageVersion, packageArchitecture))
assert.Nil(t, p)
assert.ErrorIs(t, err, ErrInvalidName)
assert.ErrorIs(t, err, GlobalVars().ErrInvalidName)
}
})
@@ -149,14 +149,14 @@ func TestParseControlFile(t *testing.T) {
for _, version := range []string{"", "1-", ":1.0", "1_0"} {
p, err := ParseControlFile(buildContent(packageName, version, packageArchitecture))
assert.Nil(t, p)
assert.ErrorIs(t, err, ErrInvalidVersion)
assert.ErrorIs(t, err, GlobalVars().ErrInvalidVersion)
}
})
t.Run("InvalidArchitecture", func(t *testing.T) {
p, err := ParseControlFile(buildContent(packageName, packageVersion, ""))
assert.Nil(t, p)
assert.ErrorIs(t, err, ErrInvalidArchitecture)
assert.ErrorIs(t, err, GlobalVars().ErrInvalidArchitecture)
})
t.Run("Valid", func(t *testing.T) {
@@ -184,4 +184,63 @@ func TestParseControlFile(t *testing.T) {
assert.NotNil(t, p)
}
})
t.Run("SingleStanzaOnly", func(t *testing.T) {
// A control file with a trailing stanza must not leak the extra fields into
// p.Control, otherwise buildPackagesIndices would emit a second package entry
// with an attacker-chosen Filename into the repository "Packages" index.
content := bytes.NewBufferString("Package: realpkg\nVersion: 1.0.0\nArchitecture: amd64\nMaintainer: a <a@b.c>\nDescription: real\n\nPackage: openssl\nVersion: 99.0\nArchitecture: amd64\nFilename: pool/main/o/openssl/evil.deb\nDescription: spoofed\n")
p, err := ParseControlFile(content)
assert.NoError(t, err)
assert.NotNil(t, p)
assert.Equal(t, "realpkg", p.Name)
assert.Equal(t, "1.0.0", p.Version)
assert.NotContains(t, p.Control, "openssl")
assert.NotContains(t, p.Control, "evil.deb")
})
}
func TestValidateDistributionOrComponent(t *testing.T) {
bad := []string{
"",
".",
"..",
"-stable",
".hidden",
"a/b",
"a b",
"bookworm\nSigned-By: evil",
"main\nFilename: pool/x",
"a\tb",
}
for _, name := range bad {
assert.False(t, IsValidDistributionOrComponent(name), "bad=%q", name)
}
good := []string{
"stable",
"bookworm",
"bookworm-backports",
"stable-updates",
"main",
"non-free-firmware",
"a",
"1",
}
for _, name := range good {
assert.True(t, IsValidDistributionOrComponent(name), "good=%q", name)
}
}
// TestParseControlFileMultilineDescription verifies a multi-line Description is assembled in order
// (the parser accumulates it in a strings.Builder); it guards the assembled value, not its timing.
func TestParseControlFileMultilineDescription(t *testing.T) {
var buf bytes.Buffer
buf.WriteString("Package: testpkg\nVersion: 1.0\nArchitecture: amd64\nDescription: short summary\n more details\n even more\n")
p, err := ParseControlFile(&buf)
assert.NoError(t, err)
assert.NotNil(t, p)
assert.Equal(t, "short summary more details even more", p.Metadata.Description)
}
@@ -0,0 +1,37 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package packages
import "gitea.dev/modules/util"
// BoundedFileList accumulates file names from a package archive while enforcing caps on the number of
// entries and their total name length, returning an error once either cap would be exceeded.
type BoundedFileList struct {
files []string
nameBytes int
maxFiles int
maxBytes int
}
// NewBoundedFileList creates a BoundedFileList with the given caps; a non-positive cap falls back to the
// corresponding default.
func NewBoundedFileList(maxFiles, maxNameBytes int) *BoundedFileList {
return &BoundedFileList{maxFiles: maxFiles, maxBytes: maxNameBytes}
}
// Add appends name, returning util.ErrInvalidArgument once the entry count or accumulated byte length
// would exceed the configured cap.
func (b *BoundedFileList) Add(name string) error {
if len(b.files) >= b.maxFiles || b.nameBytes+len(name) > b.maxBytes {
return util.NewInvalidArgumentErrorf("package contains too many file entries")
}
b.nameBytes += len(name)
b.files = append(b.files, name)
return nil
}
// Files returns the accumulated file names.
func (b *BoundedFileList) Files() []string {
return b.files
}
@@ -9,7 +9,9 @@ import (
"path"
"strings"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/util"
"golang.org/x/mod/semver"
)
const (
@@ -20,6 +22,7 @@ const (
var (
ErrInvalidStructure = util.NewInvalidArgumentErrorf("package has invalid structure")
ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid")
ErrGoModFileTooLarge = util.NewInvalidArgumentErrorf("go.mod file is too large")
)
@@ -54,6 +57,13 @@ func ParsePackage(r io.ReaderAt, size int64) (*Package, error) {
Name: strings.TrimSuffix(nameAndVersion, "@"+parts[1]),
Version: versionParts[0],
}
// the version is taken verbatim from the zip path and later written
// one per line into the @v/list proxy response, so it has to be a
// valid module version (no newlines or other stray characters)
if !semver.IsValid(p.Version) {
return nil, ErrInvalidVersion
}
}
if len(versionParts) > 1 {
@@ -59,6 +59,16 @@ func TestParsePackage(t *testing.T) {
assert.Equal(t, "module gitea.com/go-gitea/gitea", p.GoMod)
})
t.Run("InvalidVersion", func(t *testing.T) {
data := createArchive(map[string][]byte{
packageName + "@v1.0.0\nv99.0.0/go.mod": []byte("module " + packageName),
})
p, err := ParsePackage(data, int64(data.Len()))
assert.Nil(t, p)
assert.ErrorIs(t, err, ErrInvalidVersion)
})
t.Run("Valid", func(t *testing.T) {
data := createArchive(map[string][]byte{
packageName + "@" + packageVersion + "/subdir/go.mod": []byte("invalid"),
@@ -6,8 +6,8 @@ package packages
import (
"io"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util/filebuffer"
"gitea.dev/modules/setting"
"gitea.dev/modules/util/filebuffer"
)
// HashedSizeReader provide methods to read, sum hashes and a Size method
@@ -9,7 +9,7 @@ import (
"strings"
"testing"
"code.gitea.io/gitea/modules/setting"
"gitea.dev/modules/setting"
"github.com/stretchr/testify/assert"
)
@@ -9,11 +9,11 @@ import (
"io"
"strings"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"github.com/hashicorp/go-version"
"gopkg.in/yaml.v3"
"go.yaml.in/yaml/v4"
)
var (
@@ -7,8 +7,8 @@ import (
"encoding/xml"
"io"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"golang.org/x/net/html/charset"
)
@@ -7,7 +7,7 @@ import (
"strings"
"testing"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -14,9 +14,9 @@ import (
"strings"
"time"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/json"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"github.com/hashicorp/go-version"
)
@@ -103,7 +103,7 @@ type PackageMetadataVersion struct {
DevDependencies map[string]string `json:"devDependencies,omitempty"`
PeerDependencies map[string]string `json:"peerDependencies,omitempty"`
PeerDependenciesMeta map[string]any `json:"peerDependenciesMeta,omitempty"`
Bin map[string]string `json:"bin,omitempty"`
Bin Bin `json:"bin,omitempty"`
OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"`
Readme string `json:"readme,omitempty"`
Dist PackageDistribution `json:"dist"`
@@ -181,10 +181,54 @@ func (u *User) UnmarshalJSON(data []byte) error {
return nil
}
// Repository https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
// Repository https://docs.npmjs.com/cli/v11/configuring-npm/package-json#repository
type Repository struct {
Type string `json:"type"`
URL string `json:"url"`
Type string `json:"type"`
URL string `json:"url"`
Directory string `json:"directory,omitempty"`
}
// UnmarshalJSON is needed because the repository field can be a string or an object.
func (r *Repository) UnmarshalJSON(data []byte) error {
switch data[0] {
case '"':
var value string
if err := json.Unmarshal(data, &value); err != nil {
return err
}
r.URL = value
case '{':
type repositoryAlias Repository // avoid recursion into this method
var value repositoryAlias
if err := json.Unmarshal(data, &value); err != nil {
return err
}
*r = Repository(value)
}
return nil
}
// Bin maps command names to executable files. npm also allows a single string,
// in which case the command is named after the package (resolved in ParsePackage).
type Bin map[string]string
// UnmarshalJSON is needed because the bin field can be a string or an object.
func (b *Bin) UnmarshalJSON(data []byte) error {
switch data[0] {
case '"':
var value string
if err := json.Unmarshal(data, &value); err != nil {
return err
}
*b = Bin{"": value}
case '{':
var value map[string]string
if err := json.Unmarshal(data, &value); err != nil {
return err
}
*b = value
}
return nil
}
// PackageAttachment https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package
@@ -228,6 +272,11 @@ func ParsePackage(r io.Reader) (*Package, error) {
meta.Homepage = ""
}
// A string "bin" means a single executable named after the package.
if cmd, ok := meta.Bin[""]; ok && len(meta.Bin) == 1 {
meta.Bin = Bin{name: cmd}
}
p := &Package{
Name: meta.Name,
Version: v.String(),
@@ -10,7 +10,7 @@ import (
"strings"
"testing"
"code.gitea.io/gitea/modules/json"
"gitea.dev/modules/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -28,8 +28,9 @@ func TestParsePackage(t *testing.T) {
data := "H4sIAAAAAAAA/ytITM5OTE/VL4DQelnF+XkMVAYGBgZmJiYK2MRBwNDcSIHB2NTMwNDQzMwAqA7IMDUxA9LUdgg2UFpcklgEdAql5kD8ogCnhwio5lJQUMpLzE1VslJQcihOzi9I1S9JLS7RhSYIJR2QgrLUouLM/DyQGkM9Az1D3YIiqExKanFyUWZBCVQ2BKhVwQVJDKwosbQkI78IJO/tZ+LsbRykxFXLNdA+HwWjYBSMgpENACgAbtAACAAA"
integrity := "sha512-yA4FJsVhetynGfOC1jFf79BuS+jrHbm0fhh+aHzCQkOaOBXKf9oBnC4a6DnLLnEsHQDRLYd00cwj8sCXpC+wIg=="
repository := Repository{
Type: "gitea",
URL: "http://localhost:3000/gitea/test.git",
Type: "gitea",
URL: "http://localhost:3000/gitea/test.git",
Directory: "packages/test-package",
}
t.Run("InvalidUpload", func(t *testing.T) {
@@ -298,6 +299,7 @@ func TestParsePackage(t *testing.T) {
assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"])
assert.Equal(t, repository.Type, p.Metadata.Repository.Type)
assert.Equal(t, repository.URL, p.Metadata.Repository.URL)
assert.Equal(t, repository.Directory, p.Metadata.Repository.Directory)
})
t.Run("ValidLicenseMap", func(t *testing.T) {
@@ -324,4 +326,31 @@ func TestParsePackage(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "MIT", string(p.Metadata.License))
})
t.Run("ValidRepositoryAndBinAsString", func(t *testing.T) {
// npm allows "repository" and "bin" to be plain strings, not only objects.
packageJSON := `{
"versions": {
"0.1.1": {
"name": "dev-null",
"version": "0.1.1",
"bin": "./cli.js",
"repository": "https://gitea.io/gitea/test.git",
"dist": {
"integrity": "sha256-"
}
}
},
"_attachments": {
"foo": {
"data": "AAAA"
}
}
}`
p, err := ParsePackage(strings.NewReader(packageJSON))
require.NoError(t, err)
require.Equal(t, "https://gitea.io/gitea/test.git", p.Metadata.Repository.URL)
// a string bin is named after the package
require.Equal(t, "./cli.js", p.Metadata.Bin["dev-null"])
})
}
@@ -13,8 +13,8 @@ import (
"regexp"
"strings"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"github.com/hashicorp/go-version"
)
@@ -13,8 +13,8 @@ import (
"path/filepath"
"strings"
"code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/packages"
"gitea.dev/modules/util"
)
var (
@@ -9,7 +9,7 @@ import (
"encoding/base64"
"testing"
"code.gitea.io/gitea/modules/setting"
"gitea.dev/modules/setting"
"github.com/stretchr/testify/assert"
)
@@ -10,11 +10,11 @@ import (
"regexp"
"strings"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"github.com/hashicorp/go-version"
"gopkg.in/yaml.v3"
"go.yaml.in/yaml/v4"
)
var (
@@ -8,8 +8,8 @@ import (
"io"
"strings"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/timeutil"
"gitea.dev/modules/validation"
"github.com/sassoftware/go-rpmutils"
)
@@ -9,7 +9,7 @@ import (
"io"
"reflect"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/util"
)
const (
@@ -91,7 +91,7 @@ func (e *MarshalEncoder) marshal(v any) error {
val := reflect.ValueOf(v)
typ := reflect.TypeOf(v)
if typ.Kind() == reflect.Ptr {
if typ.Kind() == reflect.Pointer {
val = val.Elem()
typ = typ.Elem()
}
@@ -8,13 +8,12 @@ import (
"compress/gzip"
"io"
"regexp"
"strings"
"sync"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"gopkg.in/yaml.v3"
"go.yaml.in/yaml/v4"
)
var (
@@ -26,8 +25,14 @@ var (
ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid")
)
var versionMatcher = sync.OnceValue(func() *regexp.Regexp {
return regexp.MustCompile(`\A[0-9]+(?:\.[0-9a-zA-Z]+)*(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z`)
var globalVars = sync.OnceValue(func() (ret struct {
nameMatcher, versionMatcher *regexp.Regexp
},
) {
// https://github.com/rubygems/rubygems/blob/master/lib/rubygems/specification.rb (VALID_NAME_PATTERN)
ret.nameMatcher = regexp.MustCompile(`\A[\w.-]+\z`)
ret.versionMatcher = regexp.MustCompile(`\A[0-9]+(?:\.[0-9a-zA-Z]+)*(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z`)
return ret
})
// Package represents a RubyGems package
@@ -175,11 +180,11 @@ func parseMetadataFile(r io.Reader) (*Package, error) {
return nil, err
}
if len(spec.Name) == 0 || strings.Contains(spec.Name, "/") {
if !globalVars().nameMatcher.MatchString(spec.Name) {
return nil, ErrInvalidName
}
if !versionMatcher().MatchString(spec.Version.Version) {
if !globalVars().versionMatcher.MatchString(spec.Version.Version) {
return nil, ErrInvalidVersion
}
@@ -6,7 +6,7 @@ package rubygems
import (
"testing"
"code.gitea.io/gitea/modules/test"
"gitea.dev/modules/test"
"github.com/stretchr/testify/assert"
)
@@ -32,6 +32,17 @@ version:
assert.NoError(t, err)
assert.NotNil(t, rp)
})
t.Run("InvalidName", func(t *testing.T) {
// a name carrying a newline would be re-emitted verbatim into the
// line-based compact index, letting an upload forge extra entries
for _, quotedName := range []string{`"evil\n1.0.0"`, `"a b"`, `"a/b"`, `""`} {
content := test.CompressGzip("name: " + quotedName + "\nversion:\n version: 1\n")
rp, err := parseMetadataFile(content)
assert.ErrorIs(t, err, ErrInvalidName, "name %s should be rejected", quotedName)
assert.Nil(t, rp)
}
})
}
func TestParseMetadataFile(t *testing.T) {
@@ -11,9 +11,9 @@ import (
"regexp"
"strings"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/json"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"github.com/hashicorp/go-version"
)
@@ -21,6 +21,7 @@ import (
var (
ErrMissingManifestFile = util.NewInvalidArgumentErrorf("Package.swift file is missing")
ErrManifestFileTooLarge = util.NewInvalidArgumentErrorf("Package.swift file is too large")
ErrManifestFilesTooLarge = util.NewInvalidArgumentErrorf("Package.swift files are too large")
ErrInvalidManifestVersion = util.NewInvalidArgumentErrorf("manifest version is invalid")
manifestPattern = regexp.MustCompile(`\APackage(?:@swift-(\d+(?:\.\d+)?(?:\.\d+)?))?\.swift\z`)
@@ -29,6 +30,8 @@ var (
const (
maxManifestFileSize = 128 * 1024
maxManifestFiles = 64
maxManifestSize = maxManifestFiles * maxManifestFileSize
PropertyScope = "swift.scope"
PropertyName = "swift.name"
@@ -123,11 +126,36 @@ func ParsePackage(sr io.ReaderAt, size int64, mr io.Reader) (*Package, error) {
},
}
// Nested packages (test fixtures, examples, benchmarks) ship their own manifests, which must not
// replace the package manifest. The package sits at the archive root or in a single top level
// directory, so keep only the shallowest manifest directory, breaking ties by name for stability.
var manifestFiles []*zip.File
manifestDir, manifestDepth := "", 0
for _, file := range zr.File {
manifestMatch := manifestPattern.FindStringSubmatch(path.Base(file.Name))
if len(manifestMatch) == 0 {
if strings.HasSuffix(file.Name, "/") || !manifestPattern.MatchString(path.Base(file.Name)) {
continue
}
dir, depth := path.Dir(file.Name), strings.Count(file.Name, "/")
switch {
case manifestFiles == nil || depth < manifestDepth || (depth == manifestDepth && dir < manifestDir):
manifestDir, manifestDepth, manifestFiles = dir, depth, []*zip.File{file}
case dir == manifestDir:
manifestFiles = append(manifestFiles, file)
}
}
if len(manifestFiles) > maxManifestFiles {
return nil, ErrManifestFilesTooLarge
}
var manifestSize uint64
for _, file := range manifestFiles {
manifestSize += file.UncompressedSize64
}
if manifestSize > maxManifestSize {
return nil, ErrManifestFilesTooLarge
}
for _, file := range manifestFiles {
manifestMatch := manifestPattern.FindStringSubmatch(path.Base(file.Name))
if file.UncompressedSize64 > maxManifestFileSize {
return nil, ErrManifestFileTooLarge
@@ -4,11 +4,13 @@
package swift
import (
"archive/zip"
"bytes"
"fmt"
"strings"
"testing"
"code.gitea.io/gitea/modules/test"
"gitea.dev/modules/test"
"github.com/hashicorp/go-version"
"github.com/stretchr/testify/assert"
@@ -24,6 +26,18 @@ const (
packageLicense = "MIT"
)
// writeOrderedZipArchive writes name/content pairs in the given order, which map based test.WriteZipArchive cannot do
func writeOrderedZipArchive(entries [][2]string) *bytes.Buffer {
buf := &bytes.Buffer{}
zw := zip.NewWriter(buf)
for _, entry := range entries {
w, _ := zw.Create(entry[0])
_, _ = w.Write([]byte(entry[1]))
}
_ = zw.Close()
return buf
}
func TestParsePackage(t *testing.T) {
t.Run("MissingManifestFile", func(t *testing.T) {
data := test.WriteZipArchive(map[string]string{"dummy.txt": ""})
@@ -41,6 +55,19 @@ func TestParsePackage(t *testing.T) {
assert.ErrorIs(t, err, ErrManifestFileTooLarge)
})
t.Run("TooManyManifestFiles", func(t *testing.T) {
entries := make([][2]string, 0, maxManifestFiles+1)
entries = append(entries, [2]string{"Package.swift", "// swift-tools-version:5.7"})
for i := range maxManifestFiles {
entries = append(entries, [2]string{fmt.Sprintf("Package@swift-5.%d.swift", i), "// swift-tools-version:5.7"})
}
data := writeOrderedZipArchive(entries)
p, err := ParsePackage(bytes.NewReader(data.Bytes()), int64(data.Len()), nil)
assert.Nil(t, p)
assert.ErrorIs(t, err, ErrManifestFilesTooLarge)
})
t.Run("WithoutMetadata", func(t *testing.T) {
content1 := "// swift-tools-version:5.7\n//\n// Package.swift"
content2 := "// swift-tools-version:5.6\n//\n// Package@swift-5.6.swift"
@@ -65,6 +92,77 @@ func TestParsePackage(t *testing.T) {
assert.Equal(t, content2, m.Content)
})
t.Run("IgnoresNestedManifests", func(t *testing.T) {
rootManifest := "// swift-tools-version:5.7\n//\n// Package.swift"
rootAltManifest := "// swift-tools-version:5.5\n//\n// Package@swift-5.5.swift"
rootPatchAltManifest := "// swift-tools-version:5.7.1\n//\n// Package@swift-5.7.1.swift"
nestedManifest := "// swift-tools-version:6.3\n//\n// nested fixture package"
data := writeOrderedZipArchive([][2]string{
{"Package.swift", rootManifest},
{"Package@swift-5.5.swift", rootAltManifest},
{"Package@swift-5.7.1.swift", rootPatchAltManifest},
{"Benchmarks/Package.swift", nestedManifest},
{"Utils/Fixtures/PlainPackage/Package.swift", nestedManifest},
})
p, err := ParsePackage(bytes.NewReader(data.Bytes()), int64(data.Len()), nil)
assert.NotNil(t, p)
assert.NoError(t, err)
assert.Len(t, p.Metadata.Manifests, 3)
assert.Equal(t, rootManifest, p.Metadata.Manifests[""].Content)
assert.Equal(t, "5.7", p.Metadata.Manifests[""].ToolsVersion)
assert.Equal(t, rootAltManifest, p.Metadata.Manifests["5.5"].Content)
assert.Equal(t, rootPatchAltManifest, p.Metadata.Manifests["5.7.1"].Content)
})
t.Run("IgnoresNestedManifestsInPrefixedArchive", func(t *testing.T) {
rootManifest := "// swift-tools-version:5.7\n//\n// Package.swift"
// `swift package archive-source` produces archives with a single top level directory
data := writeOrderedZipArchive([][2]string{
{"gitea-1.0.1/Package.swift", rootManifest},
{"gitea-1.0.1/Tests/Fixtures/Package.swift", "// swift-tools-version:6.3"},
})
p, err := ParsePackage(bytes.NewReader(data.Bytes()), int64(data.Len()), nil)
assert.NotNil(t, p)
assert.NoError(t, err)
assert.Len(t, p.Metadata.Manifests, 1)
assert.Equal(t, rootManifest, p.Metadata.Manifests[""].Content)
})
t.Run("AltManifestOnlyInRootDirectory", func(t *testing.T) {
// a deeper Package.swift belongs to a nested package and must not stand in for the missing root manifest
data := test.WriteZipArchive(map[string]string{
"Package@swift-5.5.swift": "// swift-tools-version:5.5",
"Sub/Package.swift": "// swift-tools-version:5.7",
})
p, err := ParsePackage(bytes.NewReader(data.Bytes()), int64(data.Len()), nil)
assert.Nil(t, p)
assert.ErrorIs(t, err, ErrMissingManifestFile)
})
t.Run("ManifestDirectoryTieBreak", func(t *testing.T) {
contentA := "// swift-tools-version:5.7\n// A"
contentB := "// swift-tools-version:5.7\n// B"
// at equal depth the name decides, never the archive order
data := writeOrderedZipArchive([][2]string{
{"a/Package.swift", contentA},
{"b/Package.swift", contentB},
})
p, err := ParsePackage(bytes.NewReader(data.Bytes()), int64(data.Len()), nil)
assert.NotNil(t, p)
assert.NoError(t, err)
assert.Len(t, p.Metadata.Manifests, 1)
assert.Equal(t, contentA, p.Metadata.Manifests[""].Content)
})
t.Run("WithMetadata", func(t *testing.T) {
data := test.WriteZipArchive(map[string]string{
"Package.swift": "// swift-tools-version:5.7\n//\n// Package.swift",
@@ -9,10 +9,10 @@ import (
"io"
"time"
"code.gitea.io/gitea/models/db"
packages_model "code.gitea.io/gitea/models/packages"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/util"
"gitea.dev/models/db"
packages_model "gitea.dev/models/packages"
"gitea.dev/modules/json"
"gitea.dev/modules/util"
"xorm.io/builder"
)
@@ -79,13 +79,12 @@ func RemoveLock(ctx context.Context, packageID int64) error {
}
func updateLock(ctx context.Context, refID int64, value string, cond builder.Cond) error {
pp := packages_model.PackageProperty{RefType: packages_model.PropertyTypePackage, RefID: refID, Name: LockFile}
ok, err := db.GetEngine(ctx).Get(&pp)
pp, ok, err := db.Get[packages_model.PackageProperty](ctx, builder.Eq{"ref_type": packages_model.PropertyTypePackage, "ref_id": refID, "`name`": LockFile})
if err != nil {
return err
}
if ok {
n, err := db.GetEngine(ctx).Where("ref_type=? AND ref_id=? AND name=?", packages_model.PropertyTypePackage, refID, LockFile).And(cond).Cols("value").Update(&packages_model.PackageProperty{Value: value})
n, err := db.GetEngine(ctx).ID(pp.ID).And(cond).Cols("value").Update(&packages_model.PackageProperty{Value: value})
if err != nil {
return err
}
@@ -6,8 +6,8 @@ package terraform
import (
"io"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/json"
"gitea.dev/modules/util"
)
// Note: this is a subset of the Terraform state file format as the full one has two forms.
@@ -9,8 +9,8 @@ import (
"io"
"strings"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/validation"
"gitea.dev/modules/json"
"gitea.dev/modules/validation"
)
const (
@@ -10,7 +10,7 @@ import (
"io"
"testing"
"code.gitea.io/gitea/modules/json"
"gitea.dev/modules/json"
"github.com/stretchr/testify/assert"
)