feat: update gitea vendor
runner nix smoke / nix label and flake smoke (push) Failing after 1m28s

This commit is contained in:
2026-09-26 21:18:24 +00:00
parent c439c1b948
commit d9b2a4e787
3538 changed files with 116131 additions and 44340 deletions
@@ -11,28 +11,27 @@ import (
"strings"
"time"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/analyze"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/indexer"
path_filter "code.gitea.io/gitea/modules/indexer/code/bleve/token/path"
"code.gitea.io/gitea/modules/indexer/code/internal"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/typesniffer"
"code.gitea.io/gitea/modules/util"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/analyze"
"gitea.dev/modules/charset"
"gitea.dev/modules/git"
"gitea.dev/modules/git/gitcmd"
"gitea.dev/modules/gitrepo"
"gitea.dev/modules/indexer"
path_filter "gitea.dev/modules/indexer/code/bleve/token/path"
"gitea.dev/modules/indexer/code/internal"
indexer_internal "gitea.dev/modules/indexer/internal"
inner_bleve "gitea.dev/modules/indexer/internal/bleve"
"gitea.dev/modules/setting"
"gitea.dev/modules/timeutil"
"gitea.dev/modules/typesniffer"
"gitea.dev/modules/util"
"github.com/blevesearch/bleve/v2"
analyzer_custom "github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/letter"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/mapping"
"github.com/blevesearch/bleve/v2/search/query"
@@ -69,9 +68,8 @@ func (d *RepoIndexerData) Type() string {
const (
repoIndexerAnalyzer = "repoIndexerAnalyzer"
filenameIndexerAnalyzer = "filenameIndexerAnalyzer"
filenameIndexerTokenizer = "filenameIndexerTokenizer"
repoIndexerDocType = "repoIndexerDocType"
repoIndexerLatestVersion = 9
repoIndexerLatestVersion = 10
)
// generateBleveIndexMapping generates a bleve index mapping for the repo indexer
@@ -107,8 +105,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{
"type": analyzer_custom.Name,
"char_filters": []string{},
"tokenizer": letter.Name,
"token_filters": []string{unicodeNormalizeName, lowercase.Name},
"tokenizer": codeTokenizerName,
"token_filters": []string{unicodeNormalizeName, codeTokenFilterName, lowercase.Name},
}); err != nil {
return nil, err
}
@@ -0,0 +1,59 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package bleve
import (
"testing"
"time"
"gitea.dev/models/db"
"gitea.dev/modules/indexer/code/internal"
inner_bleve "gitea.dev/modules/indexer/internal/bleve"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBleveIndexerTokenFilter(t *testing.T) {
dir := t.TempDir()
indexer := NewIndexer(dir)
defer indexer.Close()
_, err := indexer.Init(t.Context())
require.NoError(t, err)
batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize)
batch.Index("2", &RepoIndexerData{RepoID: 2, Content: "mDNS.port2=12345", UpdatedAt: time.Now()})
batch.Flush()
testCases := []struct {
keyword string
expectedIDs []int64
}{
{keyword: "12345", expectedIDs: []int64{2}},
{keyword: "DNS", expectedIDs: []int64{}},
{keyword: "mdns", expectedIDs: []int64{2}},
{keyword: "port", expectedIDs: []int64{2}},
{keyword: "port2", expectedIDs: []int64{2}},
}
for _, testCase := range testCases {
t.Run(testCase.keyword, func(t *testing.T) {
_, results, _, err := indexer.Search(t.Context(), &internal.SearchOptions{
Paginator: &db.ListOptions{Page: 1, PageSize: 1},
Keyword: testCase.keyword,
})
require.NoError(t, err)
assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(results))
})
}
}
func searchResultIDs(result []*internal.SearchResult) []int64 {
ids := make([]int64, 0, len(result))
for _, hit := range result {
ids = append(ids, hit.RepoID)
}
return ids
}
@@ -0,0 +1,65 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package bleve
import (
"regexp"
"unicode"
"gitea.dev/modules/util"
"github.com/blevesearch/bleve/v2/analysis"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/character"
"github.com/blevesearch/bleve/v2/registry"
)
const codeTokenizerName = "codeTokenizer"
func codeTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) {
// Old code used "letter" tokenizer which doesn't support CJK.
// Here it still doesn't support CJK, since there is no usable CJK tokenizer at the moment.
return character.NewCharacterTokenizer(func(r rune) bool {
return unicode.IsLetter(r) || unicode.IsNumber(r)
}), nil
}
const codeTokenFilterName = "codeTokenFilter"
type codeTokenFilter struct {
re *regexp.Regexp
}
func (c codeTokenFilter) Filter(stream analysis.TokenStream) (ret analysis.TokenStream) {
// split one token to "letter" parts and "number" parts (to keep the old behavior).
// e.g.: input token="port123", then the output tokens are "port123", "port", "123"
for _, token := range stream {
ret = append(ret, token)
m := c.re.FindAllIndex(token.Term, -1)
if len(m) > 1 {
for _, it := range m {
p1, p2 := it[0], it[1]
t := &analysis.Token{
Start: token.Start + p1,
End: token.Start + p2,
Term: token.Term[p1:p2],
Position: token.Position,
Type: analysis.AlphaNumeric,
}
ret = append(ret, t)
}
}
}
return ret
}
func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
return &codeTokenFilter{
re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"),
}, nil
}
func init() {
util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor))
util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor))
}
@@ -7,13 +7,13 @@ import (
"slices"
"strings"
"gitea.dev/modules/util"
"github.com/blevesearch/bleve/v2/analysis"
"github.com/blevesearch/bleve/v2/registry"
)
const (
Name = "gitea/path"
)
const Name = "gitea/path"
type TokenFilter struct{}
@@ -98,8 +98,5 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke
func init() {
// FIXME: move it to the bleve's init function, but do not call it in global init
err := registry.RegisterTokenFilter(Name, TokenFilterConstructor)
if err != nil {
panic(err)
}
util.MustNoError(registry.RegisterTokenFilter(Name, TokenFilterConstructor))
}
@@ -10,41 +10,32 @@ import (
"strconv"
"strings"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/analyze"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/indexer/code/internal"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/typesniffer"
"code.gitea.io/gitea/modules/util"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/analyze"
"gitea.dev/modules/charset"
"gitea.dev/modules/git"
"gitea.dev/modules/git/gitcmd"
"gitea.dev/modules/gitrepo"
"gitea.dev/modules/indexer"
"gitea.dev/modules/indexer/code/internal"
es "gitea.dev/modules/indexer/internal/elasticsearch"
"gitea.dev/modules/json"
"gitea.dev/modules/log"
"gitea.dev/modules/setting"
"gitea.dev/modules/timeutil"
"gitea.dev/modules/typesniffer"
"gitea.dev/modules/util"
"github.com/go-enry/go-enry/v2"
"github.com/olivere/elastic/v7"
)
const (
esRepoIndexerLatestVersion = 3
// multi-match-types, currently only 2 types are used
// Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types
esMultiMatchTypeBestFields = "best_fields"
esMultiMatchTypePhrasePrefix = "phrase_prefix"
)
const esRepoIndexerLatestVersion = 3
var _ internal.Indexer = &Indexer{}
// Indexer implements Indexer interface
type Indexer struct {
inner *inner_elasticsearch.Indexer
indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much
*es.Indexer
}
func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
@@ -53,12 +44,7 @@ func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
// NewIndexer creates a new elasticsearch indexer
func NewIndexer(url, indexerName string) *Indexer {
inner := inner_elasticsearch.NewIndexer(url, indexerName, esRepoIndexerLatestVersion, defaultMapping)
indexer := &Indexer{
inner: inner,
Indexer: inner,
}
return indexer
return &Indexer{Indexer: es.NewIndexer(url, indexerName, esRepoIndexerLatestVersion, defaultMapping)}
}
const (
@@ -138,7 +124,7 @@ const (
}`
)
func (b *Indexer) addUpdate(ctx context.Context, catFileBatch git.CatFileBatch, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) {
func (b *Indexer) addUpdate(ctx context.Context, catFileBatch git.CatFileBatch, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]es.BulkOp, error) {
// Ignore vendored files in code search
if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
return nil, nil
@@ -157,8 +143,9 @@ func (b *Indexer) addUpdate(ctx context.Context, catFileBatch git.CatFileBatch,
}
}
id := internal.FilenameIndexerID(repo.ID, update.Filename)
if size > setting.Indexer.MaxIndexerFileSize {
return []elastic.BulkableRequest{b.addDelete(update.Filename, repo)}, nil
return []es.BulkOp{es.DeleteOp(id)}, nil
}
info, batchReader, err := catFileBatch.QueryContent(update.BlobSha)
@@ -177,33 +164,24 @@ func (b *Indexer) addUpdate(ctx context.Context, catFileBatch git.CatFileBatch,
if _, err = batchReader.Discard(1); err != nil {
return nil, err
}
id := internal.FilenameIndexerID(repo.ID, update.Filename)
return []elastic.BulkableRequest{
elastic.NewBulkIndexRequest().
Index(b.inner.VersionedIndexName()).
Id(id).
Doc(map[string]any{
"repo_id": repo.ID,
"filename": update.Filename,
"content": string(charset.ToUTF8DropErrors(fileContents)),
"commit_id": sha,
"language": analyze.GetCodeLanguage(update.Filename, fileContents),
"updated_at": timeutil.TimeStampNow(),
}),
}, nil
return []es.BulkOp{es.IndexOp(id, map[string]any{
"repo_id": repo.ID,
"filename": update.Filename,
"content": string(charset.ToUTF8DropErrors(fileContents)),
"commit_id": sha,
"language": analyze.GetCodeLanguage(update.Filename, fileContents),
"updated_at": timeutil.TimeStampNow(),
})}, nil
}
func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elastic.BulkableRequest {
id := internal.FilenameIndexerID(repo.ID, filename)
return elastic.NewBulkDeleteRequest().
Index(b.inner.VersionedIndexName()).
Id(id)
func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) es.BulkOp {
return es.DeleteOp(internal.FilenameIndexerID(repo.ID, filename))
}
// Index will save the index data
func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error {
reqs := make([]elastic.BulkableRequest, 0)
ops := make([]es.BulkOp, 0)
if len(changes.Updates) > 0 {
batch, err := gitrepo.NewBatch(ctx, repo)
if err != nil {
@@ -212,29 +190,25 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st
defer batch.Close()
for _, update := range changes.Updates {
updateReqs, err := b.addUpdate(ctx, batch, sha, update, repo)
updateOps, err := b.addUpdate(ctx, batch, sha, update, repo)
if err != nil {
return err
}
if len(updateReqs) > 0 {
reqs = append(reqs, updateReqs...)
if len(updateOps) > 0 {
ops = append(ops, updateOps...)
}
}
}
for _, filename := range changes.RemovedFilenames {
reqs = append(reqs, b.addDelete(filename, repo))
ops = append(ops, b.addDelete(filename, repo))
}
if len(reqs) > 0 {
if len(ops) > 0 {
esBatchSize := 50
for i := 0; i < len(reqs); i += esBatchSize {
_, err := b.inner.Client.Bulk().
Index(b.inner.VersionedIndexName()).
Add(reqs[i:min(i+esBatchSize, len(reqs))]...).
Do(ctx)
if err != nil {
for i := 0; i < len(ops); i += esBatchSize {
if err := b.Bulk(ctx, ops[i:min(i+esBatchSize, len(ops))]); err != nil {
return err
}
}
@@ -246,33 +220,21 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st
func (b *Indexer) Delete(ctx context.Context, repoID int64) error {
if err := b.doDelete(ctx, repoID); err != nil {
// Maybe there is a conflict during the delete operation, so we should retry after a refresh
log.Warn("Deletion of entries of repo %v within index %v was erroneous. Trying to refresh index before trying again", repoID, b.inner.VersionedIndexName(), err)
if err := b.refreshIndex(ctx); err != nil {
log.Warn("Deletion of entries of repo %v within index %v was erroneous: %v. Trying to refresh index before trying again", repoID, b.VersionedIndexName(), err)
if err := b.Refresh(ctx); err != nil {
return err
}
if err := b.doDelete(ctx, repoID); err != nil {
log.Error("Could not delete entries of repo %v within index %v", repoID, b.inner.VersionedIndexName())
log.Error("Could not delete entries of repo %v within index %v", repoID, b.VersionedIndexName())
return err
}
}
return nil
}
func (b *Indexer) refreshIndex(ctx context.Context) error {
if _, err := b.inner.Client.Refresh(b.inner.VersionedIndexName()).Do(ctx); err != nil {
log.Error("Error while trying to refresh index %v", b.inner.VersionedIndexName(), err)
return err
}
return nil
}
// Delete entries by repoId
func (b *Indexer) doDelete(ctx context.Context, repoID int64) error {
_, err := b.inner.Client.DeleteByQuery(b.inner.VersionedIndexName()).
Query(elastic.NewTermsQuery("repo_id", repoID)).
Do(ctx)
return err
return b.DeleteByQuery(ctx, es.TermsQuery("repo_id", repoID))
}
// contentMatchIndexPos find words positions for start and the following end on content. It will
@@ -291,10 +253,10 @@ func contentMatchIndexPos(content, start, end string) (int, int) {
return startIdx, (startIdx + len(start) + endIdx + len(end)) - 9 // remove the length <em></em> since we give Content the original data
}
func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
func convertResult(searchResult *es.SearchResponse, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
hits := make([]*internal.SearchResult, 0, pageSize)
for _, hit := range searchResult.Hits.Hits {
repoID, fileName := internal.ParseIndexerID(hit.Id)
for _, hit := range searchResult.Hits {
repoID, fileName := internal.ParseIndexerID(hit.ID)
res := make(map[string]any)
if err := json.Unmarshal(hit.Source, &res); err != nil {
return 0, nil, nil, err
@@ -333,111 +295,111 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int)
})
}
return searchResult.TotalHits(), hits, extractAggs(searchResult), nil
return searchResult.Total, hits, extractAggs(searchResult), nil
}
func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLanguages {
var searchResultLanguages []*internal.SearchResultLanguages
agg, found := searchResult.Aggregations.Terms("language")
if found {
searchResultLanguages = make([]*internal.SearchResultLanguages, 0, 10)
for _, bucket := range agg.Buckets {
searchResultLanguages = append(searchResultLanguages, &internal.SearchResultLanguages{
Language: bucket.Key.(string),
Color: enry.GetColor(bucket.Key.(string)),
Count: int(bucket.DocCount),
})
func extractAggs(searchResult *es.SearchResponse) []*internal.SearchResultLanguages {
buckets, found := searchResult.Aggregations["language"]
if !found {
return nil
}
searchResultLanguages := make([]*internal.SearchResultLanguages, 0, 10)
for _, bucket := range buckets {
// language is mapped as keyword so the key is always a string; if the
// mapping ever changes, skip rather than emit an empty-language bucket.
key, ok := bucket.Key.(string)
if !ok {
continue
}
searchResultLanguages = append(searchResultLanguages, &internal.SearchResultLanguages{
Language: key,
Color: enry.GetColor(key),
Count: int(bucket.DocCount),
})
}
return searchResultLanguages
}
// Search searches for codes and language stats by given conditions.
func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
var contentQuery elastic.Query
searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue)
contentQuery := es.Query(es.NewMultiMatchQuery(opts.Keyword, "content").Type(es.MultiMatchTypeBestFields).Operator("and"))
if searchMode == indexer.SearchModeExact {
// 1.21 used NewMultiMatchQuery().Type(esMultiMatchTypePhrasePrefix), but later releases changed to NewMatchPhraseQuery
contentQuery = elastic.NewMatchPhraseQuery("content", opts.Keyword)
} else /* words */ {
contentQuery = elastic.NewMultiMatchQuery("content", opts.Keyword).Type(esMultiMatchTypeBestFields).Operator("and")
contentQuery = es.MatchPhraseQuery("content", opts.Keyword)
}
kwQuery := elastic.NewBoolQuery().Should(
kwQuery := es.NewBoolQuery().Should(
contentQuery,
elastic.NewMultiMatchQuery(opts.Keyword, "filename^10").Type(esMultiMatchTypePhrasePrefix),
es.NewMultiMatchQuery(opts.Keyword, "filename^10").Type(es.MultiMatchTypePhrasePrefix),
)
query := elastic.NewBoolQuery()
query = query.Must(kwQuery)
query := es.NewBoolQuery().Must(kwQuery)
if len(opts.RepoIDs) > 0 {
repoStrs := make([]any, 0, len(opts.RepoIDs))
for _, repoID := range opts.RepoIDs {
repoStrs = append(repoStrs, repoID)
}
repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...)
query = query.Must(repoQuery)
query.Must(es.TermsQuery("repo_id", es.ToAnySlice(opts.RepoIDs)...))
}
var (
start, pageSize = opts.GetSkipTake()
kw = "<em>" + opts.Keyword + "</em>"
aggregation = elastic.NewTermsAggregation().Field("language").Size(10).OrderByCountDesc()
)
start, pageSize := opts.GetSkipTake()
kw := "<em>" + opts.Keyword + "</em>"
languageAggs := map[string]any{
"language": map[string]any{
"terms": map[string]any{
"field": "language",
"size": 10,
"order": map[string]any{"_count": "desc"},
},
},
}
// number_of_fragments=0 returns the full highlighted content (no fragmentation).
highlight := map[string]any{
"fields": map[string]any{
"content": map[string]any{},
"filename": map[string]any{},
},
"number_of_fragments": 0,
"type": "fvh",
}
sort := []es.SortField{
{Field: "_score", Desc: true},
{Field: "updated_at", Desc: false},
}
if len(opts.Language) == 0 {
searchResult, err := b.inner.Client.Search().
Index(b.inner.VersionedIndexName()).
Aggregation("language", aggregation).
Query(query).
Highlight(
elastic.NewHighlight().
Field("content").
Field("filename").
NumOfFragments(0). // return all highlighting content on fragments
HighlighterType("fvh"),
).
Sort("_score", false).
Sort("updated_at", true).
From(start).Size(pageSize).
Do(ctx)
resp, err := b.Indexer.Search(ctx, es.SearchRequest{
Query: query,
Sort: sort,
From: start,
Size: pageSize,
TrackTotal: true,
Aggregations: languageAggs,
Highlight: highlight,
})
if err != nil {
return 0, nil, nil, err
}
return convertResult(searchResult, kw, pageSize)
return convertResult(resp, kw, pageSize)
}
langQuery := elastic.NewMatchQuery("language", opts.Language)
countResult, err := b.inner.Client.Search().
Index(b.inner.VersionedIndexName()).
Aggregation("language", aggregation).
Query(query).
Size(0). // We only need stats information
Do(ctx)
countResp, err := b.Indexer.Search(ctx, es.SearchRequest{
Query: query,
Size: 0, // stats only
TrackTotal: true,
Aggregations: languageAggs,
})
if err != nil {
return 0, nil, nil, err
}
query = query.Must(langQuery)
searchResult, err := b.inner.Client.Search().
Index(b.inner.VersionedIndexName()).
Query(query).
Highlight(
elastic.NewHighlight().
Field("content").
Field("filename").
NumOfFragments(0). // return all highlighting content on fragments
HighlighterType("fvh"),
).
Sort("_score", false).
Sort("updated_at", true).
From(start).Size(pageSize).
Do(ctx)
query.Must(es.MatchQuery("language", opts.Language))
resp, err := b.Indexer.Search(ctx, es.SearchRequest{
Query: query,
Sort: sort,
From: start,
Size: pageSize,
TrackTotal: true,
Highlight: highlight,
})
if err != nil {
return 0, nil, nil, err
}
total, hits, _, err := convertResult(searchResult, kw, pageSize)
return total, hits, extractAggs(countResult), err
total, hits, _, err := convertResult(resp, kw, pageSize)
return total, hits, extractAggs(countResp), err
}
@@ -8,13 +8,13 @@ import (
"strconv"
"strings"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/indexer/code/internal"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/git"
"gitea.dev/modules/git/gitcmd"
"gitea.dev/modules/gitrepo"
"gitea.dev/modules/indexer/code/internal"
"gitea.dev/modules/log"
"gitea.dev/modules/setting"
)
func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (string, error) {
@@ -8,10 +8,10 @@ import (
"fmt"
"strings"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/indexer"
code_indexer "code.gitea.io/gitea/modules/indexer/code"
"code.gitea.io/gitea/modules/setting"
"gitea.dev/modules/git"
"gitea.dev/modules/indexer"
code_indexer "gitea.dev/modules/indexer/code"
"gitea.dev/modules/setting"
)
func indexSettingToGitGrepPathspecList() (list []string) {
@@ -6,8 +6,8 @@ package gitgrep
import (
"testing"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/test"
"gitea.dev/modules/setting"
"gitea.dev/modules/test"
"github.com/stretchr/testify/assert"
)
@@ -11,18 +11,18 @@ import (
"sync/atomic"
"time"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/indexer/code/bleve"
"code.gitea.io/gitea/modules/indexer/code/elasticsearch"
"code.gitea.io/gitea/modules/indexer/code/internal"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"gitea.dev/models/db"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/graceful"
"gitea.dev/modules/indexer"
"gitea.dev/modules/indexer/code/bleve"
"gitea.dev/modules/indexer/code/elasticsearch"
"gitea.dev/modules/indexer/code/internal"
"gitea.dev/modules/log"
"gitea.dev/modules/process"
"gitea.dev/modules/queue"
"gitea.dev/modules/setting"
"gitea.dev/modules/util"
)
var (
@@ -8,20 +8,21 @@ import (
"os"
"slices"
"testing"
"time"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/unittest"
indexer_module "code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/indexer/code/bleve"
"code.gitea.io/gitea/modules/indexer/code/elasticsearch"
"code.gitea.io/gitea/modules/indexer/code/internal"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/test"
"code.gitea.io/gitea/modules/util"
"gitea.dev/models/db"
"gitea.dev/models/unittest"
indexer_module "gitea.dev/modules/indexer"
"gitea.dev/modules/indexer/code/bleve"
"gitea.dev/modules/indexer/code/elasticsearch"
"gitea.dev/modules/indexer/code/internal"
"gitea.dev/modules/setting"
"gitea.dev/modules/test"
"gitea.dev/modules/util"
_ "code.gitea.io/gitea/models"
_ "code.gitea.io/gitea/models/actions"
_ "code.gitea.io/gitea/models/activities"
_ "gitea.dev/models"
_ "gitea.dev/models/actions"
_ "gitea.dev/models/activities"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -39,6 +40,16 @@ func TestMain(m *testing.M) {
func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
t.Run(name, func(t *testing.T) {
assert.NoError(t, setupRepositoryIndexes(t.Context(), indexer))
// Wait for the index to catch up: ES/OpenSearch make writes visible
// only after a refresh (default interval: 1s). Bleve is synchronous
// and passes on the first iteration.
require.Eventually(t, func() bool {
total, _, _, err := indexer.Search(t.Context(), &internal.SearchOptions{
Keyword: "Description",
Paginator: &db.ListOptions{Page: 1, PageSize: 1},
})
return err == nil && total > 0
}, 10*time.Second, 100*time.Millisecond, "index did not become searchable")
keywords := []struct {
RepoIDs []int64
@@ -7,10 +7,10 @@ import (
"context"
"errors"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/indexer/internal"
"gitea.dev/models/db"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/indexer"
"gitea.dev/modules/indexer/internal"
)
// Indexer defines an interface to index and search code contents
@@ -3,7 +3,7 @@
package internal
import "code.gitea.io/gitea/modules/timeutil"
import "gitea.dev/modules/timeutil"
type FileUpdate struct {
Filename string
@@ -6,8 +6,8 @@ package internal
import (
"strings"
"code.gitea.io/gitea/modules/indexer/internal"
"code.gitea.io/gitea/modules/log"
"gitea.dev/modules/indexer/internal"
"gitea.dev/modules/log"
)
const filenameMatchNumberOfLines = 7 // Copied from GitHub search
@@ -9,9 +9,9 @@ import (
"html/template"
"strings"
"code.gitea.io/gitea/modules/highlight"
"code.gitea.io/gitea/modules/indexer/code/internal"
"code.gitea.io/gitea/modules/timeutil"
"gitea.dev/modules/highlight"
"gitea.dev/modules/indexer/code/internal"
"gitea.dev/modules/timeutil"
)
// Result a search result to display
@@ -7,8 +7,8 @@ import (
"context"
"errors"
"code.gitea.io/gitea/modules/indexer/internal"
"code.gitea.io/gitea/modules/log"
"gitea.dev/modules/indexer/internal"
"gitea.dev/modules/log"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/mapping"
@@ -4,7 +4,7 @@
package bleve
import (
"code.gitea.io/gitea/modules/optional"
"gitea.dev/modules/optional"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/search/query"
@@ -8,9 +8,9 @@ import (
"os"
"unicode"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/log"
"gitea.dev/modules/setting"
"gitea.dev/modules/util"
"github.com/blevesearch/bleve/v2"
unicode_tokenizer "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
@@ -7,8 +7,8 @@ import (
"fmt"
"testing"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/test"
"gitea.dev/modules/setting"
"gitea.dev/modules/test"
"github.com/stretchr/testify/assert"
)
@@ -6,7 +6,7 @@ package db
import (
"context"
"code.gitea.io/gitea/modules/indexer/internal"
"gitea.dev/modules/indexer/internal"
)
var _ internal.Indexer = &Indexer{}
@@ -4,52 +4,80 @@
package elasticsearch
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"time"
"code.gitea.io/gitea/modules/indexer/internal"
"github.com/olivere/elastic/v7"
"gitea.dev/modules/indexer/internal"
"gitea.dev/modules/json"
)
var _ internal.Indexer = &Indexer{}
// Indexer represents a basic elasticsearch indexer implementation
// Indexer is a narrow wrapper around an Elasticsearch/OpenSearch cluster.
// It targets the REST subset shared by Elasticsearch 7/8/9 and OpenSearch 3.
type Indexer struct {
Client *elastic.Client
client *http.Client
base string // base URL with trailing slash, no userinfo
user string
pass string
url string
indexName string
version int
mapping string
}
func NewIndexer(url, indexName string, version int, mapping string) *Indexer {
// NewIndexer builds an Indexer. The connection is opened by Init.
func NewIndexer(rawURL, indexName string, version int, mapping string) *Indexer {
return &Indexer{
url: url,
base: rawURL,
indexName: indexName,
version: version,
mapping: mapping,
}
}
// Init initializes the indexer
// Init connects and creates the versioned index if missing, returning true if it already existed.
func (i *Indexer) Init(ctx context.Context) (bool, error) {
if i == nil {
return false, errors.New("cannot init nil indexer")
}
if i.Client != nil {
return false, errors.New("indexer is already initialized")
}
client, err := i.initClient()
parsed, err := url.Parse(i.base)
if err != nil {
return false, err
return false, fmt.Errorf("parse elasticsearch url: %w", err)
}
if parsed.User != nil {
i.user = parsed.User.Username()
i.pass, _ = parsed.User.Password()
parsed.User = nil
}
base := parsed.String()
if !strings.HasSuffix(base, "/") {
base += "/"
}
i.base = base
// No client-level Timeout: bulk/_delete_by_query can legitimately run for
// minutes on large repos. Per-request deadlines come from the caller's ctx;
// transport-level timeouts cover stalled connects/handshakes/headers so a
// half-open server cannot wedge the indexer indefinitely.
i.client = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
IdleConnTimeout: 90 * time.Second,
MaxIdleConns: 100,
},
}
i.Client = client
exists, err := i.Client.IndexExists(i.VersionedIndexName()).Do(ctx)
exists, err := i.indexExists(ctx, i.VersionedIndexName())
if err != nil {
return false, err
}
@@ -61,34 +89,321 @@ func (i *Indexer) Init(ctx context.Context) (bool, error) {
return false, err
}
return exists, nil
return false, nil
}
// Ping checks if the indexer is available
// Ping returns an error when the cluster is unusable (status != green/yellow).
func (i *Indexer) Ping(ctx context.Context) error {
if i == nil {
return errors.New("cannot ping nil indexer")
var body struct {
Status string `json:"status"`
}
if i.Client == nil {
return errors.New("indexer is not initialized")
}
resp, err := i.Client.ClusterHealth().Do(ctx)
if err != nil {
if err := i.doJSON(ctx, http.MethodGet, "_cluster/health", nil, &body); err != nil {
return err
}
if resp.Status != "green" && resp.Status != "yellow" {
// It's healthy if the status is green, and it's available if the status is yellow,
// see https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html
return fmt.Errorf("status of elasticsearch cluster is %s", resp.Status)
// Healthy = green; usable = yellow. Red is unusable.
// https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html
if body.Status != "green" && body.Status != "yellow" {
return fmt.Errorf("status of elasticsearch cluster is %s", body.Status)
}
return nil
}
// Close closes the indexer
// Close releases idle HTTP connections held by the client.
func (i *Indexer) Close() {
if i == nil {
if i == nil || i.client == nil {
return
}
i.Client = nil
i.client.CloseIdleConnections()
i.client = nil
}
// Bulk submits index/delete ops. Returns the first item-level failure, if any.
func (i *Indexer) Bulk(ctx context.Context, ops []BulkOp) error {
if len(ops) == 0 {
return nil
}
index := i.VersionedIndexName()
var buf bytes.Buffer
buf.Grow(len(ops) * 256)
for _, op := range ops {
meta := map[string]any{op.action: map[string]any{"_index": index, "_id": op.id}}
if err := writeJSONLine(&buf, meta); err != nil {
return err
}
if op.action == bulkActionIndex {
if err := writeJSONLine(&buf, op.doc); err != nil {
return err
}
}
}
res, err := i.do(ctx, http.MethodPost, urlPath(index, "_bulk"), "application/x-ndjson", bytes.NewReader(buf.Bytes()))
if err != nil {
return err
}
defer drainAndClose(res)
var body struct {
Errors bool `json:"errors"`
Items []map[string]struct {
Status int `json:"status"`
Error json.Value `json:"error"`
} `json:"items"`
}
if err := json.NewDecoder(res.Body).Decode(&body); err != nil {
return err
}
if !body.Errors {
return nil
}
return firstBulkError(body.Items)
}
// firstBulkError returns the first item-level failure in a bulk response.
// Each items entry is a single-key map ({"index": {...}} or {"delete": {...}}).
// Delete-of-missing (404) is idempotent and not reported.
func firstBulkError(items []map[string]struct {
Status int `json:"status"`
Error json.Value `json:"error"`
},
) error {
for _, item := range items {
for action, result := range item {
if action == bulkActionDelete && result.Status == http.StatusNotFound {
continue
}
if result.Status >= 300 {
return fmt.Errorf("bulk %s failed (status %d): %s", action, result.Status, string(result.Error))
}
}
}
return nil
}
// Index writes a single document.
func (i *Indexer) Index(ctx context.Context, id string, doc any) error {
body, err := json.Marshal(doc)
if err != nil {
return err
}
return i.doJSON(ctx, http.MethodPut, urlPath(i.VersionedIndexName(), "_doc", id), bytes.NewReader(body), nil)
}
// Delete removes a single document by id. Missing ids are not an error.
func (i *Indexer) Delete(ctx context.Context, id string) error {
res, err := i.do(ctx, http.MethodDelete, urlPath(i.VersionedIndexName(), "_doc", id), "", nil, http.StatusNotFound)
if err != nil {
return err
}
drainAndClose(res)
return nil
}
// DeleteByQuery removes every document matching the query.
func (i *Indexer) DeleteByQuery(ctx context.Context, query Query) error {
body, err := json.Marshal(map[string]any{"query": query.querySource()})
if err != nil {
return err
}
return i.doJSON(ctx, http.MethodPost, urlPath(i.VersionedIndexName(), "_delete_by_query"), bytes.NewReader(body), nil)
}
// Refresh forces a refresh so recent writes are searchable.
func (i *Indexer) Refresh(ctx context.Context) error {
return i.doJSON(ctx, http.MethodPost, urlPath(i.VersionedIndexName(), "_refresh"), nil, nil)
}
// Search runs a search request and decodes the reply.
func (i *Indexer) Search(ctx context.Context, req SearchRequest) (*SearchResponse, error) {
body := map[string]any{}
if req.Query != nil {
body["query"] = req.Query.querySource()
}
if len(req.Sort) > 0 {
sorts := make([]map[string]any, len(req.Sort))
for idx, s := range req.Sort {
sorts[idx] = s.source()
}
body["sort"] = sorts
}
if req.From > 0 {
body["from"] = req.From
}
body["size"] = req.Size
if len(req.Aggregations) > 0 {
body["aggs"] = req.Aggregations
}
if len(req.Highlight) > 0 {
body["highlight"] = req.Highlight
}
payload, err := json.Marshal(body)
if err != nil {
return nil, err
}
// Default track_total_hits is 10000 (capped count); send it explicitly so
// callers can choose between exact totals (true) and skipping counting (false).
path := urlPath(i.VersionedIndexName(), "_search") + "?track_total_hits=" + strconv.FormatBool(req.TrackTotal)
res, err := i.do(ctx, http.MethodPost, path, "application/json", bytes.NewReader(payload))
if err != nil {
return nil, err
}
defer drainAndClose(res)
return decodeSearchResponse(res.Body)
}
func (i *Indexer) indexExists(ctx context.Context, name string) (bool, error) {
res, err := i.do(ctx, http.MethodHead, urlPath(name), "", nil, http.StatusNotFound)
if err != nil {
return false, err
}
drainAndClose(res)
return res.StatusCode == http.StatusOK, nil
}
func (i *Indexer) createIndex(ctx context.Context) error {
var body struct {
Acknowledged bool `json:"acknowledged"`
}
if err := i.doJSON(ctx, http.MethodPut, urlPath(i.VersionedIndexName()), bytes.NewBufferString(i.mapping), &body); err != nil {
return fmt.Errorf("create index %s: %w", i.VersionedIndexName(), err)
}
if !body.Acknowledged {
return fmt.Errorf("create index %s not acknowledged", i.VersionedIndexName())
}
i.checkOldIndexes(ctx)
return nil
}
// do sends a request and returns the response. Status >= 300 is turned into
// an error unless the status appears in okStatus. The caller closes Body.
func (i *Indexer) do(ctx context.Context, method, path, contentType string, body io.Reader, okStatus ...int) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, method, i.base+path, body)
if err != nil {
return nil, err
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
if i.user != "" || i.pass != "" {
req.SetBasicAuth(i.user, i.pass)
}
res, err := i.client.Do(req)
if err != nil {
return nil, err
}
if res.StatusCode >= 300 && !slices.Contains(okStatus, res.StatusCode) {
msg := readErrBody(res)
res.Body.Close()
return nil, fmt.Errorf("%s %s: %s", method, path, msg)
}
return res, nil
}
// doJSON sends a request with a JSON body and, when out is non-nil, decodes
// the JSON response into it.
func (i *Indexer) doJSON(ctx context.Context, method, path string, body io.Reader, out any) error {
contentType := ""
if body != nil {
contentType = "application/json"
}
res, err := i.do(ctx, method, path, contentType, body)
if err != nil {
return err
}
defer drainAndClose(res)
if out == nil {
return nil
}
return json.NewDecoder(res.Body).Decode(out)
}
// drainAndClose discards any unread response body before closing so the
// underlying TCP connection can be reused for keep-alive.
func drainAndClose(res *http.Response) {
_, _ = io.Copy(io.Discard, res.Body)
res.Body.Close()
}
func writeJSONLine(buf *bytes.Buffer, v any) error {
enc, err := json.Marshal(v)
if err != nil {
return err
}
buf.Write(enc)
buf.WriteByte('\n')
return nil
}
// readErrBody reads up to 4 KiB of an error response and drains the rest so
// the underlying connection can be reused (keep-alive needs Body fully read).
func readErrBody(res *http.Response) string {
const limit = 4 << 10
b, _ := io.ReadAll(io.LimitReader(res.Body, limit))
_, _ = io.Copy(io.Discard, res.Body)
return fmt.Sprintf("status %d: %s", res.StatusCode, bytes.TrimSpace(b))
}
func decodeSearchResponse(r io.Reader) (*SearchResponse, error) {
var raw struct {
Hits struct {
Total struct {
Value int64 `json:"value"`
} `json:"total"`
Hits []struct {
ID string `json:"_id"`
Score float64 `json:"_score"`
Source json.Value `json:"_source"`
Highlight map[string][]string `json:"highlight"`
} `json:"hits"`
} `json:"hits"`
Aggregations map[string]struct {
Buckets []struct {
Key any `json:"key"`
DocCount int64 `json:"doc_count"`
} `json:"buckets"`
} `json:"aggregations"`
}
if err := json.NewDecoder(r).Decode(&raw); err != nil {
return nil, err
}
resp := &SearchResponse{
Total: raw.Hits.Total.Value,
Hits: make([]SearchHit, 0, len(raw.Hits.Hits)),
}
for _, h := range raw.Hits.Hits {
resp.Hits = append(resp.Hits, SearchHit{
ID: h.ID,
Score: h.Score,
Source: h.Source,
Highlight: h.Highlight,
})
}
if len(raw.Aggregations) > 0 {
resp.Aggregations = make(map[string][]AggBucket, len(raw.Aggregations))
for name, agg := range raw.Aggregations {
buckets := make([]AggBucket, len(agg.Buckets))
for idx, b := range agg.Buckets {
buckets[idx] = AggBucket{Key: b.Key, DocCount: b.DocCount}
}
resp.Aggregations[name] = buckets
}
}
return resp, nil
}
// urlPath joins path segments with `/` and percent-escapes each.
func urlPath(segments ...string) string {
var b bytes.Buffer
for idx, s := range segments {
if idx > 0 {
b.WriteByte('/')
}
b.WriteString(url.PathEscape(s))
}
return b.String()
}
@@ -0,0 +1,39 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package elasticsearch
import (
"strings"
"testing"
"gitea.dev/modules/test"
"github.com/stretchr/testify/require"
)
func newRealIndexer(t *testing.T) *Indexer {
t.Helper()
esURL := test.ExternalServiceHTTP(t, "TEST_ELASTICSEARCH_URL", "http://elasticsearch:9200")
indexName := "gitea_test_" + strings.ReplaceAll(strings.ToLower(t.Name()), "/", "_")
ix := NewIndexer(esURL, indexName, 1, `{"mappings":{"properties":{"x":{"type":"keyword"}}}}`)
_, err := ix.Init(t.Context())
require.NoError(t, err)
t.Cleanup(ix.Close)
return ix
}
func TestPing(t *testing.T) {
ix := newRealIndexer(t)
require.NoError(t, ix.Ping(t.Context()))
}
func TestDeleteSwallows404(t *testing.T) {
ix := newRealIndexer(t)
require.NoError(t, ix.Delete(t.Context(), "missing-id"))
}
func TestBulkAcceptsDelete404(t *testing.T) {
ix := newRealIndexer(t)
require.NoError(t, ix.Bulk(t.Context(), []BulkOp{DeleteOp("missing-id")}))
}
@@ -0,0 +1,132 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package elasticsearch
// MultiMatch types used by the call sites. See
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html#multi-match-types
const (
MultiMatchTypeBestFields = "best_fields"
MultiMatchTypePhrasePrefix = "phrase_prefix"
)
// ToAnySlice converts []T to []any for variadic query args like TermsQuery.
func ToAnySlice[T any](s []T) []any {
out := make([]any, len(s))
for idx, v := range s {
out[idx] = v
}
return out
}
// Query is an Elasticsearch query DSL node. It marshals to the JSON
// object expected by the ES query API.
type Query interface {
querySource() map[string]any
}
type rawQuery map[string]any
func (q rawQuery) querySource() map[string]any { return q }
// TermQuery matches documents whose `field` exactly equals `value`.
func TermQuery(field string, value any) Query {
return rawQuery{"term": map[string]any{field: value}}
}
// TermsQuery matches documents whose `field` equals any of `values`.
func TermsQuery(field string, values ...any) Query {
return rawQuery{"terms": map[string]any{field: values}}
}
// MatchQuery is a full-text match on a single field.
func MatchQuery(field string, value any) Query {
return rawQuery{"match": map[string]any{field: value}}
}
// MatchPhraseQuery matches the exact phrase on `field`.
func MatchPhraseQuery(field, value string) Query {
return rawQuery{"match_phrase": map[string]any{field: value}}
}
// MultiMatchQuery is the fluent builder for a multi_match query.
type MultiMatchQuery struct {
query any
fields []string
typ string
operator string
}
// NewMultiMatchQuery creates a multi_match query over the given fields.
func NewMultiMatchQuery(query any, fields ...string) *MultiMatchQuery {
return &MultiMatchQuery{query: query, fields: fields}
}
func (m *MultiMatchQuery) Type(t string) *MultiMatchQuery { m.typ = t; return m }
func (m *MultiMatchQuery) Operator(op string) *MultiMatchQuery { m.operator = op; return m }
func (m *MultiMatchQuery) querySource() map[string]any {
body := map[string]any{"query": m.query}
if len(m.fields) > 0 {
body["fields"] = m.fields
}
if m.typ != "" {
body["type"] = m.typ
}
if m.operator != "" {
body["operator"] = m.operator
}
return map[string]any{"multi_match": body}
}
// RangeQuery is the fluent builder for a range query.
type RangeQuery struct {
field string
body map[string]any
}
func NewRangeQuery(field string) *RangeQuery {
return &RangeQuery{field: field, body: map[string]any{}}
}
func (r *RangeQuery) Gte(v any) *RangeQuery { r.body["gte"] = v; return r }
func (r *RangeQuery) Lte(v any) *RangeQuery { r.body["lte"] = v; return r }
func (r *RangeQuery) querySource() map[string]any {
return map[string]any{"range": map[string]any{r.field: r.body}}
}
// BoolQuery is the fluent builder for a bool query.
type BoolQuery struct {
must []Query
should []Query
mustNot []Query
}
func NewBoolQuery() *BoolQuery { return &BoolQuery{} }
func (b *BoolQuery) Must(q ...Query) *BoolQuery { b.must = append(b.must, q...); return b }
func (b *BoolQuery) Should(q ...Query) *BoolQuery { b.should = append(b.should, q...); return b }
func (b *BoolQuery) MustNot(q ...Query) *BoolQuery { b.mustNot = append(b.mustNot, q...); return b }
func (b *BoolQuery) querySource() map[string]any {
body := map[string]any{}
if len(b.must) > 0 {
body["must"] = querySlice(b.must)
}
if len(b.should) > 0 {
body["should"] = querySlice(b.should)
}
if len(b.mustNot) > 0 {
body["must_not"] = querySlice(b.mustNot)
}
return map[string]any{"bool": body}
}
func querySlice(queries []Query) []map[string]any {
out := make([]map[string]any, len(queries))
for idx, q := range queries {
out[idx] = q.querySource()
}
return out
}
@@ -0,0 +1,76 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package elasticsearch
import "gitea.dev/modules/json"
const (
bulkActionIndex = "index"
bulkActionDelete = "delete"
)
// BulkOp is a single write inside a Bulk call. Construct with IndexOp or DeleteOp.
type BulkOp struct {
action string
id string
doc any
}
// IndexOp builds a bulk index operation.
func IndexOp(id string, doc any) BulkOp {
return BulkOp{action: bulkActionIndex, id: id, doc: doc}
}
// DeleteOp builds a bulk delete operation.
func DeleteOp(id string) BulkOp {
return BulkOp{action: bulkActionDelete, id: id}
}
// SortField is one entry of the search sort array.
type SortField struct {
Field string
Desc bool
}
func (s SortField) source() map[string]any {
order := "asc"
if s.Desc {
order = "desc"
}
return map[string]any{s.Field: map[string]any{"order": order}}
}
// SearchRequest captures everything Gitea sends to the _search endpoint.
// Aggregations and Highlight are raw ES JSON bodies — callers write them as
// map[string]any since each has exactly one call site with a fixed shape.
type SearchRequest struct {
Query Query
Sort []SortField
From int
Size int
TrackTotal bool
Aggregations map[string]any
Highlight map[string]any
}
// SearchHit is a single result row.
type SearchHit struct {
ID string
Score float64
Source json.Value
Highlight map[string][]string
}
// AggBucket is a terms-aggregation bucket.
type AggBucket struct {
Key any
DocCount int64
}
// SearchResponse is Gitea's decoded view of the search reply.
type SearchResponse struct {
Total int64
Hits []SearchHit
Aggregations map[string][]AggBucket
}
@@ -6,14 +6,11 @@ package elasticsearch
import (
"context"
"fmt"
"time"
"code.gitea.io/gitea/modules/log"
"github.com/olivere/elastic/v7"
"gitea.dev/modules/log"
)
// VersionedIndexName returns the full index name with version
// VersionedIndexName returns the full index name with version suffix.
func (i *Indexer) VersionedIndexName() string {
return versionedIndexName(i.indexName, i.version)
}
@@ -26,41 +23,10 @@ func versionedIndexName(indexName string, version int) string {
return fmt.Sprintf("%s.v%d", indexName, version)
}
func (i *Indexer) createIndex(ctx context.Context) error {
createIndex, err := i.Client.CreateIndex(i.VersionedIndexName()).BodyString(i.mapping).Do(ctx)
if err != nil {
return err
}
if !createIndex.Acknowledged {
return fmt.Errorf("create index %s with %s failed", i.VersionedIndexName(), i.mapping)
}
i.checkOldIndexes(ctx)
return nil
}
func (i *Indexer) initClient() (*elastic.Client, error) {
opts := []elastic.ClientOptionFunc{
elastic.SetURL(i.url),
elastic.SetSniff(false),
elastic.SetHealthcheckInterval(10 * time.Second),
elastic.SetGzip(false),
}
logger := log.GetLogger(log.DEFAULT)
opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace}))
opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info}))
opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error}))
return elastic.NewClient(opts...)
}
func (i *Indexer) checkOldIndexes(ctx context.Context) {
for v := 0; v < i.version; v++ {
for v := range i.version {
indexName := versionedIndexName(i.indexName, v)
exists, err := i.Client.IndexExists(indexName).Do(ctx)
exists, err := i.indexExists(ctx, indexName)
if err == nil && exists {
log.Warn("Found older elasticsearch index named %q, Gitea will keep the old NOT DELETED. You can delete the old version after the upgrade succeed.", indexName)
}
@@ -6,7 +6,7 @@ package meilisearch
import (
"fmt"
"code.gitea.io/gitea/modules/log"
"gitea.dev/modules/log"
)
// VersionedIndexName returns the full index name with version
@@ -6,7 +6,7 @@ package internal
import (
"math"
"code.gitea.io/gitea/models/db"
"gitea.dev/models/db"
)
// ParsePaginator parses a db.Paginator into a skip and limit
@@ -7,16 +7,14 @@ import (
"context"
"strconv"
"code.gitea.io/gitea/modules/indexer"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/util"
"gitea.dev/modules/indexer"
indexer_internal "gitea.dev/modules/indexer/internal"
inner_bleve "gitea.dev/modules/indexer/internal/bleve"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/util"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
@@ -27,7 +25,7 @@ import (
const (
issueIndexerAnalyzer = "issueIndexer"
issueIndexerDocType = "issueIndexerDocType"
issueIndexerLatestVersion = 5
issueIndexerLatestVersion = 8
)
const unicodeNormalizeName = "unicodeNormalize"
@@ -83,10 +81,11 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) {
docMapping.AddFieldMappingsAt("label_ids", numberFieldMapping)
docMapping.AddFieldMappingsAt("no_label", boolFieldMapping)
docMapping.AddFieldMappingsAt("milestone_id", numberFieldMapping)
docMapping.AddFieldMappingsAt("project_id", numberFieldMapping)
docMapping.AddFieldMappingsAt("project_board_id", numberFieldMapping)
docMapping.AddFieldMappingsAt("project_ids", numberFieldMapping)
docMapping.AddFieldMappingsAt("no_project", boolFieldMapping)
docMapping.AddFieldMappingsAt("poster_id", numberFieldMapping)
docMapping.AddFieldMappingsAt("assignee_id", numberFieldMapping)
docMapping.AddFieldMappingsAt("assignee_ids", numberFieldMapping)
docMapping.AddFieldMappingsAt("no_assignee", boolFieldMapping)
docMapping.AddFieldMappingsAt("mention_ids", numberFieldMapping)
docMapping.AddFieldMappingsAt("reviewed_ids", numberFieldMapping)
docMapping.AddFieldMappingsAt("review_requested_ids", numberFieldMapping)
@@ -103,7 +102,7 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) {
"type": custom.Name,
"char_filters": []string{},
"tokenizer": unicode.Name,
"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
"token_filters": []string{unicodeNormalizeName, camelCaseKeepWholeName, lowercase.Name},
}); err != nil {
return nil, err
}
@@ -241,11 +240,15 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
queries = append(queries, bleve.NewDisjunctionQuery(milestoneQueries...))
}
if options.ProjectID.Has() {
queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectID.Value(), "project_id"))
}
if options.ProjectColumnID.Has() {
queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectColumnID.Value(), "project_board_id"))
if options.NoProjectOnly {
queries = append(queries, inner_bleve.BoolFieldQuery(true, "no_project"))
} else if len(options.ProjectIDs) > 0 {
var projectQueries []query.Query
for _, projectID := range options.ProjectIDs {
projectQueries = append(projectQueries, inner_bleve.NumericEqualityQuery(projectID, "project_ids"))
}
// FIXME: ISSUE-MULTIPLE-PROJECTS-FILTER: this logic is not right, it should use "AND" but not "OR"
queries = append(queries, bleve.NewDisjunctionQuery(projectQueries...))
}
if options.PosterID != "" {
@@ -254,14 +257,15 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
queries = append(queries, inner_bleve.NumericEqualityQuery(posterIDInt64, "poster_id"))
}
if options.AssigneeID != "" {
if options.AssigneeID == "(any)" {
queries = append(queries, inner_bleve.NumericRangeInclusiveQuery(optional.Some[int64](1), optional.None[int64](), "assignee_id"))
} else {
// "(none)" becomes 0, it means no assignee
assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64)
queries = append(queries, inner_bleve.NumericEqualityQuery(assigneeIDInt64, "assignee_id"))
}
switch options.AssigneeID {
case "":
case "(any)":
queries = append(queries, inner_bleve.BoolFieldQuery(false, "no_assignee"))
case "(none)":
queries = append(queries, inner_bleve.BoolFieldQuery(true, "no_assignee"))
default:
assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64)
queries = append(queries, inner_bleve.NumericEqualityQuery(assigneeIDInt64, "assignee_ids"))
}
if options.MentionID.Has() {
@@ -6,7 +6,11 @@ package bleve
import (
"testing"
"code.gitea.io/gitea/modules/indexer/issues/internal/tests"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/indexer/issues/internal/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBleveIndexer(t *testing.T) {
@@ -16,3 +20,108 @@ func TestBleveIndexer(t *testing.T) {
tests.TestIndexer(t, indexer)
}
func TestBleveIndexerNoAssignee(t *testing.T) {
dir := t.TempDir()
indexer := NewIndexer(dir)
defer indexer.Close()
_, err := indexer.Init(t.Context())
require.NoError(t, err)
require.NoError(t, indexer.Index(t.Context(),
&internal.IndexerData{ID: 1, Title: "assigned through assignee_ids", AssigneeIDs: []int64{2}},
&internal.IndexerData{ID: 2, Title: "unassigned", NoAssignee: true},
&internal.IndexerData{ID: 3, Title: "assigned through multiple assignee_ids", AssigneeIDs: []int64{3, 4}},
))
testCases := []struct {
name string
opts *internal.SearchOptions
expectedIDs []int64
}{
{
name: "none",
opts: &internal.SearchOptions{AssigneeID: "(none)"},
expectedIDs: []int64{2},
},
{
name: "any",
opts: &internal.SearchOptions{AssigneeID: "(any)"},
expectedIDs: []int64{1, 3},
},
{
name: "specific",
opts: &internal.SearchOptions{AssigneeID: "2"},
expectedIDs: []int64{1},
},
{
name: "specific first multi-assignee",
opts: &internal.SearchOptions{AssigneeID: "3"},
expectedIDs: []int64{3},
},
{
name: "specific second multi-assignee",
opts: &internal.SearchOptions{AssigneeID: "4"},
expectedIDs: []int64{3},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
result, err := indexer.Search(t.Context(), testCase.opts)
require.NoError(t, err)
assert.Equal(t, int64(len(testCase.expectedIDs)), result.Total)
assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(result))
})
}
}
func TestBleveIndexerTokenFilter(t *testing.T) {
dir := t.TempDir()
indexer := NewIndexer(dir)
defer indexer.Close()
_, err := indexer.Init(t.Context())
require.NoError(t, err)
require.NoError(t, indexer.Index(t.Context(),
&internal.IndexerData{ID: 1, Title: "fix(packages): SomeThing needs a rewrite (#12345)"},
&internal.IndexerData{ID: 2, Title: "add support for mDNS discovery abc1234"},
))
testCases := []struct {
name string
keyword string
expectedIDs []int64
}{
{name: "exact original case", keyword: "SomeThing", expectedIDs: []int64{1}},
{name: "case matching original transitions", keyword: "someThing", expectedIDs: []int64{1}},
{name: "all lower case", keyword: "something", expectedIDs: []int64{1}},
{name: "all upper case", keyword: "SOMETHING", expectedIDs: []int64{1}},
{name: "number match", keyword: "12345", expectedIDs: []int64{1}},
{name: "number as part", keyword: "1234", expectedIDs: []int64{2}},
{name: "sub-word search still works", keyword: "DNS", expectedIDs: []int64{2}},
{name: "sub-word search, lower case", keyword: "mdns", expectedIDs: []int64{2}},
{name: "keyword is camel case", keyword: "addSupport", expectedIDs: []int64{2}},
{name: "keyword not match", keyword: "addsupport", expectedIDs: []int64{}},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
result, err := indexer.Search(t.Context(), &internal.SearchOptions{
Keyword: testCase.keyword,
})
require.NoError(t, err)
assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(result))
})
}
}
func searchResultIDs(result *internal.SearchResult) []int64 {
ids := make([]int64, 0, len(result.Hits))
for _, hit := range result.Hits {
ids = append(ids, hit.ID)
}
return ids
}
@@ -0,0 +1,64 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package bleve
import (
"gitea.dev/modules/util"
"github.com/blevesearch/bleve/v2/analysis"
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
"github.com/blevesearch/bleve/v2/registry"
)
const camelCaseKeepWholeName = "camelCaseKeepWhole"
// camelCaseKeepWholeFilter behaves like bleve's built-in "camelCase" token filter,
// it also uses the whole word for a token. For example: when indexing "someThing",
// CamelCaseFilter only emits "some" and "thing", this filter also emits "something".
// It is questionable why the "issue indexer" used the CamelCaseFilter, it just can't search "someThing".
// To avoid breaking existing user experiences, this "whole token filter" is introduced to make the full word can be searched.
type camelCaseKeepWholeFilter struct {
inner *camelcase.CamelCaseFilter
}
func (f *camelCaseKeepWholeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
// First, do exactly what the stock camelCase filter does: split by "camelCase" tokens
split := f.inner.Filter(input)
// Index the resulting position of the *first* sub-token produced for
// each original token (matched by start offset), so the duplicated
// whole-word token we add below lines up at the same position as the
// sub-word it stands in for, instead of drifting out of sync for
// fields with more than one original token.
posByStart := make(map[int]int, len(split))
for _, tok := range split {
if _, ok := posByStart[tok.Start]; !ok {
posByStart[tok.Start] = tok.Position
}
}
rv := make(analysis.TokenStream, 0, len(split)+len(input))
rv = append(rv, split...)
// Then append one extra, un-split copy of every original token, so the
// whole word survives as a standalone, independently searchable term.
for _, token := range input {
dup := *token
dup.Term = append([]byte(nil), token.Term...)
if pos, ok := posByStart[token.Start]; ok {
dup.Position = pos
}
rv = append(rv, &dup)
}
return rv
}
func camelCaseKeepWholeFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
return &camelCaseKeepWholeFilter{inner: camelcase.NewCamelCaseFilter()}, nil
}
func init() {
util.MustNoError(registry.RegisterTokenFilter(camelCaseKeepWholeName, camelCaseKeepWholeFilterConstructor))
}
@@ -8,13 +8,13 @@ import (
"strings"
"sync"
"code.gitea.io/gitea/models/db"
issue_model "code.gitea.io/gitea/models/issues"
"code.gitea.io/gitea/modules/indexer"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_db "code.gitea.io/gitea/modules/indexer/internal/db"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/util"
"gitea.dev/models/db"
issue_model "gitea.dev/models/issues"
"gitea.dev/modules/indexer"
indexer_internal "gitea.dev/modules/indexer/internal"
inner_db "gitea.dev/modules/indexer/internal/db"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/util"
"xorm.io/builder"
)
@@ -8,11 +8,12 @@ import (
"fmt"
"strings"
"code.gitea.io/gitea/models/db"
issue_model "code.gitea.io/gitea/models/issues"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/optional"
"gitea.dev/models/db"
issue_model "gitea.dev/models/issues"
"gitea.dev/modules/container"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/optional"
"gitea.dev/modules/util"
)
func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_model.IssuesOptions, error) {
@@ -65,8 +66,7 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m
ReviewRequestedID: convertID(options.ReviewRequestedID),
ReviewedID: convertID(options.ReviewedID),
SubscriberID: convertID(options.SubscriberID),
ProjectID: convertID(options.ProjectID),
ProjectColumnID: convertID(options.ProjectColumnID),
ProjectIDs: util.Iif(options.NoProjectOnly, []int64{db.NoConditionID}, options.ProjectIDs),
IsClosed: options.IsClosed,
IsPull: options.IsPull,
IncludedLabelNames: nil,
@@ -6,11 +6,11 @@ package issues
import (
"strings"
"code.gitea.io/gitea/models/db"
issues_model "code.gitea.io/gitea/models/issues"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/setting"
"gitea.dev/models/db"
issues_model "gitea.dev/models/issues"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/optional"
"gitea.dev/modules/setting"
)
func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOptions {
@@ -46,10 +46,10 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp
searchOpt.MilestoneIDs = opts.MilestoneIDs
}
if opts.ProjectID > 0 {
searchOpt.ProjectID = optional.Some(opts.ProjectID)
} else if opts.ProjectID == db.NoConditionID { // FIXME: this is inconsistent from other places
searchOpt.ProjectID = optional.Some[int64](0) // Those issues with no project(projectid==0)
if len(opts.ProjectIDs) == 1 && opts.ProjectIDs[0] == db.NoConditionID {
searchOpt.NoProjectOnly = true
} else {
searchOpt.ProjectIDs = opts.ProjectIDs
}
searchOpt.AssigneeID = opts.AssigneeID
@@ -65,7 +65,6 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp
return nil
}
searchOpt.ProjectColumnID = convertID(opts.ProjectColumnID)
searchOpt.PosterID = opts.PosterID
searchOpt.MentionID = convertID(opts.MentionedID)
searchOpt.ReviewedID = convertID(opts.ReviewedID)
@@ -8,30 +8,21 @@ import (
"strconv"
"strings"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/indexer"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/util"
"github.com/olivere/elastic/v7"
"gitea.dev/modules/graceful"
"gitea.dev/modules/indexer"
indexer_internal "gitea.dev/modules/indexer/internal"
es "gitea.dev/modules/indexer/internal/elasticsearch"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/util"
)
const (
issueIndexerLatestVersion = 2
// multi-match-types, currently only 2 types are used
// Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types
esMultiMatchTypeBestFields = "best_fields"
esMultiMatchTypePhrasePrefix = "phrase_prefix"
)
const issueIndexerLatestVersion = 4
var _ internal.Indexer = &Indexer{}
// Indexer implements Indexer interface
type Indexer struct {
inner *inner_elasticsearch.Indexer
indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much
*es.Indexer
}
func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
@@ -41,12 +32,7 @@ func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
// NewIndexer creates a new elasticsearch indexer
func NewIndexer(url, indexerName string) *Indexer {
inner := inner_elasticsearch.NewIndexer(url, indexerName, issueIndexerLatestVersion, defaultMapping)
indexer := &Indexer{
inner: inner,
Indexer: inner,
}
return indexer
return &Indexer{Indexer: es.NewIndexer(url, indexerName, issueIndexerLatestVersion, defaultMapping)}
}
const (
@@ -68,10 +54,11 @@ const (
"label_ids": { "type": "integer", "index": true },
"no_label": { "type": "boolean", "index": true },
"milestone_id": { "type": "integer", "index": true },
"project_id": { "type": "integer", "index": true },
"project_board_id": { "type": "integer", "index": true },
"project_ids": { "type": "integer", "index": true },
"no_project": { "type": "boolean", "index": true },
"poster_id": { "type": "integer", "index": true },
"assignee_id": { "type": "integer", "index": true },
"assignee_ids": { "type": "integer", "index": true },
"no_assignee": { "type": "boolean", "index": true },
"mention_ids": { "type": "integer", "index": true },
"reviewed_ids": { "type": "integer", "index": true },
"review_requested_ids": { "type": "integer", "index": true },
@@ -93,29 +80,14 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er
return nil
} else if len(issues) == 1 {
issue := issues[0]
_, err := b.inner.Client.Index().
Index(b.inner.VersionedIndexName()).
Id(strconv.FormatInt(issue.ID, 10)).
BodyJson(issue).
Do(ctx)
return err
return b.Indexer.Index(ctx, strconv.FormatInt(issue.ID, 10), issue)
}
reqs := make([]elastic.BulkableRequest, 0)
ops := make([]es.BulkOp, 0, len(issues))
for _, issue := range issues {
reqs = append(reqs,
elastic.NewBulkIndexRequest().
Index(b.inner.VersionedIndexName()).
Id(strconv.FormatInt(issue.ID, 10)).
Doc(issue),
)
ops = append(ops, es.IndexOp(strconv.FormatInt(issue.ID, 10), issue))
}
_, err := b.inner.Client.Bulk().
Index(b.inner.VersionedIndexName()).
Add(reqs...).
Do(graceful.GetManager().HammerContext())
return err
return b.Bulk(graceful.GetManager().HammerContext(), ops)
}
// Delete deletes indexes by ids
@@ -123,129 +95,117 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error {
if len(ids) == 0 {
return nil
} else if len(ids) == 1 {
_, err := b.inner.Client.Delete().
Index(b.inner.VersionedIndexName()).
Id(strconv.FormatInt(ids[0], 10)).
Do(ctx)
return err
return b.Indexer.Delete(ctx, strconv.FormatInt(ids[0], 10))
}
reqs := make([]elastic.BulkableRequest, 0)
ops := make([]es.BulkOp, 0, len(ids))
for _, id := range ids {
reqs = append(reqs,
elastic.NewBulkDeleteRequest().
Index(b.inner.VersionedIndexName()).
Id(strconv.FormatInt(id, 10)),
)
ops = append(ops, es.DeleteOp(strconv.FormatInt(id, 10)))
}
_, err := b.inner.Client.Bulk().
Index(b.inner.VersionedIndexName()).
Add(reqs...).
Do(graceful.GetManager().HammerContext())
return err
return b.Bulk(graceful.GetManager().HammerContext(), ops)
}
// Search searches for issues by given conditions.
// Returns the matching issue IDs
func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) {
query := elastic.NewBoolQuery()
query := es.NewBoolQuery()
if options.Keyword != "" {
searchMode := util.IfZero(options.SearchMode, b.SupportedSearchModes()[0].ModeValue)
mm := es.NewMultiMatchQuery(options.Keyword, "title", "content", "comments")
if searchMode == indexer.SearchModeExact {
query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(esMultiMatchTypePhrasePrefix))
} else /* words */ {
query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(esMultiMatchTypeBestFields).Operator("and"))
mm = mm.Type(es.MultiMatchTypePhrasePrefix)
} else {
mm = mm.Type(es.MultiMatchTypeBestFields).Operator("and")
}
query.Must(mm)
}
if len(options.RepoIDs) > 0 {
q := elastic.NewBoolQuery()
q.Should(elastic.NewTermsQuery("repo_id", toAnySlice(options.RepoIDs)...))
q := es.NewBoolQuery()
q.Should(es.TermsQuery("repo_id", es.ToAnySlice(options.RepoIDs)...))
if options.AllPublic {
q.Should(elastic.NewTermQuery("is_public", true))
q.Should(es.TermQuery("is_public", true))
}
query.Must(q)
}
if options.IsPull.Has() {
query.Must(elastic.NewTermQuery("is_pull", options.IsPull.Value()))
query.Must(es.TermQuery("is_pull", options.IsPull.Value()))
}
if options.IsClosed.Has() {
query.Must(elastic.NewTermQuery("is_closed", options.IsClosed.Value()))
query.Must(es.TermQuery("is_closed", options.IsClosed.Value()))
}
if options.IsArchived.Has() {
query.Must(elastic.NewTermQuery("is_archived", options.IsArchived.Value()))
query.Must(es.TermQuery("is_archived", options.IsArchived.Value()))
}
if options.NoLabelOnly {
query.Must(elastic.NewTermQuery("no_label", true))
query.Must(es.TermQuery("no_label", true))
} else {
if len(options.IncludedLabelIDs) > 0 {
q := elastic.NewBoolQuery()
q := es.NewBoolQuery()
for _, labelID := range options.IncludedLabelIDs {
q.Must(elastic.NewTermQuery("label_ids", labelID))
q.Must(es.TermQuery("label_ids", labelID))
}
query.Must(q)
} else if len(options.IncludedAnyLabelIDs) > 0 {
query.Must(elastic.NewTermsQuery("label_ids", toAnySlice(options.IncludedAnyLabelIDs)...))
query.Must(es.TermsQuery("label_ids", es.ToAnySlice(options.IncludedAnyLabelIDs)...))
}
if len(options.ExcludedLabelIDs) > 0 {
q := elastic.NewBoolQuery()
q := es.NewBoolQuery()
for _, labelID := range options.ExcludedLabelIDs {
q.MustNot(elastic.NewTermQuery("label_ids", labelID))
q.MustNot(es.TermQuery("label_ids", labelID))
}
query.Must(q)
}
}
if len(options.MilestoneIDs) > 0 {
query.Must(elastic.NewTermsQuery("milestone_id", toAnySlice(options.MilestoneIDs)...))
query.Must(es.TermsQuery("milestone_id", es.ToAnySlice(options.MilestoneIDs)...))
}
if options.ProjectID.Has() {
query.Must(elastic.NewTermQuery("project_id", options.ProjectID.Value()))
}
if options.ProjectColumnID.Has() {
query.Must(elastic.NewTermQuery("project_board_id", options.ProjectColumnID.Value()))
if options.NoProjectOnly {
query.Must(es.TermQuery("no_project", true))
} else if len(options.ProjectIDs) > 0 {
// FIXME: ISSUE-MULTIPLE-PROJECTS-FILTER: this logic is not right, it should use "AND" but not "OR"
query.Must(es.TermsQuery("project_ids", es.ToAnySlice(options.ProjectIDs)...))
}
if options.PosterID != "" {
// "(none)" becomes 0, it means no poster
posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64)
query.Must(elastic.NewTermQuery("poster_id", posterIDInt64))
query.Must(es.TermQuery("poster_id", posterIDInt64))
}
if options.AssigneeID != "" {
if options.AssigneeID == "(any)" {
q := elastic.NewRangeQuery("assignee_id")
q.Gte(1)
query.Must(q)
} else {
// "(none)" becomes 0, it means no assignee
assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64)
query.Must(elastic.NewTermQuery("assignee_id", assigneeIDInt64))
}
switch options.AssigneeID {
case "":
case "(any)":
query.Must(es.TermQuery("no_assignee", false))
case "(none)":
query.Must(es.TermQuery("no_assignee", true))
default:
assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64)
query.Must(es.TermQuery("assignee_ids", assigneeIDInt64))
}
if options.MentionID.Has() {
query.Must(elastic.NewTermQuery("mention_ids", options.MentionID.Value()))
query.Must(es.TermQuery("mention_ids", options.MentionID.Value()))
}
if options.ReviewedID.Has() {
query.Must(elastic.NewTermQuery("reviewed_ids", options.ReviewedID.Value()))
query.Must(es.TermQuery("reviewed_ids", options.ReviewedID.Value()))
}
if options.ReviewRequestedID.Has() {
query.Must(elastic.NewTermQuery("review_requested_ids", options.ReviewRequestedID.Value()))
query.Must(es.TermQuery("review_requested_ids", options.ReviewRequestedID.Value()))
}
if options.SubscriberID.Has() {
query.Must(elastic.NewTermQuery("subscriber_ids", options.SubscriberID.Value()))
query.Must(es.TermQuery("subscriber_ids", options.SubscriberID.Value()))
}
if options.UpdatedAfterUnix.Has() || options.UpdatedBeforeUnix.Has() {
q := elastic.NewRangeQuery("updated_unix")
q := es.NewRangeQuery("updated_unix")
if options.UpdatedAfterUnix.Has() {
q.Gte(options.UpdatedAfterUnix.Value())
}
@@ -258,9 +218,9 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
if options.SortBy == "" {
options.SortBy = internal.SortByCreatedAsc
}
sortBy := []elastic.Sorter{
sortBy := []es.SortField{
parseSortBy(options.SortBy),
elastic.NewFieldSort("id").Desc(),
{Field: "id", Desc: true},
}
// See https://stackoverflow.com/questions/35206409/elasticsearch-2-1-result-window-is-too-large-index-max-result-window/35221900
@@ -268,43 +228,30 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
const maxPageSize = 10000
skip, limit := indexer_internal.ParsePaginator(options.Paginator, maxPageSize)
searchResult, err := b.inner.Client.Search().
Index(b.inner.VersionedIndexName()).
Query(query).
SortBy(sortBy...).
From(skip).Size(limit).
Do(ctx)
resp, err := b.Indexer.Search(ctx, es.SearchRequest{
Query: query,
Sort: sortBy,
From: skip,
Size: limit,
TrackTotal: true,
})
if err != nil {
return nil, err
}
hits := make([]internal.Match, 0, limit)
for _, hit := range searchResult.Hits.Hits {
id, _ := strconv.ParseInt(hit.Id, 10, 64)
hits = append(hits, internal.Match{
ID: id,
})
hits := make([]internal.Match, 0, len(resp.Hits))
for _, hit := range resp.Hits {
id, _ := strconv.ParseInt(hit.ID, 10, 64)
hits = append(hits, internal.Match{ID: id})
}
return &internal.SearchResult{
Total: searchResult.TotalHits(),
Total: resp.Total,
Hits: hits,
}, nil
}
func toAnySlice[T any](s []T) []any {
ret := make([]any, 0, len(s))
for _, item := range s {
ret = append(ret, item)
}
return ret
}
func parseSortBy(sortBy internal.SortBy) elastic.Sorter {
field := strings.TrimPrefix(string(sortBy), "-")
ret := elastic.NewFieldSort(field)
if strings.HasPrefix(string(sortBy), "-") {
ret.Desc()
}
return ret
func parseSortBy(sortBy internal.SortBy) es.SortField {
field, desc := strings.CutPrefix(string(sortBy), "-")
return es.SortField{Field: field, Desc: desc}
}
@@ -6,30 +6,39 @@ package elasticsearch
import (
"fmt"
"net/http"
"os"
"net/url"
"testing"
"time"
"code.gitea.io/gitea/modules/indexer/issues/internal/tests"
"gitea.dev/modules/indexer/issues/internal/tests"
"gitea.dev/modules/test"
"github.com/stretchr/testify/require"
)
func TestElasticsearchIndexer(t *testing.T) {
// The elasticsearch instance started by pull-db-tests.yml > test-unit > services > elasticsearch
url := "http://elastic:changeme@elasticsearch:9200"
rawURL := test.ExternalServiceHTTP(t, "TEST_ELASTICSEARCH_URL", "http://elastic:changeme@elasticsearch:9200")
if os.Getenv("CI") == "" {
// Make it possible to run tests against a local elasticsearch instance
url = os.Getenv("TEST_ELASTICSEARCH_URL")
if url == "" {
t.Skip("TEST_ELASTICSEARCH_URL not set and not running in CI")
return
}
}
// Go's net/http does not auto-attach URL userinfo as Basic Auth, so extract
// it and set the header explicitly; otherwise auth-enforced clusters answer
// 401 and the probe never reports ready.
parsed, err := url.Parse(rawURL)
require.NoError(t, err)
user := parsed.User
parsed.User = nil
probeURL := parsed.String()
require.Eventually(t, func() bool {
resp, err := http.Get(url)
req, err := http.NewRequest(http.MethodGet, probeURL, nil)
if err != nil {
return false
}
if user != nil {
pass, _ := user.Password()
req.SetBasicAuth(user.Username(), pass)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return false
}
@@ -37,7 +46,7 @@ func TestElasticsearchIndexer(t *testing.T) {
return resp.StatusCode == http.StatusOK
}, time.Minute, time.Second, "Expected elasticsearch to be up")
indexer := NewIndexer(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix()))
indexer := NewIndexer(rawURL, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix()))
defer indexer.Close()
tests.TestIndexer(t, indexer)
@@ -11,21 +11,21 @@ import (
"sync/atomic"
"time"
db_model "code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/indexer/issues/bleve"
"code.gitea.io/gitea/modules/indexer/issues/db"
"code.gitea.io/gitea/modules/indexer/issues/elasticsearch"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/indexer/issues/meilisearch"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
db_model "gitea.dev/models/db"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/graceful"
"gitea.dev/modules/indexer"
"gitea.dev/modules/indexer/issues/bleve"
"gitea.dev/modules/indexer/issues/db"
"gitea.dev/modules/indexer/issues/elasticsearch"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/indexer/issues/meilisearch"
"gitea.dev/modules/log"
"gitea.dev/modules/optional"
"gitea.dev/modules/process"
"gitea.dev/modules/queue"
"gitea.dev/modules/setting"
"gitea.dev/modules/util"
)
// IndexerMetadata is used to send data to the queue, so it contains only the ids.
@@ -6,16 +6,16 @@ package issues
import (
"testing"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/issues"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/setting"
"gitea.dev/models/db"
"gitea.dev/models/issues"
"gitea.dev/models/unittest"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/optional"
"gitea.dev/modules/setting"
_ "code.gitea.io/gitea/models"
_ "code.gitea.io/gitea/models/actions"
_ "code.gitea.io/gitea/models/activities"
_ "gitea.dev/models"
_ "gitea.dev/models/actions"
_ "gitea.dev/models/activities"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -416,28 +416,42 @@ func searchIssueInProject(t *testing.T) {
}{
{
SearchOptions{
ProjectID: optional.Some(int64(1)),
ProjectIDs: []int64{1},
},
[]int64{5, 3, 2, 1},
},
{
SearchOptions{
ProjectColumnID: optional.Some(int64(1)),
},
[]int64{1},
},
{
SearchOptions{
ProjectColumnID: optional.Some(int64(0)), // issue with in default column
},
[]int64{2},
},
}
for _, test := range tests {
issueIDs, _, err := SearchIssues(t.Context(), &test.opts)
require.NoError(t, err)
assert.Equal(t, test.expectedIDs, issueIDs)
}
// Test filtering for issues with no project assigned using dynamic validation
t.Run("no project assigned", func(t *testing.T) {
issueIDs, total, err := SearchIssues(t.Context(), &SearchOptions{
ProjectIDs: []int64{db.NoConditionID},
})
require.NoError(t, err)
assert.NotEmpty(t, issueIDs)
assert.Equal(t, total, int64(len(issueIDs)))
// Verify each returned issue actually has no project
for _, issueID := range issueIDs {
issue, err := issues.GetIssueByID(t.Context(), issueID)
require.NoError(t, err)
err = issue.LoadProjects(t.Context())
require.NoError(t, err)
assert.Empty(t, issue.Projects, "Issue %d should have no projects", issueID)
}
// Count total issues with no project to verify we got them all
allIssues, err := issues.Issues(t.Context(), &issues.IssuesOptions{
ProjectIDs: []int64{db.NoConditionID},
})
require.NoError(t, err)
assert.Len(t, issueIDs, len(allIssues), "Should return all issues with no project")
})
}
func searchIssueWithPaginator(t *testing.T) {
@@ -7,8 +7,8 @@ import (
"context"
"errors"
"code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/indexer/internal"
"gitea.dev/modules/indexer"
"gitea.dev/modules/indexer/internal"
)
// Indexer defines an interface to indexer issues contents
@@ -6,10 +6,10 @@ package internal
import (
"strconv"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/timeutil"
"gitea.dev/models/db"
"gitea.dev/modules/indexer"
"gitea.dev/modules/optional"
"gitea.dev/modules/timeutil"
)
// IndexerData data stored in the issue indexer
@@ -30,10 +30,12 @@ type IndexerData struct {
LabelIDs []int64 `json:"label_ids"`
NoLabel bool `json:"no_label"` // True if LabelIDs is empty
MilestoneID int64 `json:"milestone_id"`
ProjectID int64 `json:"project_id"`
ProjectColumnID int64 `json:"project_board_id"` // the key should be kept as project_board_id to keep compatible
ProjectIDs []int64 `json:"project_ids"`
NoProject bool `json:"no_project"` // True if ProjectIDs is empty
ProjectColumnMap map[int64]int64 `json:"project_column_map,omitempty"` // Maps project ID to column ID for each project the issue is in
PosterID int64 `json:"poster_id"`
AssigneeID int64 `json:"assignee_id"`
AssigneeIDs []int64 `json:"assignee_ids"`
NoAssignee bool `json:"no_assignee"` // True if the issue has no assignees
MentionIDs []int64 `json:"mention_ids"`
ReviewedIDs []int64 `json:"reviewed_ids"`
ReviewRequestedIDs []int64 `json:"review_requested_ids"`
@@ -94,8 +96,8 @@ type SearchOptions struct {
MilestoneIDs []int64 // milestones the issues have
ProjectID optional.Option[int64] // project the issues belong to
ProjectColumnID optional.Option[int64] // project column the issues belong to
ProjectIDs []int64 // project the issues belong to. FIXME: ISSUE-MULTIPLE-PROJECTS-FILTER: no multiple project filter support yet. Search logic is wrong.
NoProjectOnly bool // if the issues have no project, if true, ProjectIDs will be ignored
PosterID string // poster of the issues, "(none)" or "(any)" or a user ID
AssigneeID string // assignee of the issues, "(none)" or "(any)" or a user ID
@@ -13,10 +13,10 @@ import (
"testing"
"time"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/timeutil"
"gitea.dev/models/db"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/optional"
"gitea.dev/modules/timeutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -116,6 +116,16 @@ var cases = []*testIndexerCase{
assert.Equal(t, len(data), int(result.Total))
},
},
{
// Exercises the single-doc Index/Delete fast path in backends that have one (e.g. Elasticsearch).
Name: "single-doc index",
ExtraData: []*internal.IndexerData{
{ID: 999, Title: "solo-issue-marker"},
},
SearchOptions: &internal.SearchOptions{Keyword: "solo-issue-marker"},
ExpectedIDs: []int64{999},
ExpectedTotal: 1,
},
{
Name: "Keyword",
ExtraData: []*internal.IndexerData{
@@ -301,75 +311,41 @@ var cases = []*testIndexerCase{
},
},
{
Name: "ProjectID",
Name: "ProjectIDs",
SearchOptions: &internal.SearchOptions{
Paginator: &db.ListOptions{
PageSize: 5,
},
ProjectID: optional.Some(int64(1)),
ProjectIDs: []int64{1},
},
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Len(t, result.Hits, 5)
for _, v := range result.Hits {
assert.Equal(t, int64(1), data[v.ID].ProjectID)
assert.Contains(t, data[v.ID].ProjectIDs, int64(1))
}
assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool {
return v.ProjectID == 1
return slices.Contains(v.ProjectIDs, int64(1))
}), result.Total)
},
},
{
Name: "no ProjectID",
Name: "no ProjectIDs (empty array)",
SearchOptions: &internal.SearchOptions{
Paginator: &db.ListOptions{
PageSize: 5,
PageSize: 50,
},
ProjectID: optional.Some(int64(0)),
NoProjectOnly: true,
},
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Len(t, result.Hits, 5)
// Verify only issues with no projects are returned
for _, v := range result.Hits {
assert.Equal(t, int64(0), data[v.ID].ProjectID)
assert.Empty(t, data[v.ID].ProjectIDs, "Issue %d should have no projects", v.ID)
}
assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool {
return v.ProjectID == 0
}), result.Total)
},
},
{
Name: "ProjectColumnID",
SearchOptions: &internal.SearchOptions{
Paginator: &db.ListOptions{
PageSize: 5,
},
ProjectColumnID: optional.Some(int64(1)),
},
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Len(t, result.Hits, 5)
for _, v := range result.Hits {
assert.Equal(t, int64(1), data[v.ID].ProjectColumnID)
}
assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool {
return v.ProjectColumnID == 1
}), result.Total)
},
},
{
Name: "no ProjectColumnID",
SearchOptions: &internal.SearchOptions{
Paginator: &db.ListOptions{
PageSize: 5,
},
ProjectColumnID: optional.Some(int64(0)),
},
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Len(t, result.Hits, 5)
for _, v := range result.Hits {
assert.Equal(t, int64(0), data[v.ID].ProjectColumnID)
}
assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool {
return v.ProjectColumnID == 0
}), result.Total)
// Verify we got ALL issues with no projects
expectedCount := countIndexerData(data, func(v *internal.IndexerData) bool {
return len(v.ProjectIDs) == 0
})
assert.Equal(t, expectedCount, result.Total, "Should return all %d issues with no project", expectedCount)
},
},
{
@@ -401,10 +377,10 @@ var cases = []*testIndexerCase{
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Len(t, result.Hits, 5)
for _, v := range result.Hits {
assert.Equal(t, int64(1), data[v.ID].AssigneeID)
assert.True(t, slices.Contains(data[v.ID].AssigneeIDs, int64(1)))
}
assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool {
return v.AssigneeID == 1
return slices.Contains(v.AssigneeIDs, int64(1))
}), result.Total)
},
},
@@ -419,10 +395,10 @@ var cases = []*testIndexerCase{
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Len(t, result.Hits, 5)
for _, v := range result.Hits {
assert.Equal(t, int64(0), data[v.ID].AssigneeID)
assert.True(t, data[v.ID].NoAssignee)
}
assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool {
return v.AssigneeID == 0
return v.NoAssignee
}), result.Total)
},
},
@@ -654,10 +630,10 @@ var cases = []*testIndexerCase{
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Len(t, result.Hits, 180)
for _, v := range result.Hits {
assert.GreaterOrEqual(t, data[v.ID].AssigneeID, int64(1))
assert.False(t, data[v.ID].NoAssignee)
}
assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool {
return v.AssigneeID >= 1
return !v.NoAssignee
}), result.Total)
},
},
@@ -706,6 +682,22 @@ func generateDefaultIndexerData() []*internal.IndexerData {
for i := range subscriberIDs {
subscriberIDs[i] = int64(i) + 1 // SubscriberID should not be 0
}
projectIDs := make([]int64, id%5)
for i := range projectIDs {
projectIDs[i] = int64(i) + 1 // projectID should not be 0
}
var assigneeIDs []int64
if issueIndex%10 != 0 {
assigneeID := issueIndex % 10
assigneeIDs = []int64{assigneeID}
if issueIndex%3 == 0 {
nextAssigneeID := assigneeID + 1
if nextAssigneeID == 10 {
nextAssigneeID = 1
}
assigneeIDs = append(assigneeIDs, nextAssigneeID)
}
}
data = append(data, &internal.IndexerData{
ID: id,
@@ -719,10 +711,11 @@ func generateDefaultIndexerData() []*internal.IndexerData {
LabelIDs: labelIDs,
NoLabel: len(labelIDs) == 0,
MilestoneID: issueIndex % 4,
ProjectID: issueIndex % 5,
ProjectColumnID: issueIndex % 6,
ProjectIDs: projectIDs,
NoProject: len(projectIDs) == 0,
PosterID: id%10 + 1, // PosterID should not be 0
AssigneeID: issueIndex % 10,
AssigneeIDs: assigneeIDs,
NoAssignee: len(assigneeIDs) == 0,
MentionIDs: mentionIDs,
ReviewedIDs: reviewedIDs,
ReviewRequestedIDs: reviewRequestedIDs,
@@ -10,17 +10,17 @@ import (
"strconv"
"strings"
"code.gitea.io/gitea/modules/indexer"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_meilisearch "code.gitea.io/gitea/modules/indexer/internal/meilisearch"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/json"
"gitea.dev/modules/indexer"
indexer_internal "gitea.dev/modules/indexer/internal"
inner_meilisearch "gitea.dev/modules/indexer/internal/meilisearch"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/json"
"github.com/meilisearch/meilisearch-go"
)
const (
issueIndexerLatestVersion = 4
issueIndexerLatestVersion = 6
// TODO: make this configurable if necessary
maxTotalHits = 10000
@@ -71,10 +71,11 @@ func NewIndexer(url, apiKey, indexerName string) *Indexer {
"label_ids",
"no_label",
"milestone_id",
"project_id",
"project_board_id",
"project_ids",
"no_project",
"poster_id",
"assignee_id",
"assignee_ids",
"no_assignee",
"mention_ids",
"reviewed_ids",
"review_requested_ids",
@@ -182,11 +183,11 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
query.And(inner_meilisearch.NewFilterIn("milestone_id", options.MilestoneIDs...))
}
if options.ProjectID.Has() {
query.And(inner_meilisearch.NewFilterEq("project_id", options.ProjectID.Value()))
}
if options.ProjectColumnID.Has() {
query.And(inner_meilisearch.NewFilterEq("project_board_id", options.ProjectColumnID.Value()))
if options.NoProjectOnly {
query.And(inner_meilisearch.NewFilterEq("no_project", true))
} else if len(options.ProjectIDs) > 0 {
// FIXME: ISSUE-MULTIPLE-PROJECTS-FILTER: this logic is not right, it should use "AND" but not "OR"
query.And(inner_meilisearch.NewFilterIn("project_ids", options.ProjectIDs...))
}
if options.PosterID != "" {
@@ -195,14 +196,15 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
query.And(inner_meilisearch.NewFilterEq("poster_id", posterIDInt64))
}
if options.AssigneeID != "" {
if options.AssigneeID == "(any)" {
query.And(inner_meilisearch.NewFilterGte("assignee_id", 1))
} else {
// "(none)" becomes 0, it means no assignee
assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64)
query.And(inner_meilisearch.NewFilterEq("assignee_id", assigneeIDInt64))
}
switch options.AssigneeID {
case "":
case "(any)":
query.And(inner_meilisearch.NewFilterEq("no_assignee", false))
case "(none)":
query.And(inner_meilisearch.NewFilterEq("no_assignee", true))
default:
assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64)
query.And(inner_meilisearch.NewFilterEq("assignee_ids", assigneeIDInt64))
}
if options.MentionID.Has() {
@@ -10,9 +10,10 @@ import (
"testing"
"time"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/indexer/issues/internal/tests"
"code.gitea.io/gitea/modules/json"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/indexer/issues/internal/tests"
"gitea.dev/modules/json"
"gitea.dev/modules/test"
"github.com/meilisearch/meilisearch-go"
"github.com/stretchr/testify/assert"
@@ -21,18 +22,8 @@ import (
func TestMeilisearchIndexer(t *testing.T) {
// The meilisearch instance started by pull-db-tests.yml > test-unit > services > meilisearch
url := "http://meilisearch:7700"
key := "" // auth has been disabled in test environment
if os.Getenv("CI") == "" {
// Make it possible to run tests against a local meilisearch instance
url = os.Getenv("TEST_MEILISEARCH_URL")
if url == "" {
t.Skip("TEST_MEILISEARCH_URL not set and not running in CI")
return
}
key = os.Getenv("TEST_MEILISEARCH_KEY")
}
url := test.ExternalServiceHTTP(t, "TEST_MEILISEARCH_URL", "http://meilisearch:7700")
key := os.Getenv("TEST_MEILISEARCH_KEY")
require.Eventually(t, func() bool {
resp, err := http.Get(url)
@@ -8,12 +8,12 @@ import (
"errors"
"fmt"
"code.gitea.io/gitea/models/db"
issue_model "code.gitea.io/gitea/models/issues"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
"gitea.dev/models/db"
issue_model "gitea.dev/models/issues"
"gitea.dev/modules/container"
"gitea.dev/modules/indexer/issues/internal"
"gitea.dev/modules/log"
"gitea.dev/modules/queue"
)
// getIssueIndexerData returns the indexer data of an issue and a bool value indicating whether the issue exists.
@@ -87,14 +87,14 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD
return nil, false, err
}
var projectID int64
if issue.Project != nil {
projectID = issue.Project.ID
assigneeIDs := make([]int64, 0, len(issue.Assignees))
for _, a := range issue.Assignees {
assigneeIDs = append(assigneeIDs, a.ID)
}
projectColumnID, err := issue.ProjectColumnID(ctx)
if err != nil {
return nil, false, err
projectIDs := make([]int64, 0, len(issue.Projects))
for _, project := range issue.Projects {
projectIDs = append(projectIDs, project.ID)
}
if err := issue.Repo.LoadOwner(ctx); err != nil {
@@ -114,10 +114,11 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD
LabelIDs: labels,
NoLabel: len(labels) == 0,
MilestoneID: issue.MilestoneID,
ProjectID: projectID,
ProjectColumnID: projectColumnID,
ProjectIDs: projectIDs,
NoProject: len(projectIDs) == 0,
PosterID: issue.PosterID,
AssigneeID: issue.AssigneeID,
AssigneeIDs: assigneeIDs,
NoAssignee: len(assigneeIDs) == 0,
MentionIDs: mentionIDs,
ReviewedIDs: reviewedIDs,
ReviewRequestedIDs: reviewRequestedIDs,
@@ -6,14 +6,14 @@ package stats
import (
"fmt"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/languagestats"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/git"
"gitea.dev/modules/git/languagestats"
"gitea.dev/modules/gitrepo"
"gitea.dev/modules/graceful"
"gitea.dev/modules/log"
"gitea.dev/modules/process"
"gitea.dev/modules/setting"
)
// DBIndexer implements Indexer interface to use database's like search
@@ -6,10 +6,10 @@ package stats
import (
"context"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"gitea.dev/models/db"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/graceful"
"gitea.dev/modules/log"
)
// Indexer defines an interface to index repository stats
@@ -7,14 +7,14 @@ import (
"testing"
"time"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
repo_model "gitea.dev/models/repo"
"gitea.dev/models/unittest"
"gitea.dev/modules/queue"
"gitea.dev/modules/setting"
_ "code.gitea.io/gitea/models"
_ "code.gitea.io/gitea/models/actions"
_ "code.gitea.io/gitea/models/activities"
_ "gitea.dev/models"
_ "gitea.dev/models/actions"
_ "gitea.dev/models/activities"
"github.com/stretchr/testify/assert"
)
@@ -6,11 +6,11 @@ package stats
import (
"errors"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/graceful"
"gitea.dev/modules/log"
"gitea.dev/modules/queue"
"gitea.dev/modules/setting"
)
// statsQueue represents a queue to handle repository stats updates