This commit is contained in:
@@ -7,16 +7,14 @@ import (
|
||||
"context"
|
||||
"strconv"
|
||||
|
||||
"code.gitea.io/gitea/modules/indexer"
|
||||
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
|
||||
inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve"
|
||||
"code.gitea.io/gitea/modules/indexer/issues/internal"
|
||||
"code.gitea.io/gitea/modules/optional"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
"gitea.dev/modules/indexer"
|
||||
indexer_internal "gitea.dev/modules/indexer/internal"
|
||||
inner_bleve "gitea.dev/modules/indexer/internal/bleve"
|
||||
"gitea.dev/modules/indexer/issues/internal"
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
|
||||
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
|
||||
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
|
||||
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
|
||||
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
|
||||
@@ -27,7 +25,7 @@ import (
|
||||
const (
|
||||
issueIndexerAnalyzer = "issueIndexer"
|
||||
issueIndexerDocType = "issueIndexerDocType"
|
||||
issueIndexerLatestVersion = 5
|
||||
issueIndexerLatestVersion = 8
|
||||
)
|
||||
|
||||
const unicodeNormalizeName = "unicodeNormalize"
|
||||
@@ -83,10 +81,11 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) {
|
||||
docMapping.AddFieldMappingsAt("label_ids", numberFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("no_label", boolFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("milestone_id", numberFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("project_id", numberFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("project_board_id", numberFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("project_ids", numberFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("no_project", boolFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("poster_id", numberFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("assignee_id", numberFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("assignee_ids", numberFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("no_assignee", boolFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("mention_ids", numberFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("reviewed_ids", numberFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("review_requested_ids", numberFieldMapping)
|
||||
@@ -103,7 +102,7 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) {
|
||||
"type": custom.Name,
|
||||
"char_filters": []string{},
|
||||
"tokenizer": unicode.Name,
|
||||
"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
|
||||
"token_filters": []string{unicodeNormalizeName, camelCaseKeepWholeName, lowercase.Name},
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -241,11 +240,15 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
|
||||
queries = append(queries, bleve.NewDisjunctionQuery(milestoneQueries...))
|
||||
}
|
||||
|
||||
if options.ProjectID.Has() {
|
||||
queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectID.Value(), "project_id"))
|
||||
}
|
||||
if options.ProjectColumnID.Has() {
|
||||
queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectColumnID.Value(), "project_board_id"))
|
||||
if options.NoProjectOnly {
|
||||
queries = append(queries, inner_bleve.BoolFieldQuery(true, "no_project"))
|
||||
} else if len(options.ProjectIDs) > 0 {
|
||||
var projectQueries []query.Query
|
||||
for _, projectID := range options.ProjectIDs {
|
||||
projectQueries = append(projectQueries, inner_bleve.NumericEqualityQuery(projectID, "project_ids"))
|
||||
}
|
||||
// FIXME: ISSUE-MULTIPLE-PROJECTS-FILTER: this logic is not right, it should use "AND" but not "OR"
|
||||
queries = append(queries, bleve.NewDisjunctionQuery(projectQueries...))
|
||||
}
|
||||
|
||||
if options.PosterID != "" {
|
||||
@@ -254,14 +257,15 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
|
||||
queries = append(queries, inner_bleve.NumericEqualityQuery(posterIDInt64, "poster_id"))
|
||||
}
|
||||
|
||||
if options.AssigneeID != "" {
|
||||
if options.AssigneeID == "(any)" {
|
||||
queries = append(queries, inner_bleve.NumericRangeInclusiveQuery(optional.Some[int64](1), optional.None[int64](), "assignee_id"))
|
||||
} else {
|
||||
// "(none)" becomes 0, it means no assignee
|
||||
assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64)
|
||||
queries = append(queries, inner_bleve.NumericEqualityQuery(assigneeIDInt64, "assignee_id"))
|
||||
}
|
||||
switch options.AssigneeID {
|
||||
case "":
|
||||
case "(any)":
|
||||
queries = append(queries, inner_bleve.BoolFieldQuery(false, "no_assignee"))
|
||||
case "(none)":
|
||||
queries = append(queries, inner_bleve.BoolFieldQuery(true, "no_assignee"))
|
||||
default:
|
||||
assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64)
|
||||
queries = append(queries, inner_bleve.NumericEqualityQuery(assigneeIDInt64, "assignee_ids"))
|
||||
}
|
||||
|
||||
if options.MentionID.Has() {
|
||||
|
||||
@@ -6,7 +6,11 @@ package bleve
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/modules/indexer/issues/internal/tests"
|
||||
"gitea.dev/modules/indexer/issues/internal"
|
||||
"gitea.dev/modules/indexer/issues/internal/tests"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBleveIndexer(t *testing.T) {
|
||||
@@ -16,3 +20,108 @@ func TestBleveIndexer(t *testing.T) {
|
||||
|
||||
tests.TestIndexer(t, indexer)
|
||||
}
|
||||
|
||||
func TestBleveIndexerNoAssignee(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
indexer := NewIndexer(dir)
|
||||
defer indexer.Close()
|
||||
|
||||
_, err := indexer.Init(t.Context())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, indexer.Index(t.Context(),
|
||||
&internal.IndexerData{ID: 1, Title: "assigned through assignee_ids", AssigneeIDs: []int64{2}},
|
||||
&internal.IndexerData{ID: 2, Title: "unassigned", NoAssignee: true},
|
||||
&internal.IndexerData{ID: 3, Title: "assigned through multiple assignee_ids", AssigneeIDs: []int64{3, 4}},
|
||||
))
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
opts *internal.SearchOptions
|
||||
expectedIDs []int64
|
||||
}{
|
||||
{
|
||||
name: "none",
|
||||
opts: &internal.SearchOptions{AssigneeID: "(none)"},
|
||||
expectedIDs: []int64{2},
|
||||
},
|
||||
{
|
||||
name: "any",
|
||||
opts: &internal.SearchOptions{AssigneeID: "(any)"},
|
||||
expectedIDs: []int64{1, 3},
|
||||
},
|
||||
{
|
||||
name: "specific",
|
||||
opts: &internal.SearchOptions{AssigneeID: "2"},
|
||||
expectedIDs: []int64{1},
|
||||
},
|
||||
{
|
||||
name: "specific first multi-assignee",
|
||||
opts: &internal.SearchOptions{AssigneeID: "3"},
|
||||
expectedIDs: []int64{3},
|
||||
},
|
||||
{
|
||||
name: "specific second multi-assignee",
|
||||
opts: &internal.SearchOptions{AssigneeID: "4"},
|
||||
expectedIDs: []int64{3},
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
result, err := indexer.Search(t.Context(), testCase.opts)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(testCase.expectedIDs)), result.Total)
|
||||
assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(result))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBleveIndexerTokenFilter(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
indexer := NewIndexer(dir)
|
||||
defer indexer.Close()
|
||||
|
||||
_, err := indexer.Init(t.Context())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, indexer.Index(t.Context(),
|
||||
&internal.IndexerData{ID: 1, Title: "fix(packages): SomeThing needs a rewrite (#12345)"},
|
||||
&internal.IndexerData{ID: 2, Title: "add support for mDNS discovery abc1234"},
|
||||
))
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
keyword string
|
||||
expectedIDs []int64
|
||||
}{
|
||||
{name: "exact original case", keyword: "SomeThing", expectedIDs: []int64{1}},
|
||||
{name: "case matching original transitions", keyword: "someThing", expectedIDs: []int64{1}},
|
||||
{name: "all lower case", keyword: "something", expectedIDs: []int64{1}},
|
||||
{name: "all upper case", keyword: "SOMETHING", expectedIDs: []int64{1}},
|
||||
{name: "number match", keyword: "12345", expectedIDs: []int64{1}},
|
||||
{name: "number as part", keyword: "1234", expectedIDs: []int64{2}},
|
||||
{name: "sub-word search still works", keyword: "DNS", expectedIDs: []int64{2}},
|
||||
{name: "sub-word search, lower case", keyword: "mdns", expectedIDs: []int64{2}},
|
||||
{name: "keyword is camel case", keyword: "addSupport", expectedIDs: []int64{2}},
|
||||
{name: "keyword not match", keyword: "addsupport", expectedIDs: []int64{}},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
result, err := indexer.Search(t.Context(), &internal.SearchOptions{
|
||||
Keyword: testCase.keyword,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(result))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func searchResultIDs(result *internal.SearchResult) []int64 {
|
||||
ids := make([]int64, 0, len(result.Hits))
|
||||
for _, hit := range result.Hits {
|
||||
ids = append(ids, hit.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/analysis"
|
||||
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
|
||||
"github.com/blevesearch/bleve/v2/registry"
|
||||
)
|
||||
|
||||
const camelCaseKeepWholeName = "camelCaseKeepWhole"
|
||||
|
||||
// camelCaseKeepWholeFilter behaves like bleve's built-in "camelCase" token filter,
|
||||
// it also uses the whole word for a token. For example: when indexing "someThing",
|
||||
// CamelCaseFilter only emits "some" and "thing", this filter also emits "something".
|
||||
// It is questionable why the "issue indexer" used the CamelCaseFilter, it just can't search "someThing".
|
||||
// To avoid breaking existing user experiences, this "whole token filter" is introduced to make the full word can be searched.
|
||||
type camelCaseKeepWholeFilter struct {
|
||||
inner *camelcase.CamelCaseFilter
|
||||
}
|
||||
|
||||
func (f *camelCaseKeepWholeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
|
||||
// First, do exactly what the stock camelCase filter does: split by "camelCase" tokens
|
||||
split := f.inner.Filter(input)
|
||||
|
||||
// Index the resulting position of the *first* sub-token produced for
|
||||
// each original token (matched by start offset), so the duplicated
|
||||
// whole-word token we add below lines up at the same position as the
|
||||
// sub-word it stands in for, instead of drifting out of sync for
|
||||
// fields with more than one original token.
|
||||
posByStart := make(map[int]int, len(split))
|
||||
for _, tok := range split {
|
||||
if _, ok := posByStart[tok.Start]; !ok {
|
||||
posByStart[tok.Start] = tok.Position
|
||||
}
|
||||
}
|
||||
|
||||
rv := make(analysis.TokenStream, 0, len(split)+len(input))
|
||||
rv = append(rv, split...)
|
||||
|
||||
// Then append one extra, un-split copy of every original token, so the
|
||||
// whole word survives as a standalone, independently searchable term.
|
||||
for _, token := range input {
|
||||
dup := *token
|
||||
dup.Term = append([]byte(nil), token.Term...)
|
||||
if pos, ok := posByStart[token.Start]; ok {
|
||||
dup.Position = pos
|
||||
}
|
||||
rv = append(rv, &dup)
|
||||
}
|
||||
|
||||
return rv
|
||||
}
|
||||
|
||||
func camelCaseKeepWholeFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
|
||||
return &camelCaseKeepWholeFilter{inner: camelcase.NewCamelCaseFilter()}, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
util.MustNoError(registry.RegisterTokenFilter(camelCaseKeepWholeName, camelCaseKeepWholeFilterConstructor))
|
||||
}
|
||||
Reference in New Issue
Block a user