From 09508f7fca763c9436782f78cf0c6e14d534d5ae Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 28 Apr 2024 23:04:17 -0400 Subject: [PATCH 01/21] Add elastic 8 support via `github.com/elastic/go-elasticsearch/v8` --- go.mod | 7 +- go.sum | 17 +- .../code/elasticsearch/elasticsearch.go | 251 ++++++++------ .../indexer/internal/elasticsearch/indexer.go | 15 +- .../indexer/internal/elasticsearch/util.go | 40 ++- .../issues/elasticsearch/elasticsearch.go | 324 ++++++++++++------ 6 files changed, 408 insertions(+), 246 deletions(-) diff --git a/go.mod b/go.mod index 2c1fc5d6f2046..6339bc14232e3 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 github.com/dustin/go-humanize v1.0.1 github.com/editorconfig/editorconfig-core-go/v2 v2.6.1 + github.com/elastic/go-elasticsearch/v8 v8.13.1 github.com/emersion/go-imap v1.2.1 github.com/emirpasic/gods v1.18.1 github.com/ethantkoenig/rupture v1.0.1 @@ -81,7 +82,6 @@ require ( github.com/msteinert/pam v1.2.0 github.com/nektos/act v0.2.52 github.com/niklasfasching/go-org v1.7.0 - github.com/olivere/elastic/v7 v7.0.32 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0 github.com/pkg/errors v0.9.1 @@ -173,6 +173,7 @@ require ( github.com/davidmz/go-pageant v1.0.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.11.0 // indirect + github.com/elastic/elastic-transport-go/v8 v8.5.0 // indirect github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43 // indirect github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -183,6 +184,8 @@ require ( github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.23.0 // indirect github.com/go-openapi/errors v0.22.0 // indirect github.com/go-openapi/inflect v0.21.0 // indirect @@ -261,6 +264,7 @@ require ( github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect + github.com/smartystreets/assertions v1.1.1 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect @@ -283,6 +287,7 @@ require ( go.etcd.io/bbolt v1.3.9 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index 8c26b4a7a6394..a1cbbaa774846 100644 --- a/go.sum +++ b/go.sum @@ -236,6 +236,10 @@ github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJ github.com/editorconfig/editorconfig-core-go/v2 v2.6.1 h1:iPCqofzMO41WVbcS/B5Ym7AwHQg9cyQ7Ie/R2XU5L3A= github.com/editorconfig/editorconfig-core-go/v2 v2.6.1/go.mod h1:VY4oyqUnpULFB3SCRpl24GFDIN1PmfiQIvN/G4ScSNg= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= +github.com/elastic/elastic-transport-go/v8 v8.5.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= +github.com/elastic/go-elasticsearch/v8 v8.13.1 h1:du5F8IzUUyCkzxyHdrO9AtopcG95I/qwi2WK8Kf1xlg= +github.com/elastic/go-elasticsearch/v8 v8.13.1/go.mod h1:DIn7HopJs4oZC/w0WoJR13uMUxtHeq92eI5bqv5CRfI= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= @@ -256,8 +260,6 @@ github.com/felixge/fgprof v0.9.4 h1:ocDNwMFlnA0NU0zSB3I52xkO4sFXk80VK9lXjLClu88= github.com/felixge/fgprof v0.9.4/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -306,6 +308,11 @@ github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3c github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A= github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= @@ -603,8 +610,6 @@ github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= -github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -829,6 +834,10 @@ go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index e4622fd66ef95..337554e983ea5 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -13,7 +13,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/analyze" - "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/indexer/code/internal" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" @@ -24,16 +23,19 @@ import ( "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/typesniffer" + "github.com/elastic/go-elasticsearch/v8/typedapi/core/bulk" + "github.com/elastic/go-elasticsearch/v8/typedapi/core/search" + "github.com/elastic/go-elasticsearch/v8/typedapi/some" + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/highlightertype" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/sortorder" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/termvectoroption" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/textquerytype" "github.com/go-enry/go-enry/v2" - "github.com/olivere/elastic/v7" ) const ( esRepoIndexerLatestVersion = 1 - // multi-match-types, currently only 2 types are used - // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types - esMultiMatchTypeBestFields = "best_fields" - esMultiMatchTypePhrasePrefix = "phrase_prefix" ) var _ internal.Indexer = &Indexer{} @@ -54,40 +56,27 @@ func NewIndexer(url, indexerName string) *Indexer { return indexer } -const ( - defaultMapping = `{ - "mappings": { - "properties": { - "repo_id": { - "type": "long", - "index": true - }, - "content": { - "type": "text", - "term_vector": "with_positions_offsets", - "index": true - }, - "commit_id": { - "type": "keyword", - "index": true - }, - "language": { - "type": "keyword", - "index": true - }, - "updated_at": { - "type": "long", - "index": true - } - } - } - }` +var ( + defaultMapping = &types.TypeMapping{ + Properties: map[string]types.Property{ + "repo_id": types.NewLongNumberProperty(), + "content": &types.TextProperty{ + Fields: make(map[string]types.Property, 0), + Meta: make(map[string]string, 0), + Properties: make(map[string]types.Property, 0), + TermVector: &termvectoroption.Withpositions, + }, + "commit_id": types.NewKeywordProperty(), + "language": types.NewKeywordProperty(), + "updated_at": types.NewLongNumberProperty(), + }, + } ) -func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { +func (b *Indexer) addUpdate(ctx context.Context, blk *bulk.Bulk, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) error { // Ignore vendored files in code search if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { - return nil, nil + return nil } size := update.Size @@ -96,63 +85,68 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro var stdout string stdout, _, err = git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) if err != nil { - return nil, err + return err } if size, err = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64); err != nil { - return nil, fmt.Errorf("misformatted git cat-file output: %w", err) + return fmt.Errorf("misformatted git cat-file output: %w", err) } } if size > setting.Indexer.MaxIndexerFileSize { - return []elastic.BulkableRequest{b.addDelete(update.Filename, repo)}, nil + return b.addDelete(blk, update.Filename, repo) } if _, err := batchWriter.Write([]byte(update.BlobSha + "\n")); err != nil { - return nil, err + return err } _, _, size, err = git.ReadBatchLine(batchReader) if err != nil { - return nil, err + return err } fileContents, err := io.ReadAll(io.LimitReader(batchReader, size)) if err != nil { - return nil, err + return err } else if !typesniffer.DetectContentType(fileContents).IsText() { // FIXME: UTF-16 files will probably fail here - return nil, nil + return nil } if _, err = batchReader.Discard(1); err != nil { - return nil, err + return err } id := internal.FilenameIndexerID(repo.ID, update.Filename) - return []elastic.BulkableRequest{ - elastic.NewBulkIndexRequest(). - Index(b.inner.VersionedIndexName()). - Id(id). - Doc(map[string]any{ - "repo_id": repo.ID, - "content": string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})), - "commit_id": sha, - "language": analyze.GetCodeLanguage(update.Filename, fileContents), - "updated_at": timeutil.TimeStampNow(), - }), - }, nil + return blk.IndexOp(types.IndexOperation{ + Index_: some.String(b.inner.VersionedIndexName()), + Id_: some.String(id), + }, map[string]any{ + "id": id, + "repo_id": repo.ID, + "content": string(fileContents), + "commit_id": sha, + "language": analyze.GetCodeLanguage(update.Filename, fileContents), + "updated_at": timeutil.TimeStampNow(), + }) } -func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elastic.BulkableRequest { +func (b *Indexer) addDelete(blk *bulk.Bulk, filename string, repo *repo_model.Repository) error { id := internal.FilenameIndexerID(repo.ID, filename) - return elastic.NewBulkDeleteRequest(). - Index(b.inner.VersionedIndexName()). - Id(id) + return blk.DeleteOp(types.DeleteOperation{ + Index_: some.String(b.inner.VersionedIndexName()), + Id_: some.String(id), + }) } // Index will save the index data func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { - reqs := make([]elastic.BulkableRequest, 0) + if len(changes.Updates) == 0 && len(changes.RemovedFilenames) == 0 { + return nil + } + + blk := b.inner.Client.Bulk().Index(b.inner.VersionedIndexName()) + if len(changes.Updates) > 0 { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! if err := git.EnsureValidGitRepository(ctx, repo.RepoPath()); err != nil { @@ -164,41 +158,34 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st defer cancel() for _, update := range changes.Updates { - updateReqs, err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo) + err := b.addUpdate(ctx, blk, batchWriter, batchReader, sha, update, repo) if err != nil { return err } - if len(updateReqs) > 0 { - reqs = append(reqs, updateReqs...) - } } cancel() } for _, filename := range changes.RemovedFilenames { - reqs = append(reqs, b.addDelete(filename, repo)) - } - - if len(reqs) > 0 { - esBatchSize := 50 - - for i := 0; i < len(reqs); i += esBatchSize { - _, err := b.inner.Client.Bulk(). - Index(b.inner.VersionedIndexName()). - Add(reqs[i:min(i+esBatchSize, len(reqs))]...). - Do(ctx) - if err != nil { - return err - } + err := b.addDelete(blk, filename, repo) + if err != nil { + return err } } - return nil + + _, err := blk.Do(ctx) + return err } // Delete deletes indexes by ids func (b *Indexer) Delete(ctx context.Context, repoID int64) error { _, err := b.inner.Client.DeleteByQuery(b.inner.VersionedIndexName()). - Query(elastic.NewTermsQuery("repo_id", repoID)). + Query(&types.Query{ + Term: map[string]types.TermQuery{ + "repo_id": {Value: repoID}, + }, + }). + // Query(elastic.NewTermsQuery("repo_id", repoID)). Do(ctx) return err } @@ -219,7 +206,7 @@ func indexPos(content, start, end string) (int, int) { return startIdx, startIdx + len(start) + endIdx + len(end) } -func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { +func convertResult(searchResult *search.Response, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { hits := make([]*internal.SearchResult, 0, pageSize) for _, hit := range searchResult.Hits.Hits { // FIXME: There is no way to get the position the keyword on the content currently on the same request. @@ -239,9 +226,9 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) panic(fmt.Sprintf("2===%#v", hit.Highlight)) } - repoID, fileName := internal.ParseIndexerID(hit.Id) + repoID, fileName := internal.ParseIndexerID(hit.Id_) res := make(map[string]any) - if err := json.Unmarshal(hit.Source, &res); err != nil { + if err := json.Unmarshal(hit.Source_, &res); err != nil { return 0, nil, nil, err } @@ -260,16 +247,18 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) }) } - return searchResult.TotalHits(), hits, extractAggs(searchResult), nil + return searchResult.Hits.Total.Value, hits, extractAggregates(searchResult), nil } -func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLanguages { +func extractAggregates(searchResult *search.Response) []*internal.SearchResultLanguages { var searchResultLanguages []*internal.SearchResultLanguages - agg, found := searchResult.Aggregations.Terms("language") + agg, found := searchResult.Aggregations["language"] if found { searchResultLanguages = make([]*internal.SearchResultLanguages, 0, 10) - for _, bucket := range agg.Buckets { + languageAgg := agg.(*types.StringTermsAggregate) + buckets := languageAgg.Buckets.([]types.StringTermsBucket) + for _, bucket := range buckets { searchResultLanguages = append(searchResultLanguages, &internal.SearchResultLanguages{ Language: bucket.Key.(string), Color: enry.GetColor(bucket.Key.(string)), @@ -282,39 +271,69 @@ func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLan // Search searches for codes and language stats by given conditions. func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { - searchType := esMultiMatchTypePhrasePrefix + // searchType := esMultiMatchTypePhrasePrefix + searchType := &textquerytype.Phraseprefix if opts.IsKeywordFuzzy { - searchType = esMultiMatchTypeBestFields + searchType = &textquerytype.Bestfields } - kwQuery := elastic.NewMultiMatchQuery(opts.Keyword, "content").Type(searchType) - query := elastic.NewBoolQuery() - query = query.Must(kwQuery) + kwQuery := types.Query{ + MultiMatch: &types.MultiMatchQuery{ + Query: opts.Keyword, + Fields: []string{"content"}, + Type: searchType, + }, + } + query := &types.Query{ + Bool: &types.BoolQuery{ + Must: []types.Query{kwQuery}, + }, + } if len(opts.RepoIDs) > 0 { - repoStrs := make([]any, 0, len(opts.RepoIDs)) + repoIDs := make([]types.FieldValue, 0, len(opts.RepoIDs)) for _, repoID := range opts.RepoIDs { - repoStrs = append(repoStrs, repoID) + repoIDs = append(repoIDs, types.FieldValue(repoID)) + } + repoQuery := types.Query{ + Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{ + "repo_id": repoIDs, + }, + }, } - repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...) - query = query.Must(repoQuery) + query.Bool.Must = append(query.Bool.Must, repoQuery) } var ( start, pageSize = opts.GetSkipTake() kw = "" + opts.Keyword + "" - aggregation = elastic.NewTermsAggregation().Field("language").Size(10).OrderByCountDesc() + aggregation = map[string]types.Aggregations{ + "language": { + Terms: &types.TermsAggregation{ + Field: some.String("language"), + Size: some.Int(10), + Order: map[string]sortorder.SortOrder{ + "_count": sortorder.Desc, + }, + }, + }, + } ) if len(opts.Language) == 0 { searchResult, err := b.inner.Client.Search(). Index(b.inner.VersionedIndexName()). - Aggregation("language", aggregation). + Aggregations(aggregation). Query(query). Highlight( - elastic.NewHighlight(). - Field("content"). - NumOfFragments(0). // return all highting content on fragments - HighlighterType("fvh"), + &types.Highlight{ + Fields: map[string]types.HighlightField{ + "content": { + NumberOfFragments: some.Int(0), // return all highting content on fragments + Type: &highlightertype.Fvh, + }, + }, + }, ). Sort("repo_id", true). From(start).Size(pageSize). @@ -326,10 +345,16 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int return convertResult(searchResult, kw, pageSize) } - langQuery := elastic.NewMatchQuery("language", opts.Language) + langQuery := types.Query{ + Match: map[string]types.MatchQuery{ + "language": { + Query: opts.Language, + }, + }, + } countResult, err := b.inner.Client.Search(). Index(b.inner.VersionedIndexName()). - Aggregation("language", aggregation). + Aggregations(aggregation). Query(query). Size(0). // We only need stats information Do(ctx) @@ -337,15 +362,19 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int return 0, nil, nil, err } - query = query.Must(langQuery) + query.Bool.Must = append(query.Bool.Must, langQuery) searchResult, err := b.inner.Client.Search(). Index(b.inner.VersionedIndexName()). Query(query). Highlight( - elastic.NewHighlight(). - Field("content"). - NumOfFragments(0). // return all highting content on fragments - HighlighterType("fvh"), + &types.Highlight{ + Fields: map[string]types.HighlightField{ + "content": { + NumberOfFragments: some.Int(0), // return all highting content on fragments + Type: &highlightertype.Fvh, + }, + }, + }, ). Sort("repo_id", true). From(start).Size(pageSize). @@ -356,5 +385,5 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int total, hits, _, err := convertResult(searchResult, kw, pageSize) - return total, hits, extractAggs(countResult), err + return total, hits, extractAggregates(countResult), err } diff --git a/modules/indexer/internal/elasticsearch/indexer.go b/modules/indexer/internal/elasticsearch/indexer.go index 395eea3bce652..a4233e88eefd1 100644 --- a/modules/indexer/internal/elasticsearch/indexer.go +++ b/modules/indexer/internal/elasticsearch/indexer.go @@ -9,22 +9,23 @@ import ( "code.gitea.io/gitea/modules/indexer/internal" - "github.com/olivere/elastic/v7" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/typedapi/types" ) var _ internal.Indexer = &Indexer{} // Indexer represents a basic elasticsearch indexer implementation type Indexer struct { - Client *elastic.Client + Client *elasticsearch.TypedClient url string indexName string version int - mapping string + mapping *types.TypeMapping } -func NewIndexer(url, indexName string, version int, mapping string) *Indexer { +func NewIndexer(url, indexName string, version int, mapping *types.TypeMapping) *Indexer { return &Indexer{ url: url, indexName: indexName, @@ -48,7 +49,7 @@ func (i *Indexer) Init(ctx context.Context) (bool, error) { } i.Client = client - exists, err := i.Client.IndexExists(i.VersionedIndexName()).Do(ctx) + exists, err := i.Client.Indices.Exists(i.VersionedIndexName()).Do(ctx) if err != nil { return false, err } @@ -72,11 +73,11 @@ func (i *Indexer) Ping(ctx context.Context) error { return fmt.Errorf("indexer is not initialized") } - resp, err := i.Client.ClusterHealth().Do(ctx) + resp, err := i.Client.Cluster.Health().Do(ctx) if err != nil { return err } - if resp.Status != "green" && resp.Status != "yellow" { + if resp.Status.Name != "green" && resp.Status.Name != "yellow" { // It's healthy if the status is green, and it's available if the status is yellow, // see https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html return fmt.Errorf("status of elasticsearch cluster is %s", resp.Status) diff --git a/modules/indexer/internal/elasticsearch/util.go b/modules/indexer/internal/elasticsearch/util.go index 9e034bd553095..cd8f5d557bed7 100644 --- a/modules/indexer/internal/elasticsearch/util.go +++ b/modules/indexer/internal/elasticsearch/util.go @@ -6,11 +6,11 @@ package elasticsearch import ( "context" "fmt" - "time" "code.gitea.io/gitea/modules/log" - "github.com/olivere/elastic/v7" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/typedapi/indices/create" ) // VersionedIndexName returns the full index name with version @@ -27,12 +27,16 @@ func versionedIndexName(indexName string, version int) string { } func (i *Indexer) createIndex(ctx context.Context) error { - createIndex, err := i.Client.CreateIndex(i.VersionedIndexName()).BodyString(i.mapping).Do(ctx) + createIndex, err := i.Client.Indices.Create(i.VersionedIndexName()).Request(&create.Request{ + Mappings: i.mapping, + }).Do(ctx) + + // .BodyString(i.mapping) if err != nil { return err } if !createIndex.Acknowledged { - return fmt.Errorf("create index %s with %s failed", i.VersionedIndexName(), i.mapping) + return fmt.Errorf("create index %s failed", i.VersionedIndexName()) } i.checkOldIndexes(ctx) @@ -40,27 +44,31 @@ func (i *Indexer) createIndex(ctx context.Context) error { return nil } -func (i *Indexer) initClient() (*elastic.Client, error) { - opts := []elastic.ClientOptionFunc{ - elastic.SetURL(i.url), - elastic.SetSniff(false), - elastic.SetHealthcheckInterval(10 * time.Second), - elastic.SetGzip(false), +func (i *Indexer) initClient() (*elasticsearch.TypedClient, error) { + // opts := []elastic.ClientOptionFunc{ + // elastic.SetURL(i.url), + // elastic.SetSniff(false), + // elastic.SetHealthcheckInterval(10 * time.Second), + // elastic.SetGzip(false), + // } + + cfg := elasticsearch.Config{ + Addresses: []string{i.url}, } - logger := log.GetLogger(log.DEFAULT) + // logger := log.GetLogger(log.DEFAULT) - opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace})) - opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info})) - opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error})) + // opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace})) + // opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info})) + // opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error})) - return elastic.NewClient(opts...) + return elasticsearch.NewTypedClient(cfg) } func (i *Indexer) checkOldIndexes(ctx context.Context) { for v := 0; v < i.version; v++ { indexName := versionedIndexName(i.indexName, v) - exists, err := i.Client.IndexExists(indexName).Do(ctx) + exists, err := i.Client.Indices.Exists(indexName).Do(ctx) if err == nil && exists { log.Warn("Found older elasticsearch index named %q, Gitea will keep the old NOT DELETED. You can delete the old version after the upgrade succeed.", indexName) } diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index c7cb59f2cf009..67327671916e7 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -4,7 +4,9 @@ package elasticsearch import ( + "bytes" "context" + "encoding/json" "fmt" "strconv" "strings" @@ -14,15 +16,17 @@ import ( inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" "code.gitea.io/gitea/modules/indexer/issues/internal" - "github.com/olivere/elastic/v7" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esutil" + "github.com/elastic/go-elasticsearch/v8/typedapi/core/bulk" + "github.com/elastic/go-elasticsearch/v8/typedapi/some" + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/sortorder" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/textquerytype" ) const ( issueIndexerLatestVersion = 1 - // multi-match-types, currently only 2 types are used - // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types - esMultiMatchTypeBestFields = "best_fields" - esMultiMatchTypePhrasePrefix = "phrase_prefix" ) var _ internal.Indexer = &Indexer{} @@ -43,41 +47,36 @@ func NewIndexer(url, indexerName string) *Indexer { return indexer } -const ( - defaultMapping = ` -{ - "mappings": { - "properties": { - "id": { "type": "integer", "index": true }, - "repo_id": { "type": "integer", "index": true }, - "is_public": { "type": "boolean", "index": true }, - - "title": { "type": "text", "index": true }, - "content": { "type": "text", "index": true }, - "comments": { "type" : "text", "index": true }, - - "is_pull": { "type": "boolean", "index": true }, - "is_closed": { "type": "boolean", "index": true }, - "label_ids": { "type": "integer", "index": true }, - "no_label": { "type": "boolean", "index": true }, - "milestone_id": { "type": "integer", "index": true }, - "project_id": { "type": "integer", "index": true }, - "project_board_id": { "type": "integer", "index": true }, - "poster_id": { "type": "integer", "index": true }, - "assignee_id": { "type": "integer", "index": true }, - "mention_ids": { "type": "integer", "index": true }, - "reviewed_ids": { "type": "integer", "index": true }, - "review_requested_ids": { "type": "integer", "index": true }, - "subscriber_ids": { "type": "integer", "index": true }, - "updated_unix": { "type": "integer", "index": true }, - - "created_unix": { "type": "integer", "index": true }, - "deadline_unix": { "type": "integer", "index": true }, - "comment_count": { "type": "integer", "index": true } - } +var ( + defaultMapping = &types.TypeMapping{ + Properties: map[string]types.Property{ + "id": types.NewIntegerNumberProperty(), + "repo_id": types.NewIntegerNumberProperty(), + "is_public": types.NewBooleanProperty(), + + "title": types.NewTextProperty(), + "content": types.NewTextProperty(), + "comments": types.NewTextProperty(), + + "is_pull": types.NewBooleanProperty(), + "is_closed": types.NewBooleanProperty(), + "label_ids": types.NewIntegerNumberProperty(), + "no_label": types.NewBooleanProperty(), + "milestone_id": types.NewIntegerNumberProperty(), + "project_id": types.NewIntegerNumberProperty(), + "project_board_id": types.NewIntegerNumberProperty(), + "poster_id": types.NewIntegerNumberProperty(), + "assignee_id": types.NewIntegerNumberProperty(), + "mention_ids": types.NewIntegerNumberProperty(), + "reviewed_ids": types.NewIntegerNumberProperty(), + "review_requested_ids": types.NewIntegerNumberProperty(), + "subscriber_ids": types.NewIntegerNumberProperty(), + "updated_unix": types.NewIntegerNumberProperty(), + "created_unix": types.NewIntegerNumberProperty(), + "deadline_unix": types.NewIntegerNumberProperty(), + "comment_count": types.NewIntegerNumberProperty(), + }, } -} -` ) // Index will save the index data @@ -86,27 +85,27 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er return nil } else if len(issues) == 1 { issue := issues[0] - _, err := b.inner.Client.Index(). - Index(b.inner.VersionedIndexName()). + + raw, err := json.Marshal(issue) + if err != nil { + return err + } + + _, err = b.inner.Client.Index(b.inner.VersionedIndexName()). Id(fmt.Sprintf("%d", issue.ID)). - BodyJson(issue). + Raw(bytes.NewBuffer(raw)). Do(ctx) return err } - reqs := make([]elastic.BulkableRequest, 0) + reqs := make(bulk.Request, 0) for _, issue := range issues { - reqs = append(reqs, - elastic.NewBulkIndexRequest(). - Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", issue.ID)). - Doc(issue), - ) + reqs = append(reqs, issue) } _, err := b.inner.Client.Bulk(). Index(b.inner.VersionedIndexName()). - Add(reqs...). + Request(&reqs). Do(graceful.GetManager().HammerContext()) return err } @@ -115,132 +114,240 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { if len(ids) == 0 { return nil - } else if len(ids) == 1 { - _, err := b.inner.Client.Delete(). - Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", ids[0])). - Do(ctx) + } + if len(ids) == 1 { + _, err := b.inner.Client.Delete( + b.inner.VersionedIndexName(), + fmt.Sprintf("%d", ids[0]), + ).Do(ctx) + return err + } + + bulkIndexer, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{ + Client: &elasticsearch.Client{ + BaseClient: elasticsearch.BaseClient{ + Transport: b.inner.Client.Transport, + }, + }, + Index: b.inner.VersionedIndexName(), + }) + if err != nil { return err } - reqs := make([]elastic.BulkableRequest, 0) for _, id := range ids { - reqs = append(reqs, - elastic.NewBulkDeleteRequest(). - Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", id)), - ) + bulkIndexer.Add(ctx, esutil.BulkIndexerItem{ + Action: "delete", + Index: b.inner.VersionedIndexName(), + DocumentID: fmt.Sprintf("%d", id), + }) } - _, err := b.inner.Client.Bulk(). - Index(b.inner.VersionedIndexName()). - Add(reqs...). - Do(graceful.GetManager().HammerContext()) - return err + if err := bulkIndexer.Close(context.Background()); err != nil { + return err + } + return nil } // Search searches for issues by given conditions. // Returns the matching issue IDs func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { - query := elastic.NewBoolQuery() + query := &types.Query{ + Bool: &types.BoolQuery{ + Must: make([]types.Query, 0), + }, + } if options.Keyword != "" { - searchType := esMultiMatchTypePhrasePrefix + searchType := &textquerytype.Phraseprefix if options.IsFuzzyKeyword { - searchType = esMultiMatchTypeBestFields + searchType = &textquerytype.Bestfields } - query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(searchType)) + query.Bool.Must = append(query.Bool.Must, types.Query{ + MultiMatch: &types.MultiMatchQuery{ + Query: options.Keyword, + Fields: []string{"title", "content", "comments"}, + Type: searchType, + }, + }) } if len(options.RepoIDs) > 0 { - q := elastic.NewBoolQuery() - q.Should(elastic.NewTermsQuery("repo_id", toAnySlice(options.RepoIDs)...)) + q := types.Query{ + Bool: &types.BoolQuery{ + Should: make([]types.Query, 0), + }, + } if options.AllPublic { - q.Should(elastic.NewTermQuery("is_public", true)) + q.Bool.Should = append(q.Bool.Should, types.Query{ + Term: map[string]types.TermQuery{ + "is_public": {Value: true}, + }, + }) } - query.Must(q) + query.Bool.Must = append(query.Bool.Must, q) } if options.IsPull.Has() { - query.Must(elastic.NewTermQuery("is_pull", options.IsPull.Value())) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "is_pull": {Value: options.IsPull.Value()}, + }, + }) } if options.IsClosed.Has() { - query.Must(elastic.NewTermQuery("is_closed", options.IsClosed.Value())) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "is_closed": {Value: options.IsClosed.Value()}, + }, + }) } if options.NoLabelOnly { - query.Must(elastic.NewTermQuery("no_label", true)) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "no_label": {Value: true}, + }, + }) } else { if len(options.IncludedLabelIDs) > 0 { - q := elastic.NewBoolQuery() + q := types.Query{ + Bool: &types.BoolQuery{ + Must: make([]types.Query, 0), + }, + } for _, labelID := range options.IncludedLabelIDs { - q.Must(elastic.NewTermQuery("label_ids", labelID)) + q.Bool.Must = append(q.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "label_ids": {Value: labelID}, + }, + }) } - query.Must(q) + query.Bool.Must = append(query.Bool.Must, q) } else if len(options.IncludedAnyLabelIDs) > 0 { - query.Must(elastic.NewTermsQuery("label_ids", toAnySlice(options.IncludedAnyLabelIDs)...)) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{ + "label_ids": toAnySlice(options.IncludedAnyLabelIDs), + }, + }, + }) } if len(options.ExcludedLabelIDs) > 0 { - q := elastic.NewBoolQuery() + q := types.Query{ + Bool: &types.BoolQuery{ + MustNot: make([]types.Query, 0), + }, + } for _, labelID := range options.ExcludedLabelIDs { - q.MustNot(elastic.NewTermQuery("label_ids", labelID)) + q.Bool.MustNot = append(q.Bool.MustNot, types.Query{ + Term: map[string]types.TermQuery{ + "label_ids": {Value: labelID}, + }, + }) } - query.Must(q) + query.Bool.Must = append(query.Bool.Must, q) } } if len(options.MilestoneIDs) > 0 { - query.Must(elastic.NewTermsQuery("milestone_id", toAnySlice(options.MilestoneIDs)...)) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{ + "milestone_id": toAnySlice(options.MilestoneIDs), + }, + }, + }) } if options.ProjectID.Has() { - query.Must(elastic.NewTermQuery("project_id", options.ProjectID.Value())) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "project_id": {Value: options.ProjectID.Value()}, + }, + }) } if options.ProjectBoardID.Has() { - query.Must(elastic.NewTermQuery("project_board_id", options.ProjectBoardID.Value())) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "project_board_id": {Value: options.ProjectBoardID.Value()}, + }, + }) } if options.PosterID.Has() { - query.Must(elastic.NewTermQuery("poster_id", options.PosterID.Value())) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "poster_id": {Value: options.PosterID.Value()}, + }, + }) } if options.AssigneeID.Has() { - query.Must(elastic.NewTermQuery("assignee_id", options.AssigneeID.Value())) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "assignee_id": {Value: options.AssigneeID.Value()}, + }, + }) } if options.MentionID.Has() { - query.Must(elastic.NewTermQuery("mention_ids", options.MentionID.Value())) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "mention_ids": {Value: options.MentionID.Value()}, + }, + }) } if options.ReviewedID.Has() { - query.Must(elastic.NewTermQuery("reviewed_ids", options.ReviewedID.Value())) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "reviewed_ids": {Value: options.ReviewedID.Value()}, + }, + }) } + if options.ReviewRequestedID.Has() { - query.Must(elastic.NewTermQuery("review_requested_ids", options.ReviewRequestedID.Value())) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "review_requested_ids": {Value: options.ReviewRequestedID.Value()}, + }, + }) } if options.SubscriberID.Has() { - query.Must(elastic.NewTermQuery("subscriber_ids", options.SubscriberID.Value())) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Term: map[string]types.TermQuery{ + "subscriber_ids": {Value: options.SubscriberID.Value()}, + }, + }) } if options.UpdatedAfterUnix.Has() || options.UpdatedBeforeUnix.Has() { - q := elastic.NewRangeQuery("updated_unix") + rangeQuery := types.NumberRangeQuery{} if options.UpdatedAfterUnix.Has() { - q.Gte(options.UpdatedAfterUnix.Value()) + rangeQuery.Gte = some.Float64(float64(options.UpdatedAfterUnix.Value())) } if options.UpdatedBeforeUnix.Has() { - q.Lte(options.UpdatedBeforeUnix.Value()) + rangeQuery.Lte = some.Float64(float64(options.UpdatedBeforeUnix.Value())) } - query.Must(q) + query.Bool.Must = append(query.Bool.Must, types.Query{ + Range: map[string]types.RangeQuery{ + "updated_unix": rangeQuery, + }, + }) } if options.SortBy == "" { options.SortBy = internal.SortByCreatedAsc } - sortBy := []elastic.Sorter{ - parseSortBy(options.SortBy), - elastic.NewFieldSort("id").Desc(), + field, fieldSort := parseSortBy(options.SortBy) + sort := []types.SortCombinations{ + &types.SortOptions{SortOptions: map[string]types.FieldSort{ + field: fieldSort, + "id": {Order: &sortorder.Desc}, + }}, } // See https://stackoverflow.com/questions/35206409/elasticsearch-2-1-result-window-is-too-large-index-max-result-window/35221900 @@ -251,7 +358,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( searchResult, err := b.inner.Client.Search(). Index(b.inner.VersionedIndexName()). Query(query). - SortBy(sortBy...). + Sort(sort...). From(skip).Size(limit). Do(ctx) if err != nil { @@ -260,14 +367,14 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( hits := make([]internal.Match, 0, limit) for _, hit := range searchResult.Hits.Hits { - id, _ := strconv.ParseInt(hit.Id, 10, 64) + id, _ := strconv.ParseInt(hit.Id_, 10, 64) hits = append(hits, internal.Match{ ID: id, }) } return &internal.SearchResult{ - Total: searchResult.TotalHits(), + Total: searchResult.Hits.Total.Value, Hits: hits, }, nil } @@ -280,11 +387,14 @@ func toAnySlice[T any](s []T) []any { return ret } -func parseSortBy(sortBy internal.SortBy) elastic.Sorter { +func parseSortBy(sortBy internal.SortBy) (string, types.FieldSort) { field := strings.TrimPrefix(string(sortBy), "-") - ret := elastic.NewFieldSort(field) + sort := types.FieldSort{ + Order: &sortorder.Asc, + } if strings.HasPrefix(string(sortBy), "-") { - ret.Desc() + sort.Order = &sortorder.Desc } - return ret + + return field, sort } From 4b107db88383a71ea4a0d30b34f319e649206aa5 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 28 Apr 2024 23:14:36 -0400 Subject: [PATCH 02/21] Run make tidy --- assets/go-licenses.json | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index b8905da284c92..8e7c77d897672 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -339,6 +339,16 @@ "path": "github.com/editorconfig/editorconfig-core-go/v2/LICENSE", "licenseText": "MIT License\nCopyright (c) 2016 The Editorconfig Team\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "github.com/elastic/elastic-transport-go/v8/elastictransport", + "path": "github.com/elastic/elastic-transport-go/v8/elastictransport/LICENSE", + "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, + { + "name": "github.com/elastic/go-elasticsearch/v8", + "path": "github.com/elastic/go-elasticsearch/v8/LICENSE", + "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2018 Elasticsearch BV\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, { "name": "github.com/emersion/go-imap", "path": "github.com/emersion/go-imap/LICENSE", @@ -459,6 +469,16 @@ "path": "github.com/go-ldap/ldap/v3/LICENSE", "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com)\nPortions copyright (c) 2015-2016 go-ldap Authors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, + { + "name": "github.com/go-logr/logr", + "path": "github.com/go-logr/logr/LICENSE", + "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, + { + "name": "github.com/go-logr/stdr", + "path": "github.com/go-logr/stdr/LICENSE", + "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, { "name": "github.com/go-sql-driver/mysql", "path": "github.com/go-sql-driver/mysql/LICENSE", @@ -819,16 +839,6 @@ "path": "github.com/olekukonko/tablewriter/LICENSE.md", "licenseText": "Copyright (C) 2014 by Oleku Konko\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n" }, - { - "name": "github.com/olivere/elastic/v7", - "path": "github.com/olivere/elastic/v7/LICENSE", - "licenseText": "The MIT License (MIT)\nCopyright © 2012-2015 Oliver Eilhard\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the “Software”), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included\nin all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\nFROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\nIN THE SOFTWARE.\n" - }, - { - "name": "github.com/olivere/elastic/v7/uritemplates", - "path": "github.com/olivere/elastic/v7/uritemplates/LICENSE", - "licenseText": "Copyright (c) 2013 Joshua Tacoma\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" - }, { "name": "github.com/opencontainers/go-digest", "path": "github.com/opencontainers/go-digest/LICENSE", @@ -1059,6 +1069,11 @@ "path": "go.opentelemetry.io/otel/LICENSE", "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, + { + "name": "go.opentelemetry.io/otel/metric", + "path": "go.opentelemetry.io/otel/metric/LICENSE", + "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, { "name": "go.opentelemetry.io/otel/trace", "path": "go.opentelemetry.io/otel/trace/LICENSE", From 2341d5d99d71720e6643e19a62cd1f6dd0dd80ec Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 28 Apr 2024 23:16:18 -0400 Subject: [PATCH 03/21] Use gitea json module --- modules/indexer/issues/elasticsearch/elasticsearch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 67327671916e7..1b00f9c1b2335 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -6,7 +6,6 @@ package elasticsearch import ( "bytes" "context" - "encoding/json" "fmt" "strconv" "strings" @@ -15,6 +14,7 @@ import ( indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/json" "github.com/elastic/go-elasticsearch/v8" "github.com/elastic/go-elasticsearch/v8/esutil" From 44ed7867facfd706874a430b5234f93c46546ef2 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 28 Apr 2024 23:22:15 -0400 Subject: [PATCH 04/21] Fix lint --- .../code/elasticsearch/elasticsearch.go | 28 ++++----- .../issues/elasticsearch/elasticsearch.go | 60 +++++++++---------- 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index 337554e983ea5..68df7722ba84c 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -56,22 +56,20 @@ func NewIndexer(url, indexerName string) *Indexer { return indexer } -var ( - defaultMapping = &types.TypeMapping{ - Properties: map[string]types.Property{ - "repo_id": types.NewLongNumberProperty(), - "content": &types.TextProperty{ - Fields: make(map[string]types.Property, 0), - Meta: make(map[string]string, 0), - Properties: make(map[string]types.Property, 0), - TermVector: &termvectoroption.Withpositions, - }, - "commit_id": types.NewKeywordProperty(), - "language": types.NewKeywordProperty(), - "updated_at": types.NewLongNumberProperty(), +var defaultMapping = &types.TypeMapping{ + Properties: map[string]types.Property{ + "repo_id": types.NewLongNumberProperty(), + "content": &types.TextProperty{ + Fields: make(map[string]types.Property, 0), + Meta: make(map[string]string, 0), + Properties: make(map[string]types.Property, 0), + TermVector: &termvectoroption.Withpositions, }, - } -) + "commit_id": types.NewKeywordProperty(), + "language": types.NewKeywordProperty(), + "updated_at": types.NewLongNumberProperty(), + }, +} func (b *Indexer) addUpdate(ctx context.Context, blk *bulk.Bulk, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) error { // Ignore vendored files in code search diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 1b00f9c1b2335..ae110be22874e 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -47,37 +47,35 @@ func NewIndexer(url, indexerName string) *Indexer { return indexer } -var ( - defaultMapping = &types.TypeMapping{ - Properties: map[string]types.Property{ - "id": types.NewIntegerNumberProperty(), - "repo_id": types.NewIntegerNumberProperty(), - "is_public": types.NewBooleanProperty(), - - "title": types.NewTextProperty(), - "content": types.NewTextProperty(), - "comments": types.NewTextProperty(), - - "is_pull": types.NewBooleanProperty(), - "is_closed": types.NewBooleanProperty(), - "label_ids": types.NewIntegerNumberProperty(), - "no_label": types.NewBooleanProperty(), - "milestone_id": types.NewIntegerNumberProperty(), - "project_id": types.NewIntegerNumberProperty(), - "project_board_id": types.NewIntegerNumberProperty(), - "poster_id": types.NewIntegerNumberProperty(), - "assignee_id": types.NewIntegerNumberProperty(), - "mention_ids": types.NewIntegerNumberProperty(), - "reviewed_ids": types.NewIntegerNumberProperty(), - "review_requested_ids": types.NewIntegerNumberProperty(), - "subscriber_ids": types.NewIntegerNumberProperty(), - "updated_unix": types.NewIntegerNumberProperty(), - "created_unix": types.NewIntegerNumberProperty(), - "deadline_unix": types.NewIntegerNumberProperty(), - "comment_count": types.NewIntegerNumberProperty(), - }, - } -) +var defaultMapping = &types.TypeMapping{ + Properties: map[string]types.Property{ + "id": types.NewIntegerNumberProperty(), + "repo_id": types.NewIntegerNumberProperty(), + "is_public": types.NewBooleanProperty(), + + "title": types.NewTextProperty(), + "content": types.NewTextProperty(), + "comments": types.NewTextProperty(), + + "is_pull": types.NewBooleanProperty(), + "is_closed": types.NewBooleanProperty(), + "label_ids": types.NewIntegerNumberProperty(), + "no_label": types.NewBooleanProperty(), + "milestone_id": types.NewIntegerNumberProperty(), + "project_id": types.NewIntegerNumberProperty(), + "project_board_id": types.NewIntegerNumberProperty(), + "poster_id": types.NewIntegerNumberProperty(), + "assignee_id": types.NewIntegerNumberProperty(), + "mention_ids": types.NewIntegerNumberProperty(), + "reviewed_ids": types.NewIntegerNumberProperty(), + "review_requested_ids": types.NewIntegerNumberProperty(), + "subscriber_ids": types.NewIntegerNumberProperty(), + "updated_unix": types.NewIntegerNumberProperty(), + "created_unix": types.NewIntegerNumberProperty(), + "deadline_unix": types.NewIntegerNumberProperty(), + "comment_count": types.NewIntegerNumberProperty(), + }, +} // Index will save the index data func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) error { From 75cd37990a8a3589ca5d7f66a2000ac9578d8103 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 28 Apr 2024 23:34:30 -0400 Subject: [PATCH 05/21] Fix more lints --- modules/indexer/issues/elasticsearch/elasticsearch.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index ae110be22874e..0c729fccfce1c 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -134,17 +134,17 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { } for _, id := range ids { - bulkIndexer.Add(ctx, esutil.BulkIndexerItem{ + err = bulkIndexer.Add(ctx, esutil.BulkIndexerItem{ Action: "delete", Index: b.inner.VersionedIndexName(), DocumentID: fmt.Sprintf("%d", id), }) + if err != nil { + return err + } } - if err := bulkIndexer.Close(context.Background()); err != nil { - return err - } - return nil + return bulkIndexer.Close(ctx) } // Search searches for issues by given conditions. From 1e5fa911bffd7b4d2c2916135cdcb0fb6b3aed3c Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 28 Apr 2024 23:47:49 -0400 Subject: [PATCH 06/21] One more lint --- modules/indexer/internal/elasticsearch/util.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/modules/indexer/internal/elasticsearch/util.go b/modules/indexer/internal/elasticsearch/util.go index cd8f5d557bed7..f2330c4a757ab 100644 --- a/modules/indexer/internal/elasticsearch/util.go +++ b/modules/indexer/internal/elasticsearch/util.go @@ -30,8 +30,6 @@ func (i *Indexer) createIndex(ctx context.Context) error { createIndex, err := i.Client.Indices.Create(i.VersionedIndexName()).Request(&create.Request{ Mappings: i.mapping, }).Do(ctx) - - // .BodyString(i.mapping) if err != nil { return err } @@ -45,13 +43,6 @@ func (i *Indexer) createIndex(ctx context.Context) error { } func (i *Indexer) initClient() (*elasticsearch.TypedClient, error) { - // opts := []elastic.ClientOptionFunc{ - // elastic.SetURL(i.url), - // elastic.SetSniff(false), - // elastic.SetHealthcheckInterval(10 * time.Second), - // elastic.SetGzip(false), - // } - cfg := elasticsearch.Config{ Addresses: []string{i.url}, } From 1f809d6362d5565d0c62e85a02878a26ba88c37d Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sat, 20 Jul 2024 22:22:20 -0400 Subject: [PATCH 07/21] Fix board to column change from merge --- modules/indexer/issues/elasticsearch/elasticsearch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index bcc8dbaf3a459..974971dd295ca 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -269,7 +269,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( if options.ProjectColumnID.Has() { query.Bool.Must = append(query.Bool.Must, types.Query{ Term: map[string]types.TermQuery{ - "project_board_id": {Value: options.ProjectBoardID.Value()}, + "project_board_id": {Value: options.ProjectColumnID.Value()}, }, }) } From e132d174846ab38c66cf6d13b961aec25e5b0e37 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sat, 20 Jul 2024 22:43:05 -0400 Subject: [PATCH 08/21] Explicit version for v8 --- .../indexer/internal/elasticsearch/indexer.go | 10 +- .../indexer/internal/elasticsearch/util.go | 12 +- .../issues/elasticsearch/elasticsearch.go | 202 +++++++++--------- 3 files changed, 112 insertions(+), 112 deletions(-) diff --git a/modules/indexer/internal/elasticsearch/indexer.go b/modules/indexer/internal/elasticsearch/indexer.go index a4233e88eefd1..497581b523c67 100644 --- a/modules/indexer/internal/elasticsearch/indexer.go +++ b/modules/indexer/internal/elasticsearch/indexer.go @@ -9,23 +9,23 @@ import ( "code.gitea.io/gitea/modules/indexer/internal" - "github.com/elastic/go-elasticsearch/v8" - "github.com/elastic/go-elasticsearch/v8/typedapi/types" + elasticsearch8 "github.com/elastic/go-elasticsearch/v8" + types8 "github.com/elastic/go-elasticsearch/v8/typedapi/types" ) var _ internal.Indexer = &Indexer{} // Indexer represents a basic elasticsearch indexer implementation type Indexer struct { - Client *elasticsearch.TypedClient + Client *elasticsearch8.TypedClient url string indexName string version int - mapping *types.TypeMapping + mapping *types8.TypeMapping } -func NewIndexer(url, indexName string, version int, mapping *types.TypeMapping) *Indexer { +func NewIndexer(url, indexName string, version int, mapping *types8.TypeMapping) *Indexer { return &Indexer{ url: url, indexName: indexName, diff --git a/modules/indexer/internal/elasticsearch/util.go b/modules/indexer/internal/elasticsearch/util.go index f2330c4a757ab..846851b6df9d0 100644 --- a/modules/indexer/internal/elasticsearch/util.go +++ b/modules/indexer/internal/elasticsearch/util.go @@ -9,8 +9,8 @@ import ( "code.gitea.io/gitea/modules/log" - "github.com/elastic/go-elasticsearch/v8" - "github.com/elastic/go-elasticsearch/v8/typedapi/indices/create" + elasticsearch8 "github.com/elastic/go-elasticsearch/v8" + create8 "github.com/elastic/go-elasticsearch/v8/typedapi/indices/create" ) // VersionedIndexName returns the full index name with version @@ -27,7 +27,7 @@ func versionedIndexName(indexName string, version int) string { } func (i *Indexer) createIndex(ctx context.Context) error { - createIndex, err := i.Client.Indices.Create(i.VersionedIndexName()).Request(&create.Request{ + createIndex, err := i.Client.Indices.Create(i.VersionedIndexName()).Request(&create8.Request{ Mappings: i.mapping, }).Do(ctx) if err != nil { @@ -42,8 +42,8 @@ func (i *Indexer) createIndex(ctx context.Context) error { return nil } -func (i *Indexer) initClient() (*elasticsearch.TypedClient, error) { - cfg := elasticsearch.Config{ +func (i *Indexer) initClient() (*elasticsearch8.TypedClient, error) { + cfg := elasticsearch8.Config{ Addresses: []string{i.url}, } @@ -53,7 +53,7 @@ func (i *Indexer) initClient() (*elasticsearch.TypedClient, error) { // opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info})) // opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error})) - return elasticsearch.NewTypedClient(cfg) + return elasticsearch8.NewTypedClient(cfg) } func (i *Indexer) checkOldIndexes(ctx context.Context) { diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 974971dd295ca..e632588b9d587 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -16,13 +16,13 @@ import ( "code.gitea.io/gitea/modules/indexer/issues/internal" "code.gitea.io/gitea/modules/json" - "github.com/elastic/go-elasticsearch/v8" - "github.com/elastic/go-elasticsearch/v8/esutil" - "github.com/elastic/go-elasticsearch/v8/typedapi/core/bulk" - "github.com/elastic/go-elasticsearch/v8/typedapi/some" - "github.com/elastic/go-elasticsearch/v8/typedapi/types" - "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/sortorder" - "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/textquerytype" + elasticsearch8 "github.com/elastic/go-elasticsearch/v8" + esutil8 "github.com/elastic/go-elasticsearch/v8/esutil" + bulk8 "github.com/elastic/go-elasticsearch/v8/typedapi/core/bulk" + some8 "github.com/elastic/go-elasticsearch/v8/typedapi/some" + types8 "github.com/elastic/go-elasticsearch/v8/typedapi/types" + sortorder8 "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/sortorder" + textquerytype8 "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/textquerytype" ) const ( @@ -47,33 +47,33 @@ func NewIndexer(url, indexerName string) *Indexer { return indexer } -var defaultMapping = &types.TypeMapping{ - Properties: map[string]types.Property{ - "id": types.NewIntegerNumberProperty(), - "repo_id": types.NewIntegerNumberProperty(), - "is_public": types.NewBooleanProperty(), - - "title": types.NewTextProperty(), - "content": types.NewTextProperty(), - "comments": types.NewTextProperty(), - - "is_pull": types.NewBooleanProperty(), - "is_closed": types.NewBooleanProperty(), - "label_ids": types.NewIntegerNumberProperty(), - "no_label": types.NewBooleanProperty(), - "milestone_id": types.NewIntegerNumberProperty(), - "project_id": types.NewIntegerNumberProperty(), - "project_board_id": types.NewIntegerNumberProperty(), - "poster_id": types.NewIntegerNumberProperty(), - "assignee_id": types.NewIntegerNumberProperty(), - "mention_ids": types.NewIntegerNumberProperty(), - "reviewed_ids": types.NewIntegerNumberProperty(), - "review_requested_ids": types.NewIntegerNumberProperty(), - "subscriber_ids": types.NewIntegerNumberProperty(), - "updated_unix": types.NewIntegerNumberProperty(), - "created_unix": types.NewIntegerNumberProperty(), - "deadline_unix": types.NewIntegerNumberProperty(), - "comment_count": types.NewIntegerNumberProperty(), +var defaultMapping = &types8.TypeMapping{ + Properties: map[string]types8.Property{ + "id": types8.NewIntegerNumberProperty(), + "repo_id": types8.NewIntegerNumberProperty(), + "is_public": types8.NewBooleanProperty(), + + "title": types8.NewTextProperty(), + "content": types8.NewTextProperty(), + "comments": types8.NewTextProperty(), + + "is_pull": types8.NewBooleanProperty(), + "is_closed": types8.NewBooleanProperty(), + "label_ids": types8.NewIntegerNumberProperty(), + "no_label": types8.NewBooleanProperty(), + "milestone_id": types8.NewIntegerNumberProperty(), + "project_id": types8.NewIntegerNumberProperty(), + "project_board_id": types8.NewIntegerNumberProperty(), + "poster_id": types8.NewIntegerNumberProperty(), + "assignee_id": types8.NewIntegerNumberProperty(), + "mention_ids": types8.NewIntegerNumberProperty(), + "reviewed_ids": types8.NewIntegerNumberProperty(), + "review_requested_ids": types8.NewIntegerNumberProperty(), + "subscriber_ids": types8.NewIntegerNumberProperty(), + "updated_unix": types8.NewIntegerNumberProperty(), + "created_unix": types8.NewIntegerNumberProperty(), + "deadline_unix": types8.NewIntegerNumberProperty(), + "comment_count": types8.NewIntegerNumberProperty(), }, } @@ -96,7 +96,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er return err } - reqs := make(bulk.Request, 0) + reqs := make(bulk8.Request, 0) for _, issue := range issues { reqs = append(reqs, issue) } @@ -121,9 +121,9 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { return err } - bulkIndexer, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{ - Client: &elasticsearch.Client{ - BaseClient: elasticsearch.BaseClient{ + bulkIndexer, err := esutil8.NewBulkIndexer(esutil8.BulkIndexerConfig{ + Client: &elasticsearch8.Client{ + BaseClient: elasticsearch8.BaseClient{ Transport: b.inner.Client.Transport, }, }, @@ -134,7 +134,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { } for _, id := range ids { - err = bulkIndexer.Add(ctx, esutil.BulkIndexerItem{ + err = bulkIndexer.Add(ctx, esutil8.BulkIndexerItem{ Action: "delete", Index: b.inner.VersionedIndexName(), DocumentID: fmt.Sprintf("%d", id), @@ -150,20 +150,20 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { - query := &types.Query{ - Bool: &types.BoolQuery{ - Must: make([]types.Query, 0), + query := &types8.Query{ + Bool: &types8.BoolQuery{ + Must: make([]types8.Query, 0), }, } if options.Keyword != "" { - searchType := &textquerytype.Phraseprefix + searchType := &textquerytype8.Phraseprefix if options.IsFuzzyKeyword { - searchType = &textquerytype.Bestfields + searchType = &textquerytype8.Bestfields } - query.Bool.Must = append(query.Bool.Must, types.Query{ - MultiMatch: &types.MultiMatchQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + MultiMatch: &types8.MultiMatchQuery{ Query: options.Keyword, Fields: []string{"title", "content", "comments"}, Type: searchType, @@ -172,14 +172,14 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( } if len(options.RepoIDs) > 0 { - q := types.Query{ - Bool: &types.BoolQuery{ - Should: make([]types.Query, 0), + q := types8.Query{ + Bool: &types8.BoolQuery{ + Should: make([]types8.Query, 0), }, } if options.AllPublic { - q.Bool.Should = append(q.Bool.Should, types.Query{ - Term: map[string]types.TermQuery{ + q.Bool.Should = append(q.Bool.Should, types8.Query{ + Term: map[string]types8.TermQuery{ "is_public": {Value: true}, }, }) @@ -188,59 +188,59 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( } if options.IsPull.Has() { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "is_pull": {Value: options.IsPull.Value()}, }, }) } if options.IsClosed.Has() { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "is_closed": {Value: options.IsClosed.Value()}, }, }) } if options.NoLabelOnly { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "no_label": {Value: true}, }, }) } else { if len(options.IncludedLabelIDs) > 0 { - q := types.Query{ - Bool: &types.BoolQuery{ - Must: make([]types.Query, 0), + q := types8.Query{ + Bool: &types8.BoolQuery{ + Must: make([]types8.Query, 0), }, } for _, labelID := range options.IncludedLabelIDs { - q.Bool.Must = append(q.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + q.Bool.Must = append(q.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "label_ids": {Value: labelID}, }, }) } query.Bool.Must = append(query.Bool.Must, q) } else if len(options.IncludedAnyLabelIDs) > 0 { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Terms: &types.TermsQuery{ - TermsQuery: map[string]types.TermsQueryField{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Terms: &types8.TermsQuery{ + TermsQuery: map[string]types8.TermsQueryField{ "label_ids": toAnySlice(options.IncludedAnyLabelIDs), }, }, }) } if len(options.ExcludedLabelIDs) > 0 { - q := types.Query{ - Bool: &types.BoolQuery{ - MustNot: make([]types.Query, 0), + q := types8.Query{ + Bool: &types8.BoolQuery{ + MustNot: make([]types8.Query, 0), }, } for _, labelID := range options.ExcludedLabelIDs { - q.Bool.MustNot = append(q.Bool.MustNot, types.Query{ - Term: map[string]types.TermQuery{ + q.Bool.MustNot = append(q.Bool.MustNot, types8.Query{ + Term: map[string]types8.TermQuery{ "label_ids": {Value: labelID}, }, }) @@ -250,9 +250,9 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( } if len(options.MilestoneIDs) > 0 { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Terms: &types.TermsQuery{ - TermsQuery: map[string]types.TermsQueryField{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Terms: &types8.TermsQuery{ + TermsQuery: map[string]types8.TermsQueryField{ "milestone_id": toAnySlice(options.MilestoneIDs), }, }, @@ -260,78 +260,78 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( } if options.ProjectID.Has() { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "project_id": {Value: options.ProjectID.Value()}, }, }) } if options.ProjectColumnID.Has() { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "project_board_id": {Value: options.ProjectColumnID.Value()}, }, }) } if options.PosterID.Has() { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "poster_id": {Value: options.PosterID.Value()}, }, }) } if options.AssigneeID.Has() { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "assignee_id": {Value: options.AssigneeID.Value()}, }, }) } if options.MentionID.Has() { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "mention_ids": {Value: options.MentionID.Value()}, }, }) } if options.ReviewedID.Has() { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "reviewed_ids": {Value: options.ReviewedID.Value()}, }, }) } if options.ReviewRequestedID.Has() { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "review_requested_ids": {Value: options.ReviewRequestedID.Value()}, }, }) } if options.SubscriberID.Has() { - query.Bool.Must = append(query.Bool.Must, types.Query{ - Term: map[string]types.TermQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Term: map[string]types8.TermQuery{ "subscriber_ids": {Value: options.SubscriberID.Value()}, }, }) } if options.UpdatedAfterUnix.Has() || options.UpdatedBeforeUnix.Has() { - rangeQuery := types.NumberRangeQuery{} + rangeQuery := types8.NumberRangeQuery{} if options.UpdatedAfterUnix.Has() { - rangeQuery.Gte = some.Float64(float64(options.UpdatedAfterUnix.Value())) + rangeQuery.Gte = some8.Float64(float64(options.UpdatedAfterUnix.Value())) } if options.UpdatedBeforeUnix.Has() { - rangeQuery.Lte = some.Float64(float64(options.UpdatedBeforeUnix.Value())) + rangeQuery.Lte = some8.Float64(float64(options.UpdatedBeforeUnix.Value())) } - query.Bool.Must = append(query.Bool.Must, types.Query{ - Range: map[string]types.RangeQuery{ + query.Bool.Must = append(query.Bool.Must, types8.Query{ + Range: map[string]types8.RangeQuery{ "updated_unix": rangeQuery, }, }) @@ -341,10 +341,10 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( options.SortBy = internal.SortByCreatedAsc } field, fieldSort := parseSortBy(options.SortBy) - sort := []types.SortCombinations{ - &types.SortOptions{SortOptions: map[string]types.FieldSort{ + sort := []types8.SortCombinations{ + &types8.SortOptions{SortOptions: map[string]types8.FieldSort{ field: fieldSort, - "id": {Order: &sortorder.Desc}, + "id": {Order: &sortorder8.Desc}, }}, } @@ -385,13 +385,13 @@ func toAnySlice[T any](s []T) []any { return ret } -func parseSortBy(sortBy internal.SortBy) (string, types.FieldSort) { +func parseSortBy(sortBy internal.SortBy) (string, types8.FieldSort) { field := strings.TrimPrefix(string(sortBy), "-") - sort := types.FieldSort{ - Order: &sortorder.Asc, + sort := types8.FieldSort{ + Order: &sortorder8.Asc, } if strings.HasPrefix(string(sortBy), "-") { - sort.Order = &sortorder.Desc + sort.Order = &sortorder8.Desc } return field, sort From ae43bf446066d2fb9a5c51e3a4cae8e9425e49ce Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sat, 20 Jul 2024 23:30:41 -0400 Subject: [PATCH 09/21] Add back v7 code --- go.mod | 2 +- go.sum | 4 + .../code/elasticsearch/elasticsearch_v7.go | 343 ++++++++++++++++++ .../{elasticsearch.go => elasticsearch_v8.go} | 144 ++++---- .../internal/elasticsearch/indexer_v7.go | 93 +++++ .../{indexer.go => indexer_v8.go} | 14 +- .../indexer/internal/elasticsearch/util_v7.go | 60 +++ .../elasticsearch/{util.go => util_v8.go} | 8 +- .../issues/elasticsearch/elasticsearch_v7.go | 289 +++++++++++++++ .../{elasticsearch.go => elasticsearch_v8.go} | 30 +- ...earch_test.go => elasticsearch_v8_test.go} | 0 11 files changed, 884 insertions(+), 103 deletions(-) create mode 100644 modules/indexer/code/elasticsearch/elasticsearch_v7.go rename modules/indexer/code/elasticsearch/{elasticsearch.go => elasticsearch_v8.go} (66%) create mode 100644 modules/indexer/internal/elasticsearch/indexer_v7.go rename modules/indexer/internal/elasticsearch/{indexer.go => indexer_v8.go} (84%) create mode 100644 modules/indexer/internal/elasticsearch/util_v7.go rename modules/indexer/internal/elasticsearch/{util.go => util_v8.go} (87%) create mode 100644 modules/indexer/issues/elasticsearch/elasticsearch_v7.go rename modules/indexer/issues/elasticsearch/{elasticsearch.go => elasticsearch_v8.go} (92%) rename modules/indexer/issues/elasticsearch/{elasticsearch_test.go => elasticsearch_v8_test.go} (100%) diff --git a/go.mod b/go.mod index 29baad9b788a3..5e94a69b09f01 100644 --- a/go.mod +++ b/go.mod @@ -85,6 +85,7 @@ require ( github.com/msteinert/pam v1.2.0 github.com/nektos/act v0.2.63 github.com/niklasfasching/go-org v1.7.0 + github.com/olivere/elastic/v7 v7.0.32 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0 github.com/pkg/errors v0.9.1 @@ -270,7 +271,6 @@ require ( github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect - github.com/smartystreets/assertions v1.1.1 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect diff --git a/go.sum b/go.sum index 8f228a4368b62..a4ada1092071b 100644 --- a/go.sum +++ b/go.sum @@ -270,6 +270,8 @@ github.com/felixge/fgprof v0.9.4 h1:ocDNwMFlnA0NU0zSB3I52xkO4sFXk80VK9lXjLClu88= github.com/felixge/fgprof v0.9.4/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -622,6 +624,8 @@ github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= +github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/modules/indexer/code/elasticsearch/elasticsearch_v7.go b/modules/indexer/code/elasticsearch/elasticsearch_v7.go new file mode 100644 index 0000000000000..a74300f3fcacb --- /dev/null +++ b/modules/indexer/code/elasticsearch/elasticsearch_v7.go @@ -0,0 +1,343 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package elasticsearch + +import ( + "bufio" + "context" + "fmt" + "io" + "strconv" + "strings" + + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/analyze" + "code.gitea.io/gitea/modules/charset" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/indexer/code/internal" + indexer_internal "code.gitea.io/gitea/modules/indexer/internal" + inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/typesniffer" + + "github.com/go-enry/go-enry/v2" + "github.com/olivere/elastic/v7" +) + +const ( + // multi-match-types, currently only 2 types are used + // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types + esMultiMatchTypeBestFields = "best_fields" + esMultiMatchTypePhrasePrefix = "phrase_prefix" +) + +var _ internal.Indexer = &Indexer{} + +// Indexer implements Indexer interface +type Indexer struct { + inner *inner_elasticsearch.Indexer + indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much +} + +// NewIndexer creates a new elasticsearch indexer +func NewIndexer(url, indexerName string) *Indexer { + inner := inner_elasticsearch.NewIndexer(url, indexerName, esRepoIndexerLatestVersion, defaultMapping) + indexer := &Indexer{ + inner: inner, + Indexer: inner, + } + return indexer +} + +const ( + defaultMapping = `{ + "mappings": { + "properties": { + "repo_id": { + "type": "long", + "index": true + }, + "content": { + "type": "text", + "term_vector": "with_positions_offsets", + "index": true + }, + "commit_id": { + "type": "keyword", + "index": true + }, + "language": { + "type": "keyword", + "index": true + }, + "updated_at": { + "type": "long", + "index": true + } + } + } + }` +) + +func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { + // Ignore vendored files in code search + if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { + return nil, nil + } + + size := update.Size + var err error + if !update.Sized { + var stdout string + stdout, _, err = git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + if err != nil { + return nil, err + } + if size, err = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64); err != nil { + return nil, fmt.Errorf("misformatted git cat-file output: %w", err) + } + } + + if size > setting.Indexer.MaxIndexerFileSize { + return []elastic.BulkableRequest{b.addDelete(update.Filename, repo)}, nil + } + + if _, err := batchWriter.Write([]byte(update.BlobSha + "\n")); err != nil { + return nil, err + } + + _, _, size, err = git.ReadBatchLine(batchReader) + if err != nil { + return nil, err + } + + fileContents, err := io.ReadAll(io.LimitReader(batchReader, size)) + if err != nil { + return nil, err + } else if !typesniffer.DetectContentType(fileContents).IsText() { + // FIXME: UTF-16 files will probably fail here + return nil, nil + } + + if _, err = batchReader.Discard(1); err != nil { + return nil, err + } + id := internal.FilenameIndexerID(repo.ID, update.Filename) + + return []elastic.BulkableRequest{ + elastic.NewBulkIndexRequest(). + Index(b.inner.VersionedIndexName()). + Id(id). + Doc(map[string]any{ + "repo_id": repo.ID, + "content": string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})), + "commit_id": sha, + "language": analyze.GetCodeLanguage(update.Filename, fileContents), + "updated_at": timeutil.TimeStampNow(), + }), + }, nil +} + +func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elastic.BulkableRequest { + id := internal.FilenameIndexerID(repo.ID, filename) + return elastic.NewBulkDeleteRequest(). + Index(b.inner.VersionedIndexName()). + Id(id) +} + +// Index will save the index data +func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { + reqs := make([]elastic.BulkableRequest, 0) + if len(changes.Updates) > 0 { + // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! + if err := git.EnsureValidGitRepository(ctx, repo.RepoPath()); err != nil { + log.Error("Unable to open git repo: %s for %-v: %v", repo.RepoPath(), repo, err) + return err + } + + batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repo.RepoPath()) + defer cancel() + + for _, update := range changes.Updates { + updateReqs, err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo) + if err != nil { + return err + } + if len(updateReqs) > 0 { + reqs = append(reqs, updateReqs...) + } + } + cancel() + } + + for _, filename := range changes.RemovedFilenames { + reqs = append(reqs, b.addDelete(filename, repo)) + } + + if len(reqs) > 0 { + esBatchSize := 50 + + for i := 0; i < len(reqs); i += esBatchSize { + _, err := b.inner.Client.Bulk(). + Index(b.inner.VersionedIndexName()). + Add(reqs[i:min(i+esBatchSize, len(reqs))]...). + Do(ctx) + if err != nil { + return err + } + } + } + return nil +} + +// Delete deletes indexes by ids +func (b *Indexer) Delete(ctx context.Context, repoID int64) error { + _, err := b.inner.Client.DeleteByQuery(b.inner.VersionedIndexName()). + Query(elastic.NewTermsQuery("repo_id", repoID)). + Do(ctx) + return err +} + +func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { + hits := make([]*internal.SearchResult, 0, pageSize) + for _, hit := range searchResult.Hits.Hits { + // FIXME: There is no way to get the position the keyword on the content currently on the same request. + // So we get it from content, this may made the query slower. See + // https://discuss.elastic.co/t/fetching-position-of-keyword-in-matched-document/94291 + var startIndex, endIndex int + c, ok := hit.Highlight["content"] + if ok && len(c) > 0 { + // FIXME: Since the highlighting content will include and for the keywords, + // now we should find the positions. But how to avoid html content which contains the + // and tags? If elastic search has handled that? + startIndex, endIndex = indexPos(c[0], "", "") + if startIndex == -1 { + panic(fmt.Sprintf("1===%s,,,%#v,,,%s", kw, hit.Highlight, c[0])) + } + } else { + panic(fmt.Sprintf("2===%#v", hit.Highlight)) + } + + repoID, fileName := internal.ParseIndexerID(hit.Id) + res := make(map[string]any) + if err := json.Unmarshal(hit.Source, &res); err != nil { + return 0, nil, nil, err + } + + language := res["language"].(string) + + hits = append(hits, &internal.SearchResult{ + RepoID: repoID, + Filename: fileName, + CommitID: res["commit_id"].(string), + Content: res["content"].(string), + UpdatedUnix: timeutil.TimeStamp(res["updated_at"].(float64)), + Language: language, + StartIndex: startIndex, + EndIndex: endIndex - 9, // remove the length since we give Content the original data + Color: enry.GetColor(language), + }) + } + + return searchResult.TotalHits(), hits, extractAggs(searchResult), nil +} + +func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLanguages { + var searchResultLanguages []*internal.SearchResultLanguages + agg, found := searchResult.Aggregations.Terms("language") + if found { + searchResultLanguages = make([]*internal.SearchResultLanguages, 0, 10) + + for _, bucket := range agg.Buckets { + searchResultLanguages = append(searchResultLanguages, &internal.SearchResultLanguages{ + Language: bucket.Key.(string), + Color: enry.GetColor(bucket.Key.(string)), + Count: int(bucket.DocCount), + }) + } + } + return searchResultLanguages +} + +// Search searches for codes and language stats by given conditions. +func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { + searchType := esMultiMatchTypePhrasePrefix + if opts.IsKeywordFuzzy { + searchType = esMultiMatchTypeBestFields + } + + kwQuery := elastic.NewMultiMatchQuery(opts.Keyword, "content").Type(searchType) + query := elastic.NewBoolQuery() + query = query.Must(kwQuery) + if len(opts.RepoIDs) > 0 { + repoStrs := make([]any, 0, len(opts.RepoIDs)) + for _, repoID := range opts.RepoIDs { + repoStrs = append(repoStrs, repoID) + } + repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...) + query = query.Must(repoQuery) + } + + var ( + start, pageSize = opts.GetSkipTake() + kw = "" + opts.Keyword + "" + aggregation = elastic.NewTermsAggregation().Field("language").Size(10).OrderByCountDesc() + ) + + if len(opts.Language) == 0 { + searchResult, err := b.inner.Client.Search(). + Index(b.inner.VersionedIndexName()). + Aggregation("language", aggregation). + Query(query). + Highlight( + elastic.NewHighlight(). + Field("content"). + NumOfFragments(0). // return all highting content on fragments + HighlighterType("fvh"), + ). + Sort("repo_id", true). + From(start).Size(pageSize). + Do(ctx) + if err != nil { + return 0, nil, nil, err + } + + return convertResult(searchResult, kw, pageSize) + } + + langQuery := elastic.NewMatchQuery("language", opts.Language) + countResult, err := b.inner.Client.Search(). + Index(b.inner.VersionedIndexName()). + Aggregation("language", aggregation). + Query(query). + Size(0). // We only need stats information + Do(ctx) + if err != nil { + return 0, nil, nil, err + } + + query = query.Must(langQuery) + searchResult, err := b.inner.Client.Search(). + Index(b.inner.VersionedIndexName()). + Query(query). + Highlight( + elastic.NewHighlight(). + Field("content"). + NumOfFragments(0). // return all highting content on fragments + HighlighterType("fvh"), + ). + Sort("repo_id", true). + From(start).Size(pageSize). + Do(ctx) + if err != nil { + return 0, nil, nil, err + } + + total, hits, _, err := convertResult(searchResult, kw, pageSize) + + return total, hits, extractAggs(countResult), err +} diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch_v8.go similarity index 66% rename from modules/indexer/code/elasticsearch/elasticsearch.go rename to modules/indexer/code/elasticsearch/elasticsearch_v8.go index 68df7722ba84c..8fda07b9675f5 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch_v8.go @@ -23,14 +23,14 @@ import ( "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/typesniffer" - "github.com/elastic/go-elasticsearch/v8/typedapi/core/bulk" - "github.com/elastic/go-elasticsearch/v8/typedapi/core/search" - "github.com/elastic/go-elasticsearch/v8/typedapi/some" - "github.com/elastic/go-elasticsearch/v8/typedapi/types" - "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/highlightertype" - "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/sortorder" - "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/termvectoroption" - "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/textquerytype" + bulkV8 "github.com/elastic/go-elasticsearch/v8/typedapi/core/bulk" + searchV8 "github.com/elastic/go-elasticsearch/v8/typedapi/core/search" + someV8 "github.com/elastic/go-elasticsearch/v8/typedapi/some" + typesV8 "github.com/elastic/go-elasticsearch/v8/typedapi/types" + highlightertypeV8 "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/highlightertype" + sortorderV8 "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/sortorder" + termvectoroptionV8 "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/termvectoroption" + textquerytypeV8 "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/textquerytype" "github.com/go-enry/go-enry/v2" ) @@ -38,40 +38,40 @@ const ( esRepoIndexerLatestVersion = 1 ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = &IndexerV8{} -// Indexer implements Indexer interface -type Indexer struct { - inner *inner_elasticsearch.Indexer +// IndexerV8 implements Indexer interface +type IndexerV8 struct { + inner *inner_elasticsearch.IndexerV8 indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much } // NewIndexer creates a new elasticsearch indexer -func NewIndexer(url, indexerName string) *Indexer { - inner := inner_elasticsearch.NewIndexer(url, indexerName, esRepoIndexerLatestVersion, defaultMapping) - indexer := &Indexer{ +func NewIndexerV8(url, indexerName string) *IndexerV8 { + inner := inner_elasticsearch.NewIndexerV8(url, indexerName, esRepoIndexerLatestVersion, defaultMappingV8) + indexer := &IndexerV8{ inner: inner, Indexer: inner, } return indexer } -var defaultMapping = &types.TypeMapping{ - Properties: map[string]types.Property{ - "repo_id": types.NewLongNumberProperty(), - "content": &types.TextProperty{ - Fields: make(map[string]types.Property, 0), +var defaultMappingV8 = &typesV8.TypeMapping{ + Properties: map[string]typesV8.Property{ + "repo_id": typesV8.NewLongNumberProperty(), + "content": &typesV8.TextProperty{ + Fields: make(map[string]typesV8.Property, 0), Meta: make(map[string]string, 0), - Properties: make(map[string]types.Property, 0), - TermVector: &termvectoroption.Withpositions, + Properties: make(map[string]typesV8.Property, 0), + TermVector: &termvectoroptionV8.Withpositions, }, - "commit_id": types.NewKeywordProperty(), - "language": types.NewKeywordProperty(), - "updated_at": types.NewLongNumberProperty(), + "commit_id": typesV8.NewKeywordProperty(), + "language": typesV8.NewKeywordProperty(), + "updated_at": typesV8.NewLongNumberProperty(), }, } -func (b *Indexer) addUpdate(ctx context.Context, blk *bulk.Bulk, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) error { +func (b *IndexerV8) addUpdate(ctx context.Context, blk *bulkV8.Bulk, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) error { // Ignore vendored files in code search if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { return nil @@ -116,9 +116,9 @@ func (b *Indexer) addUpdate(ctx context.Context, blk *bulk.Bulk, batchWriter git } id := internal.FilenameIndexerID(repo.ID, update.Filename) - return blk.IndexOp(types.IndexOperation{ - Index_: some.String(b.inner.VersionedIndexName()), - Id_: some.String(id), + return blk.IndexOp(typesV8.IndexOperation{ + Index_: someV8.String(b.inner.VersionedIndexName()), + Id_: someV8.String(id), }, map[string]any{ "id": id, "repo_id": repo.ID, @@ -129,16 +129,16 @@ func (b *Indexer) addUpdate(ctx context.Context, blk *bulk.Bulk, batchWriter git }) } -func (b *Indexer) addDelete(blk *bulk.Bulk, filename string, repo *repo_model.Repository) error { +func (b *IndexerV8) addDelete(blk *bulkV8.Bulk, filename string, repo *repo_model.Repository) error { id := internal.FilenameIndexerID(repo.ID, filename) - return blk.DeleteOp(types.DeleteOperation{ - Index_: some.String(b.inner.VersionedIndexName()), - Id_: some.String(id), + return blk.DeleteOp(typesV8.DeleteOperation{ + Index_: someV8.String(b.inner.VersionedIndexName()), + Id_: someV8.String(id), }) } // Index will save the index data -func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { +func (b *IndexerV8) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { if len(changes.Updates) == 0 && len(changes.RemovedFilenames) == 0 { return nil } @@ -176,10 +176,10 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st } // Delete deletes indexes by ids -func (b *Indexer) Delete(ctx context.Context, repoID int64) error { +func (b *IndexerV8) Delete(ctx context.Context, repoID int64) error { _, err := b.inner.Client.DeleteByQuery(b.inner.VersionedIndexName()). - Query(&types.Query{ - Term: map[string]types.TermQuery{ + Query(&typesV8.Query{ + Term: map[string]typesV8.TermQuery{ "repo_id": {Value: repoID}, }, }). @@ -204,7 +204,7 @@ func indexPos(content, start, end string) (int, int) { return startIdx, startIdx + len(start) + endIdx + len(end) } -func convertResult(searchResult *search.Response, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { +func convertResultV8(searchResult *searchV8.Response, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { hits := make([]*internal.SearchResult, 0, pageSize) for _, hit := range searchResult.Hits.Hits { // FIXME: There is no way to get the position the keyword on the content currently on the same request. @@ -248,14 +248,14 @@ func convertResult(searchResult *search.Response, kw string, pageSize int) (int6 return searchResult.Hits.Total.Value, hits, extractAggregates(searchResult), nil } -func extractAggregates(searchResult *search.Response) []*internal.SearchResultLanguages { +func extractAggregates(searchResult *searchV8.Response) []*internal.SearchResultLanguages { var searchResultLanguages []*internal.SearchResultLanguages agg, found := searchResult.Aggregations["language"] if found { searchResultLanguages = make([]*internal.SearchResultLanguages, 0, 10) - languageAgg := agg.(*types.StringTermsAggregate) - buckets := languageAgg.Buckets.([]types.StringTermsBucket) + languageAgg := agg.(*typesV8.StringTermsAggregate) + buckets := languageAgg.Buckets.([]typesV8.StringTermsBucket) for _, bucket := range buckets { searchResultLanguages = append(searchResultLanguages, &internal.SearchResultLanguages{ Language: bucket.Key.(string), @@ -268,33 +268,33 @@ func extractAggregates(searchResult *search.Response) []*internal.SearchResultLa } // Search searches for codes and language stats by given conditions. -func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { +func (b *IndexerV8) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { // searchType := esMultiMatchTypePhrasePrefix - searchType := &textquerytype.Phraseprefix + searchType := &textquerytypeV8.Phraseprefix if opts.IsKeywordFuzzy { - searchType = &textquerytype.Bestfields + searchType = &textquerytypeV8.Bestfields } - kwQuery := types.Query{ - MultiMatch: &types.MultiMatchQuery{ + kwQuery := typesV8.Query{ + MultiMatch: &typesV8.MultiMatchQuery{ Query: opts.Keyword, Fields: []string{"content"}, Type: searchType, }, } - query := &types.Query{ - Bool: &types.BoolQuery{ - Must: []types.Query{kwQuery}, + query := &typesV8.Query{ + Bool: &typesV8.BoolQuery{ + Must: []typesV8.Query{kwQuery}, }, } if len(opts.RepoIDs) > 0 { - repoIDs := make([]types.FieldValue, 0, len(opts.RepoIDs)) + repoIDs := make([]typesV8.FieldValue, 0, len(opts.RepoIDs)) for _, repoID := range opts.RepoIDs { - repoIDs = append(repoIDs, types.FieldValue(repoID)) + repoIDs = append(repoIDs, typesV8.FieldValue(repoID)) } - repoQuery := types.Query{ - Terms: &types.TermsQuery{ - TermsQuery: map[string]types.TermsQueryField{ + repoQuery := typesV8.Query{ + Terms: &typesV8.TermsQuery{ + TermsQuery: map[string]typesV8.TermsQueryField{ "repo_id": repoIDs, }, }, @@ -305,13 +305,13 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int var ( start, pageSize = opts.GetSkipTake() kw = "" + opts.Keyword + "" - aggregation = map[string]types.Aggregations{ + aggregation = map[string]typesV8.Aggregations{ "language": { - Terms: &types.TermsAggregation{ - Field: some.String("language"), - Size: some.Int(10), - Order: map[string]sortorder.SortOrder{ - "_count": sortorder.Desc, + Terms: &typesV8.TermsAggregation{ + Field: someV8.String("language"), + Size: someV8.Int(10), + Order: map[string]sortorderV8.SortOrder{ + "_count": sortorderV8.Desc, }, }, }, @@ -324,11 +324,11 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int Aggregations(aggregation). Query(query). Highlight( - &types.Highlight{ - Fields: map[string]types.HighlightField{ + &typesV8.Highlight{ + Fields: map[string]typesV8.HighlightField{ "content": { - NumberOfFragments: some.Int(0), // return all highting content on fragments - Type: &highlightertype.Fvh, + NumberOfFragments: someV8.Int(0), // return all highting content on fragments + Type: &highlightertypeV8.Fvh, }, }, }, @@ -340,11 +340,11 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int return 0, nil, nil, err } - return convertResult(searchResult, kw, pageSize) + return convertResultV8(searchResult, kw, pageSize) } - langQuery := types.Query{ - Match: map[string]types.MatchQuery{ + langQuery := typesV8.Query{ + Match: map[string]typesV8.MatchQuery{ "language": { Query: opts.Language, }, @@ -365,11 +365,11 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int Index(b.inner.VersionedIndexName()). Query(query). Highlight( - &types.Highlight{ - Fields: map[string]types.HighlightField{ + &typesV8.Highlight{ + Fields: map[string]typesV8.HighlightField{ "content": { - NumberOfFragments: some.Int(0), // return all highting content on fragments - Type: &highlightertype.Fvh, + NumberOfFragments: someV8.Int(0), // return all highting content on fragments + Type: &highlightertypeV8.Fvh, }, }, }, @@ -381,7 +381,7 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int return 0, nil, nil, err } - total, hits, _, err := convertResult(searchResult, kw, pageSize) + total, hits, _, err := convertResultV8(searchResult, kw, pageSize) return total, hits, extractAggregates(countResult), err } diff --git a/modules/indexer/internal/elasticsearch/indexer_v7.go b/modules/indexer/internal/elasticsearch/indexer_v7.go new file mode 100644 index 0000000000000..395eea3bce652 --- /dev/null +++ b/modules/indexer/internal/elasticsearch/indexer_v7.go @@ -0,0 +1,93 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package elasticsearch + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/modules/indexer/internal" + + "github.com/olivere/elastic/v7" +) + +var _ internal.Indexer = &Indexer{} + +// Indexer represents a basic elasticsearch indexer implementation +type Indexer struct { + Client *elastic.Client + + url string + indexName string + version int + mapping string +} + +func NewIndexer(url, indexName string, version int, mapping string) *Indexer { + return &Indexer{ + url: url, + indexName: indexName, + version: version, + mapping: mapping, + } +} + +// Init initializes the indexer +func (i *Indexer) Init(ctx context.Context) (bool, error) { + if i == nil { + return false, fmt.Errorf("cannot init nil indexer") + } + if i.Client != nil { + return false, fmt.Errorf("indexer is already initialized") + } + + client, err := i.initClient() + if err != nil { + return false, err + } + i.Client = client + + exists, err := i.Client.IndexExists(i.VersionedIndexName()).Do(ctx) + if err != nil { + return false, err + } + if exists { + return true, nil + } + + if err := i.createIndex(ctx); err != nil { + return false, err + } + + return exists, nil +} + +// Ping checks if the indexer is available +func (i *Indexer) Ping(ctx context.Context) error { + if i == nil { + return fmt.Errorf("cannot ping nil indexer") + } + if i.Client == nil { + return fmt.Errorf("indexer is not initialized") + } + + resp, err := i.Client.ClusterHealth().Do(ctx) + if err != nil { + return err + } + if resp.Status != "green" && resp.Status != "yellow" { + // It's healthy if the status is green, and it's available if the status is yellow, + // see https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html + return fmt.Errorf("status of elasticsearch cluster is %s", resp.Status) + } + return nil +} + +// Close closes the indexer +func (i *Indexer) Close() { + if i == nil { + return + } + i.Client = nil +} diff --git a/modules/indexer/internal/elasticsearch/indexer.go b/modules/indexer/internal/elasticsearch/indexer_v8.go similarity index 84% rename from modules/indexer/internal/elasticsearch/indexer.go rename to modules/indexer/internal/elasticsearch/indexer_v8.go index 497581b523c67..4a042e3fcbfb0 100644 --- a/modules/indexer/internal/elasticsearch/indexer.go +++ b/modules/indexer/internal/elasticsearch/indexer_v8.go @@ -13,10 +13,10 @@ import ( types8 "github.com/elastic/go-elasticsearch/v8/typedapi/types" ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = &IndexerV8{} // Indexer represents a basic elasticsearch indexer implementation -type Indexer struct { +type IndexerV8 struct { Client *elasticsearch8.TypedClient url string @@ -25,8 +25,8 @@ type Indexer struct { mapping *types8.TypeMapping } -func NewIndexer(url, indexName string, version int, mapping *types8.TypeMapping) *Indexer { - return &Indexer{ +func NewIndexerV8(url, indexName string, version int, mapping *types8.TypeMapping) *IndexerV8 { + return &IndexerV8{ url: url, indexName: indexName, version: version, @@ -35,7 +35,7 @@ func NewIndexer(url, indexName string, version int, mapping *types8.TypeMapping) } // Init initializes the indexer -func (i *Indexer) Init(ctx context.Context) (bool, error) { +func (i *IndexerV8) Init(ctx context.Context) (bool, error) { if i == nil { return false, fmt.Errorf("cannot init nil indexer") } @@ -65,7 +65,7 @@ func (i *Indexer) Init(ctx context.Context) (bool, error) { } // Ping checks if the indexer is available -func (i *Indexer) Ping(ctx context.Context) error { +func (i *IndexerV8) Ping(ctx context.Context) error { if i == nil { return fmt.Errorf("cannot ping nil indexer") } @@ -86,7 +86,7 @@ func (i *Indexer) Ping(ctx context.Context) error { } // Close closes the indexer -func (i *Indexer) Close() { +func (i *IndexerV8) Close() { if i == nil { return } diff --git a/modules/indexer/internal/elasticsearch/util_v7.go b/modules/indexer/internal/elasticsearch/util_v7.go new file mode 100644 index 0000000000000..418d4ff5084da --- /dev/null +++ b/modules/indexer/internal/elasticsearch/util_v7.go @@ -0,0 +1,60 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package elasticsearch + +import ( + "context" + "fmt" + "time" + + "code.gitea.io/gitea/modules/log" + + "github.com/olivere/elastic/v7" +) + +// VersionedIndexName returns the full index name with version +func (i *Indexer) VersionedIndexName() string { + return versionedIndexName(i.indexName, i.version) +} + +func (i *Indexer) createIndex(ctx context.Context) error { + createIndex, err := i.Client.CreateIndex(i.VersionedIndexName()).BodyString(i.mapping).Do(ctx) + if err != nil { + return err + } + if !createIndex.Acknowledged { + return fmt.Errorf("create index %s with %s failed", i.VersionedIndexName(), i.mapping) + } + + i.checkOldIndexes(ctx) + + return nil +} + +func (i *Indexer) initClient() (*elastic.Client, error) { + opts := []elastic.ClientOptionFunc{ + elastic.SetURL(i.url), + elastic.SetSniff(false), + elastic.SetHealthcheckInterval(10 * time.Second), + elastic.SetGzip(false), + } + + logger := log.GetLogger(log.DEFAULT) + + opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace})) + opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info})) + opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error})) + + return elastic.NewClient(opts...) +} + +func (i *Indexer) checkOldIndexes(ctx context.Context) { + for v := 0; v < i.version; v++ { + indexName := versionedIndexName(i.indexName, v) + exists, err := i.Client.IndexExists(indexName).Do(ctx) + if err == nil && exists { + log.Warn("Found older elasticsearch index named %q, Gitea will keep the old NOT DELETED. You can delete the old version after the upgrade succeed.", indexName) + } + } +} diff --git a/modules/indexer/internal/elasticsearch/util.go b/modules/indexer/internal/elasticsearch/util_v8.go similarity index 87% rename from modules/indexer/internal/elasticsearch/util.go rename to modules/indexer/internal/elasticsearch/util_v8.go index 846851b6df9d0..01dd1f36b038f 100644 --- a/modules/indexer/internal/elasticsearch/util.go +++ b/modules/indexer/internal/elasticsearch/util_v8.go @@ -14,7 +14,7 @@ import ( ) // VersionedIndexName returns the full index name with version -func (i *Indexer) VersionedIndexName() string { +func (i *IndexerV8) VersionedIndexName() string { return versionedIndexName(i.indexName, i.version) } @@ -26,7 +26,7 @@ func versionedIndexName(indexName string, version int) string { return fmt.Sprintf("%s.v%d", indexName, version) } -func (i *Indexer) createIndex(ctx context.Context) error { +func (i *IndexerV8) createIndex(ctx context.Context) error { createIndex, err := i.Client.Indices.Create(i.VersionedIndexName()).Request(&create8.Request{ Mappings: i.mapping, }).Do(ctx) @@ -42,7 +42,7 @@ func (i *Indexer) createIndex(ctx context.Context) error { return nil } -func (i *Indexer) initClient() (*elasticsearch8.TypedClient, error) { +func (i *IndexerV8) initClient() (*elasticsearch8.TypedClient, error) { cfg := elasticsearch8.Config{ Addresses: []string{i.url}, } @@ -56,7 +56,7 @@ func (i *Indexer) initClient() (*elasticsearch8.TypedClient, error) { return elasticsearch8.NewTypedClient(cfg) } -func (i *Indexer) checkOldIndexes(ctx context.Context) { +func (i *IndexerV8) checkOldIndexes(ctx context.Context) { for v := 0; v < i.version; v++ { indexName := versionedIndexName(i.indexName, v) exists, err := i.Client.Indices.Exists(indexName).Do(ctx) diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_v7.go b/modules/indexer/issues/elasticsearch/elasticsearch_v7.go new file mode 100644 index 0000000000000..967b2d0357f8d --- /dev/null +++ b/modules/indexer/issues/elasticsearch/elasticsearch_v7.go @@ -0,0 +1,289 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package elasticsearch + +import ( + "context" + "fmt" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/graceful" + indexer_internal "code.gitea.io/gitea/modules/indexer/internal" + inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" + "code.gitea.io/gitea/modules/indexer/issues/internal" + + "github.com/olivere/elastic/v7" +) + +const ( + // multi-match-types, currently only 2 types are used + // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types + esMultiMatchTypeBestFields = "best_fields" + esMultiMatchTypePhrasePrefix = "phrase_prefix" +) + +var _ internal.Indexer = &Indexer{} + +// Indexer implements Indexer interface +type Indexer struct { + inner *inner_elasticsearch.Indexer + indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much +} + +// NewIndexer creates a new elasticsearch indexer +func NewIndexer(url, indexerName string) *Indexer { + inner := inner_elasticsearch.NewIndexer(url, indexerName, issueIndexerLatestVersion, defaultMapping) + indexer := &Indexer{ + inner: inner, + Indexer: inner, + } + return indexer +} + +const ( + defaultMapping = ` +{ + "mappings": { + "properties": { + "id": { "type": "integer", "index": true }, + "repo_id": { "type": "integer", "index": true }, + "is_public": { "type": "boolean", "index": true }, + + "title": { "type": "text", "index": true }, + "content": { "type": "text", "index": true }, + "comments": { "type" : "text", "index": true }, + + "is_pull": { "type": "boolean", "index": true }, + "is_closed": { "type": "boolean", "index": true }, + "label_ids": { "type": "integer", "index": true }, + "no_label": { "type": "boolean", "index": true }, + "milestone_id": { "type": "integer", "index": true }, + "project_id": { "type": "integer", "index": true }, + "project_board_id": { "type": "integer", "index": true }, + "poster_id": { "type": "integer", "index": true }, + "assignee_id": { "type": "integer", "index": true }, + "mention_ids": { "type": "integer", "index": true }, + "reviewed_ids": { "type": "integer", "index": true }, + "review_requested_ids": { "type": "integer", "index": true }, + "subscriber_ids": { "type": "integer", "index": true }, + "updated_unix": { "type": "integer", "index": true }, + + "created_unix": { "type": "integer", "index": true }, + "deadline_unix": { "type": "integer", "index": true }, + "comment_count": { "type": "integer", "index": true } + } + } +} +` +) + +// Index will save the index data +func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) error { + if len(issues) == 0 { + return nil + } else if len(issues) == 1 { + issue := issues[0] + _, err := b.inner.Client.Index(). + Index(b.inner.VersionedIndexName()). + Id(fmt.Sprintf("%d", issue.ID)). + BodyJson(issue). + Do(ctx) + return err + } + + reqs := make([]elastic.BulkableRequest, 0) + for _, issue := range issues { + reqs = append(reqs, + elastic.NewBulkIndexRequest(). + Index(b.inner.VersionedIndexName()). + Id(fmt.Sprintf("%d", issue.ID)). + Doc(issue), + ) + } + + _, err := b.inner.Client.Bulk(). + Index(b.inner.VersionedIndexName()). + Add(reqs...). + Do(graceful.GetManager().HammerContext()) + return err +} + +// Delete deletes indexes by ids +func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { + if len(ids) == 0 { + return nil + } else if len(ids) == 1 { + _, err := b.inner.Client.Delete(). + Index(b.inner.VersionedIndexName()). + Id(fmt.Sprintf("%d", ids[0])). + Do(ctx) + return err + } + + reqs := make([]elastic.BulkableRequest, 0) + for _, id := range ids { + reqs = append(reqs, + elastic.NewBulkDeleteRequest(). + Index(b.inner.VersionedIndexName()). + Id(fmt.Sprintf("%d", id)), + ) + } + + _, err := b.inner.Client.Bulk(). + Index(b.inner.VersionedIndexName()). + Add(reqs...). + Do(graceful.GetManager().HammerContext()) + return err +} + +// Search searches for issues by given conditions. +// Returns the matching issue IDs +func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { + query := elastic.NewBoolQuery() + + if options.Keyword != "" { + searchType := esMultiMatchTypePhrasePrefix + if options.IsFuzzyKeyword { + searchType = esMultiMatchTypeBestFields + } + + query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(searchType)) + } + + if len(options.RepoIDs) > 0 { + q := elastic.NewBoolQuery() + q.Should(elastic.NewTermsQuery("repo_id", toAnySlice(options.RepoIDs)...)) + if options.AllPublic { + q.Should(elastic.NewTermQuery("is_public", true)) + } + query.Must(q) + } + + if options.IsPull.Has() { + query.Must(elastic.NewTermQuery("is_pull", options.IsPull.Value())) + } + if options.IsClosed.Has() { + query.Must(elastic.NewTermQuery("is_closed", options.IsClosed.Value())) + } + + if options.NoLabelOnly { + query.Must(elastic.NewTermQuery("no_label", true)) + } else { + if len(options.IncludedLabelIDs) > 0 { + q := elastic.NewBoolQuery() + for _, labelID := range options.IncludedLabelIDs { + q.Must(elastic.NewTermQuery("label_ids", labelID)) + } + query.Must(q) + } else if len(options.IncludedAnyLabelIDs) > 0 { + query.Must(elastic.NewTermsQuery("label_ids", toAnySlice(options.IncludedAnyLabelIDs)...)) + } + if len(options.ExcludedLabelIDs) > 0 { + q := elastic.NewBoolQuery() + for _, labelID := range options.ExcludedLabelIDs { + q.MustNot(elastic.NewTermQuery("label_ids", labelID)) + } + query.Must(q) + } + } + + if len(options.MilestoneIDs) > 0 { + query.Must(elastic.NewTermsQuery("milestone_id", toAnySlice(options.MilestoneIDs)...)) + } + + if options.ProjectID.Has() { + query.Must(elastic.NewTermQuery("project_id", options.ProjectID.Value())) + } + if options.ProjectColumnID.Has() { + query.Must(elastic.NewTermQuery("project_board_id", options.ProjectColumnID.Value())) + } + + if options.PosterID.Has() { + query.Must(elastic.NewTermQuery("poster_id", options.PosterID.Value())) + } + + if options.AssigneeID.Has() { + query.Must(elastic.NewTermQuery("assignee_id", options.AssigneeID.Value())) + } + + if options.MentionID.Has() { + query.Must(elastic.NewTermQuery("mention_ids", options.MentionID.Value())) + } + + if options.ReviewedID.Has() { + query.Must(elastic.NewTermQuery("reviewed_ids", options.ReviewedID.Value())) + } + if options.ReviewRequestedID.Has() { + query.Must(elastic.NewTermQuery("review_requested_ids", options.ReviewRequestedID.Value())) + } + + if options.SubscriberID.Has() { + query.Must(elastic.NewTermQuery("subscriber_ids", options.SubscriberID.Value())) + } + + if options.UpdatedAfterUnix.Has() || options.UpdatedBeforeUnix.Has() { + q := elastic.NewRangeQuery("updated_unix") + if options.UpdatedAfterUnix.Has() { + q.Gte(options.UpdatedAfterUnix.Value()) + } + if options.UpdatedBeforeUnix.Has() { + q.Lte(options.UpdatedBeforeUnix.Value()) + } + query.Must(q) + } + + if options.SortBy == "" { + options.SortBy = internal.SortByCreatedAsc + } + sortBy := []elastic.Sorter{ + parseSortBy(options.SortBy), + elastic.NewFieldSort("id").Desc(), + } + + // See https://stackoverflow.com/questions/35206409/elasticsearch-2-1-result-window-is-too-large-index-max-result-window/35221900 + // TODO: make it configurable since it's configurable in elasticsearch + const maxPageSize = 10000 + + skip, limit := indexer_internal.ParsePaginator(options.Paginator, maxPageSize) + searchResult, err := b.inner.Client.Search(). + Index(b.inner.VersionedIndexName()). + Query(query). + SortBy(sortBy...). + From(skip).Size(limit). + Do(ctx) + if err != nil { + return nil, err + } + + hits := make([]internal.Match, 0, limit) + for _, hit := range searchResult.Hits.Hits { + id, _ := strconv.ParseInt(hit.Id, 10, 64) + hits = append(hits, internal.Match{ + ID: id, + }) + } + + return &internal.SearchResult{ + Total: searchResult.TotalHits(), + Hits: hits, + }, nil +} + +func toAnySlice[T any](s []T) []any { + ret := make([]any, 0, len(s)) + for _, item := range s { + ret = append(ret, item) + } + return ret +} + +func parseSortBy(sortBy internal.SortBy) elastic.Sorter { + field := strings.TrimPrefix(string(sortBy), "-") + ret := elastic.NewFieldSort(field) + if strings.HasPrefix(string(sortBy), "-") { + ret.Desc() + } + return ret +} diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch_v8.go similarity index 92% rename from modules/indexer/issues/elasticsearch/elasticsearch.go rename to modules/indexer/issues/elasticsearch/elasticsearch_v8.go index e632588b9d587..d2375c7cc54b4 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_v8.go @@ -32,22 +32,22 @@ const ( var _ internal.Indexer = &Indexer{} // Indexer implements Indexer interface -type Indexer struct { - inner *inner_elasticsearch.Indexer +type IndexerV8 struct { + inner *inner_elasticsearch.IndexerV8 indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much } // NewIndexer creates a new elasticsearch indexer -func NewIndexer(url, indexerName string) *Indexer { - inner := inner_elasticsearch.NewIndexer(url, indexerName, issueIndexerLatestVersion, defaultMapping) - indexer := &Indexer{ +func NewIndexerV8(url, indexerName string) *IndexerV8 { + inner := inner_elasticsearch.NewIndexerV8(url, indexerName, issueIndexerLatestVersion, defaultMappingV8) + indexer := &IndexerV8{ inner: inner, Indexer: inner, } return indexer } -var defaultMapping = &types8.TypeMapping{ +var defaultMappingV8 = &types8.TypeMapping{ Properties: map[string]types8.Property{ "id": types8.NewIntegerNumberProperty(), "repo_id": types8.NewIntegerNumberProperty(), @@ -78,7 +78,7 @@ var defaultMapping = &types8.TypeMapping{ } // Index will save the index data -func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) error { +func (b *IndexerV8) Index(ctx context.Context, issues ...*internal.IndexerData) error { if len(issues) == 0 { return nil } else if len(issues) == 1 { @@ -109,7 +109,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er } // Delete deletes indexes by ids -func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { +func (b *IndexerV8) Delete(ctx context.Context, ids ...int64) error { if len(ids) == 0 { return nil } @@ -149,7 +149,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs -func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { +func (b *IndexerV8) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { query := &types8.Query{ Bool: &types8.BoolQuery{ Must: make([]types8.Query, 0), @@ -340,7 +340,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( if options.SortBy == "" { options.SortBy = internal.SortByCreatedAsc } - field, fieldSort := parseSortBy(options.SortBy) + field, fieldSort := parseSortByV8(options.SortBy) sort := []types8.SortCombinations{ &types8.SortOptions{SortOptions: map[string]types8.FieldSort{ field: fieldSort, @@ -377,15 +377,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( }, nil } -func toAnySlice[T any](s []T) []any { - ret := make([]any, 0, len(s)) - for _, item := range s { - ret = append(ret, item) - } - return ret -} - -func parseSortBy(sortBy internal.SortBy) (string, types8.FieldSort) { +func parseSortByV8(sortBy internal.SortBy) (string, types8.FieldSort) { field := strings.TrimPrefix(string(sortBy), "-") sort := types8.FieldSort{ Order: &sortorder8.Asc, diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_test.go b/modules/indexer/issues/elasticsearch/elasticsearch_v8_test.go similarity index 100% rename from modules/indexer/issues/elasticsearch/elasticsearch_test.go rename to modules/indexer/issues/elasticsearch/elasticsearch_v8_test.go From df246e997b2102d59f257cb6d3b7b1fb1a452335 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 21 Jul 2024 00:47:14 -0400 Subject: [PATCH 10/21] Config elastic version --- .golangci.yml | 3 +++ modules/indexer/code/elasticsearch/common.go | 16 ++++++++++++++ .../code/elasticsearch/elasticsearch_v7.go | 22 +++++++++---------- modules/indexer/code/indexer.go | 2 +- modules/indexer/code/indexer_test.go | 7 +++++- .../internal/elasticsearch/indexer_v7.go | 16 +++++++------- .../indexer/internal/elasticsearch/util_v7.go | 8 +++---- .../indexer/issues/elasticsearch/common.go | 16 ++++++++++++++ ...earch_v8_test.go => elasticsearch_test.go} | 16 +++++++++++++- .../issues/elasticsearch/elasticsearch_v7.go | 22 +++++++++---------- .../issues/elasticsearch/elasticsearch_v8.go | 2 +- modules/indexer/issues/indexer.go | 2 +- modules/setting/indexer.go | 4 ++++ 13 files changed, 97 insertions(+), 39 deletions(-) create mode 100644 modules/indexer/code/elasticsearch/common.go create mode 100644 modules/indexer/issues/elasticsearch/common.go rename modules/indexer/issues/elasticsearch/{elasticsearch_v8_test.go => elasticsearch_test.go} (69%) diff --git a/.golangci.yml b/.golangci.yml index 37617ad36529d..f53c6b113f603 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -139,3 +139,6 @@ issues: - text: "exitAfterDefer:" linters: - gocritic + - path: modules/indexer/code/elasticsearch/elasticsearch_*.go + linters: + - dupl # TODO: figure out a good way to avoid this diff --git a/modules/indexer/code/elasticsearch/common.go b/modules/indexer/code/elasticsearch/common.go new file mode 100644 index 0000000000000..04a895bfe7749 --- /dev/null +++ b/modules/indexer/code/elasticsearch/common.go @@ -0,0 +1,16 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package elasticsearch + +import ( + "code.gitea.io/gitea/modules/indexer/code/internal" +) + +// NewIndexer creates a new elasticsearch indexer +func NewIndexer(url, indexerName string, version int) internal.Indexer { + if version == 8 { + return NewIndexerV8(url, indexerName) + } + return NewIndexerV7(url, indexerName) +} diff --git a/modules/indexer/code/elasticsearch/elasticsearch_v7.go b/modules/indexer/code/elasticsearch/elasticsearch_v7.go index a74300f3fcacb..7458cd706dc7a 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch_v7.go +++ b/modules/indexer/code/elasticsearch/elasticsearch_v7.go @@ -35,18 +35,18 @@ const ( esMultiMatchTypePhrasePrefix = "phrase_prefix" ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = &IndexerV7{} // Indexer implements Indexer interface -type Indexer struct { - inner *inner_elasticsearch.Indexer +type IndexerV7 struct { + inner *inner_elasticsearch.IndexerV7 indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much } // NewIndexer creates a new elasticsearch indexer -func NewIndexer(url, indexerName string) *Indexer { - inner := inner_elasticsearch.NewIndexer(url, indexerName, esRepoIndexerLatestVersion, defaultMapping) - indexer := &Indexer{ +func NewIndexerV7(url, indexerName string) *IndexerV7 { + inner := inner_elasticsearch.NewIndexerV7(url, indexerName, esRepoIndexerLatestVersion, defaultMapping) + indexer := &IndexerV7{ inner: inner, Indexer: inner, } @@ -83,7 +83,7 @@ const ( }` ) -func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { +func (b *IndexerV7) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { // Ignore vendored files in code search if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { return nil, nil @@ -142,7 +142,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro }, nil } -func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elastic.BulkableRequest { +func (b *IndexerV7) addDelete(filename string, repo *repo_model.Repository) elastic.BulkableRequest { id := internal.FilenameIndexerID(repo.ID, filename) return elastic.NewBulkDeleteRequest(). Index(b.inner.VersionedIndexName()). @@ -150,7 +150,7 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elasti } // Index will save the index data -func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { +func (b *IndexerV7) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { reqs := make([]elastic.BulkableRequest, 0) if len(changes.Updates) > 0 { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! @@ -195,7 +195,7 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st } // Delete deletes indexes by ids -func (b *Indexer) Delete(ctx context.Context, repoID int64) error { +func (b *IndexerV7) Delete(ctx context.Context, repoID int64) error { _, err := b.inner.Client.DeleteByQuery(b.inner.VersionedIndexName()). Query(elastic.NewTermsQuery("repo_id", repoID)). Do(ctx) @@ -264,7 +264,7 @@ func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLan } // Search searches for codes and language stats by given conditions. -func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { +func (b *IndexerV7) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { searchType := esMultiMatchTypePhrasePrefix if opts.IsKeywordFuzzy { searchType = esMultiMatchTypeBestFields diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index c1ab26569c6f4..f3a54d39424d9 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -177,7 +177,7 @@ func Init() { } }() - rIndexer = elasticsearch.NewIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName) + rIndexer = elasticsearch.NewIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName, setting.Indexer.ElasticSearchVersion) existed, err = rIndexer.Init(ctx) if err != nil { cancel() diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index 8975c5ce4083b..6d410d0bc7416 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -124,7 +124,12 @@ func TestESIndexAndSearch(t *testing.T) { return } - indexer := elasticsearch.NewIndexer(u, "gitea_codes") + version := 7 + if os.Getenv("TEST_ELASTICSEARCH_VERSION") == "8" { + version = 8 + } + + indexer := elasticsearch.NewIndexer(u, "gitea_codes", version) if _, err := indexer.Init(context.Background()); err != nil { if indexer != nil { indexer.Close() diff --git a/modules/indexer/internal/elasticsearch/indexer_v7.go b/modules/indexer/internal/elasticsearch/indexer_v7.go index 395eea3bce652..0d8a494d48d82 100644 --- a/modules/indexer/internal/elasticsearch/indexer_v7.go +++ b/modules/indexer/internal/elasticsearch/indexer_v7.go @@ -12,10 +12,10 @@ import ( "github.com/olivere/elastic/v7" ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = &IndexerV7{} -// Indexer represents a basic elasticsearch indexer implementation -type Indexer struct { +// IndexerV7 represents a basic elasticsearch indexer implementation +type IndexerV7 struct { Client *elastic.Client url string @@ -24,8 +24,8 @@ type Indexer struct { mapping string } -func NewIndexer(url, indexName string, version int, mapping string) *Indexer { - return &Indexer{ +func NewIndexerV7(url, indexName string, version int, mapping string) *IndexerV7 { + return &IndexerV7{ url: url, indexName: indexName, version: version, @@ -34,7 +34,7 @@ func NewIndexer(url, indexName string, version int, mapping string) *Indexer { } // Init initializes the indexer -func (i *Indexer) Init(ctx context.Context) (bool, error) { +func (i *IndexerV7) Init(ctx context.Context) (bool, error) { if i == nil { return false, fmt.Errorf("cannot init nil indexer") } @@ -64,7 +64,7 @@ func (i *Indexer) Init(ctx context.Context) (bool, error) { } // Ping checks if the indexer is available -func (i *Indexer) Ping(ctx context.Context) error { +func (i *IndexerV7) Ping(ctx context.Context) error { if i == nil { return fmt.Errorf("cannot ping nil indexer") } @@ -85,7 +85,7 @@ func (i *Indexer) Ping(ctx context.Context) error { } // Close closes the indexer -func (i *Indexer) Close() { +func (i *IndexerV7) Close() { if i == nil { return } diff --git a/modules/indexer/internal/elasticsearch/util_v7.go b/modules/indexer/internal/elasticsearch/util_v7.go index 418d4ff5084da..40f21dd91362a 100644 --- a/modules/indexer/internal/elasticsearch/util_v7.go +++ b/modules/indexer/internal/elasticsearch/util_v7.go @@ -14,11 +14,11 @@ import ( ) // VersionedIndexName returns the full index name with version -func (i *Indexer) VersionedIndexName() string { +func (i *IndexerV7) VersionedIndexName() string { return versionedIndexName(i.indexName, i.version) } -func (i *Indexer) createIndex(ctx context.Context) error { +func (i *IndexerV7) createIndex(ctx context.Context) error { createIndex, err := i.Client.CreateIndex(i.VersionedIndexName()).BodyString(i.mapping).Do(ctx) if err != nil { return err @@ -32,7 +32,7 @@ func (i *Indexer) createIndex(ctx context.Context) error { return nil } -func (i *Indexer) initClient() (*elastic.Client, error) { +func (i *IndexerV7) initClient() (*elastic.Client, error) { opts := []elastic.ClientOptionFunc{ elastic.SetURL(i.url), elastic.SetSniff(false), @@ -49,7 +49,7 @@ func (i *Indexer) initClient() (*elastic.Client, error) { return elastic.NewClient(opts...) } -func (i *Indexer) checkOldIndexes(ctx context.Context) { +func (i *IndexerV7) checkOldIndexes(ctx context.Context) { for v := 0; v < i.version; v++ { indexName := versionedIndexName(i.indexName, v) exists, err := i.Client.IndexExists(indexName).Do(ctx) diff --git a/modules/indexer/issues/elasticsearch/common.go b/modules/indexer/issues/elasticsearch/common.go new file mode 100644 index 0000000000000..b3ea7e4c0ff1f --- /dev/null +++ b/modules/indexer/issues/elasticsearch/common.go @@ -0,0 +1,16 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package elasticsearch + +import ( + "code.gitea.io/gitea/modules/indexer/issues/internal" +) + +// NewIndexer creates a new elasticsearch indexer +func NewIndexer(url, indexerName string, version int) internal.Indexer { + if version == 8 { + return NewIndexerV8(url, indexerName) + } + return NewIndexerV7(url, indexerName) +} diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_v8_test.go b/modules/indexer/issues/elasticsearch/elasticsearch_test.go similarity index 69% rename from modules/indexer/issues/elasticsearch/elasticsearch_v8_test.go rename to modules/indexer/issues/elasticsearch/elasticsearch_test.go index ffd85b1aa1dbb..9879be41c8265 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch_v8_test.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "code.gitea.io/gitea/modules/indexer/issues/internal" "code.gitea.io/gitea/modules/indexer/issues/internal/tests" ) @@ -26,6 +27,11 @@ func TestElasticsearchIndexer(t *testing.T) { } } + version := 7 + if os.Getenv("TEST_ELASTICSEARCH_VERSION") == "8" { + version = 8 + } + ok := false for i := 0; i < 60; i++ { resp, err := http.Get(url) @@ -41,7 +47,15 @@ func TestElasticsearchIndexer(t *testing.T) { return } - indexer := NewIndexer(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix())) + var indexer internal.Indexer + switch version { + case 7: + indexer = NewIndexerV7(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix())) + case 8: + indexer = NewIndexerV8(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix())) + default: + t.Fatalf("Unsupported version %d", version) + } defer indexer.Close() tests.TestIndexer(t, indexer) diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_v7.go b/modules/indexer/issues/elasticsearch/elasticsearch_v7.go index 967b2d0357f8d..b7c42153b50ab 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch_v7.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_v7.go @@ -24,18 +24,18 @@ const ( esMultiMatchTypePhrasePrefix = "phrase_prefix" ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = &IndexerV7{} -// Indexer implements Indexer interface -type Indexer struct { - inner *inner_elasticsearch.Indexer +// IndexerV7 implements Indexer interface +type IndexerV7 struct { + inner *inner_elasticsearch.IndexerV7 indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much } -// NewIndexer creates a new elasticsearch indexer -func NewIndexer(url, indexerName string) *Indexer { - inner := inner_elasticsearch.NewIndexer(url, indexerName, issueIndexerLatestVersion, defaultMapping) - indexer := &Indexer{ +// NewIndexerV7 creates a new elasticsearch indexer +func NewIndexerV7(url, indexerName string) *IndexerV7 { + inner := inner_elasticsearch.NewIndexerV7(url, indexerName, issueIndexerLatestVersion, defaultMapping) + indexer := &IndexerV7{ inner: inner, Indexer: inner, } @@ -80,7 +80,7 @@ const ( ) // Index will save the index data -func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) error { +func (b *IndexerV7) Index(ctx context.Context, issues ...*internal.IndexerData) error { if len(issues) == 0 { return nil } else if len(issues) == 1 { @@ -111,7 +111,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er } // Delete deletes indexes by ids -func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { +func (b *IndexerV7) Delete(ctx context.Context, ids ...int64) error { if len(ids) == 0 { return nil } else if len(ids) == 1 { @@ -140,7 +140,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs -func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { +func (b *IndexerV7) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { query := elastic.NewBoolQuery() if options.Keyword != "" { diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_v8.go b/modules/indexer/issues/elasticsearch/elasticsearch_v8.go index d2375c7cc54b4..587a874fda8e1 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch_v8.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_v8.go @@ -29,7 +29,7 @@ const ( issueIndexerLatestVersion = 1 ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = &IndexerV8{} // Indexer implements Indexer interface type IndexerV8 struct { diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index c82dc0867ea4c..de4962b689b21 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -96,7 +96,7 @@ func InitIssueIndexer(syncReindex bool) { log.Fatal("Unable to initialize Bleve Issue Indexer at path: %s Error: %v", setting.Indexer.IssuePath, err) } case "elasticsearch": - issueIndexer = elasticsearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueIndexerName) + issueIndexer = elasticsearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueIndexerName, setting.Indexer.ElasticSearchVersion) existed, err = issueIndexer.Init(ctx) if err != nil { log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err) diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 18585602c3dd2..d39446113e1ed 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -21,6 +21,8 @@ var Indexer = struct { IssueIndexerName string StartupTimeout time.Duration + ElasticSearchVersion int + RepoIndexerEnabled bool RepoIndexerRepoTypes []string RepoType string @@ -88,6 +90,8 @@ func loadIndexerFrom(rootCfg ConfigProvider) { Indexer.ExcludeVendored = sec.Key("REPO_INDEXER_EXCLUDE_VENDORED").MustBool(true) Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024) Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second) + + Indexer.ElasticSearchVersion = sec.Key("ELASTICSEARCH_VERSION").MustInt(7) } // IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing From 6268ea48ff30bce2313ea89cba3574a48084926c Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 21 Jul 2024 00:48:31 -0400 Subject: [PATCH 11/21] Bump elastic/go-elasticsearch/v8 to 8.14.0 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5e94a69b09f01..3b1d75b1180f4 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 github.com/dustin/go-humanize v1.0.1 github.com/editorconfig/editorconfig-core-go/v2 v2.6.2 - github.com/elastic/go-elasticsearch/v8 v8.13.1 + github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/emersion/go-imap v1.2.1 github.com/emirpasic/gods v1.18.1 github.com/ethantkoenig/rupture v1.0.1 @@ -180,7 +180,7 @@ require ( github.com/davidmz/go-pageant v1.0.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.11.0 // indirect - github.com/elastic/elastic-transport-go/v8 v8.5.0 // indirect + github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43 // indirect github.com/fatih/color v1.17.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect diff --git a/go.sum b/go.sum index a4ada1092071b..711d9af91f1e6 100644 --- a/go.sum +++ b/go.sum @@ -246,10 +246,10 @@ github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJ github.com/editorconfig/editorconfig-core-go/v2 v2.6.2 h1:dKG8sc7n321deIVRcQtwlMNoBEra7j0qQ8RwxO8RN0w= github.com/editorconfig/editorconfig-core-go/v2 v2.6.2/go.mod h1:7dvD3GCm7eBw53xZ/lsiq72LqobdMg3ITbMBxnmJmqY= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= -github.com/elastic/elastic-transport-go/v8 v8.5.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= -github.com/elastic/go-elasticsearch/v8 v8.13.1 h1:du5F8IzUUyCkzxyHdrO9AtopcG95I/qwi2WK8Kf1xlg= -github.com/elastic/go-elasticsearch/v8 v8.13.1/go.mod h1:DIn7HopJs4oZC/w0WoJR13uMUxtHeq92eI5bqv5CRfI= +github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= +github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= +github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow= +github.com/elastic/go-elasticsearch/v8 v8.14.0/go.mod h1:WRvnlGkSuZyp83M2U8El/LGXpCjYLrvlkSgkAH4O5I4= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= From 9c791d100f0385ee166fa5227ee89f33d8d4e8f5 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 21 Jul 2024 00:52:18 -0400 Subject: [PATCH 12/21] Update licenses json --- assets/go-licenses.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index a84c9568be106..a2b98eba02fd2 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -859,6 +859,16 @@ "path": "github.com/olekukonko/tablewriter/LICENSE.md", "licenseText": "Copyright (C) 2014 by Oleku Konko\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n" }, + { + "name": "github.com/olivere/elastic/v7", + "path": "github.com/olivere/elastic/v7/LICENSE", + "licenseText": "The MIT License (MIT)\nCopyright © 2012-2015 Oliver Eilhard\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the “Software”), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included\nin all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\nFROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\nIN THE SOFTWARE.\n" + }, + { + "name": "github.com/olivere/elastic/v7/uritemplates", + "path": "github.com/olivere/elastic/v7/uritemplates/LICENSE", + "licenseText": "Copyright (c) 2013 Joshua Tacoma\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" + }, { "name": "github.com/opencontainers/go-digest", "path": "github.com/opencontainers/go-digest/LICENSE", From b556945ad81830a40bc318046fca3ce518ab9bdb Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 21 Jul 2024 01:04:01 -0400 Subject: [PATCH 13/21] Fix type errors --- modules/indexer/code/elasticsearch/elasticsearch_v8.go | 6 +++--- modules/indexer/issues/elasticsearch/elasticsearch_v8.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/indexer/code/elasticsearch/elasticsearch_v8.go b/modules/indexer/code/elasticsearch/elasticsearch_v8.go index 8fda07b9675f5..3e10ed70b8c5f 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch_v8.go +++ b/modules/indexer/code/elasticsearch/elasticsearch_v8.go @@ -224,7 +224,7 @@ func convertResultV8(searchResult *searchV8.Response, kw string, pageSize int) ( panic(fmt.Sprintf("2===%#v", hit.Highlight)) } - repoID, fileName := internal.ParseIndexerID(hit.Id_) + repoID, fileName := internal.ParseIndexerID(*hit.Id_) res := make(map[string]any) if err := json.Unmarshal(hit.Source_, &res); err != nil { return 0, nil, nil, err @@ -328,7 +328,7 @@ func (b *IndexerV8) Search(ctx context.Context, opts *internal.SearchOptions) (i Fields: map[string]typesV8.HighlightField{ "content": { NumberOfFragments: someV8.Int(0), // return all highting content on fragments - Type: &highlightertypeV8.Fvh, + Type: &highlightertypeV8.Fastvector, }, }, }, @@ -369,7 +369,7 @@ func (b *IndexerV8) Search(ctx context.Context, opts *internal.SearchOptions) (i Fields: map[string]typesV8.HighlightField{ "content": { NumberOfFragments: someV8.Int(0), // return all highting content on fragments - Type: &highlightertypeV8.Fvh, + Type: &highlightertypeV8.Fastvector, }, }, }, diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_v8.go b/modules/indexer/issues/elasticsearch/elasticsearch_v8.go index 587a874fda8e1..0e09f339fb9c2 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch_v8.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_v8.go @@ -365,7 +365,7 @@ func (b *IndexerV8) Search(ctx context.Context, options *internal.SearchOptions) hits := make([]internal.Match, 0, limit) for _, hit := range searchResult.Hits.Hits { - id, _ := strconv.ParseInt(hit.Id_, 10, 64) + id, _ := strconv.ParseInt(*hit.Id_, 10, 64) hits = append(hits, internal.Match{ ID: id, }) From 3de4b57655dadd0b68c0f81e1cc3e54d05b487ad Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 21 Jul 2024 01:25:25 -0400 Subject: [PATCH 14/21] Add elastic 8.14.0 to pssql test --- .github/workflows/pull-db-tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 246884f24b1bd..1a6d6b41d552a 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -37,6 +37,12 @@ jobs: MINIO_ROOT_PASSWORD: 12345678 ports: - "9000:9000" + elasticsearch: + image: elasticsearch:8.14.0 + env: + discovery.type: single-node + ports: + - "9200:9200" steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 @@ -60,6 +66,7 @@ jobs: TEST_TAGS: gogit TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 + TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" test-sqlite: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' From c378649d1057bd0aa2e8a99abd119cdb168b24c2 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 21 Jul 2024 17:23:13 -0400 Subject: [PATCH 15/21] Provide test elastic 8 env --- .github/workflows/pull-db-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 1a6d6b41d552a..5b252a0ebb14f 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -67,6 +67,7 @@ jobs: TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" + TEST_ELASTICSEARCH_VERSION: 8 test-sqlite: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' From 35e7ab00bc6db5cb83b25e1fa0156118bc9a52a7 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 23 Jul 2024 23:05:32 -0400 Subject: [PATCH 16/21] Detect elastic version --- modules/indexer/code/elasticsearch/common.go | 12 +++-- modules/indexer/code/indexer.go | 8 ++- modules/indexer/code/indexer_test.go | 8 ++- .../indexer/internal/elasticsearch/common.go | 54 +++++++++++++++++++ .../internal/elasticsearch/common_test.go | 49 +++++++++++++++++ .../indexer/issues/elasticsearch/common.go | 12 +++-- .../elasticsearch/elasticsearch_test.go | 17 ++---- modules/indexer/issues/indexer.go | 5 +- modules/setting/indexer.go | 4 -- 9 files changed, 138 insertions(+), 31 deletions(-) create mode 100644 modules/indexer/internal/elasticsearch/common.go create mode 100644 modules/indexer/internal/elasticsearch/common_test.go diff --git a/modules/indexer/code/elasticsearch/common.go b/modules/indexer/code/elasticsearch/common.go index 04a895bfe7749..00ee2fef3b4c7 100644 --- a/modules/indexer/code/elasticsearch/common.go +++ b/modules/indexer/code/elasticsearch/common.go @@ -5,12 +5,18 @@ package elasticsearch import ( "code.gitea.io/gitea/modules/indexer/code/internal" + inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" ) // NewIndexer creates a new elasticsearch indexer -func NewIndexer(url, indexerName string, version int) internal.Indexer { +func NewIndexer(url, indexerName string) (internal.Indexer, error) { + version, err := inner_elasticsearch.DetectVersion(url) + if err != nil { + return nil, err + } + if version == 8 { - return NewIndexerV8(url, indexerName) + return NewIndexerV8(url, indexerName), nil } - return NewIndexerV7(url, indexerName) + return NewIndexerV7(url, indexerName), nil } diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index f3a54d39424d9..eaa327a5eccff 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -177,7 +177,13 @@ func Init() { } }() - rIndexer = elasticsearch.NewIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName, setting.Indexer.ElasticSearchVersion) + rIndexer, err = elasticsearch.NewIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName) + if err != nil { + cancel() + (*globalIndexer.Load()).Close() + close(waitChannel) + log.Fatal("PID: %d Unable to initialize the elasticsearch Repository Indexer connstr: %s Error: %v", os.Getpid(), setting.Indexer.RepoConnStr, err) + } existed, err = rIndexer.Init(ctx) if err != nil { cancel() diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index 6d410d0bc7416..13061513a6204 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -124,12 +124,10 @@ func TestESIndexAndSearch(t *testing.T) { return } - version := 7 - if os.Getenv("TEST_ELASTICSEARCH_VERSION") == "8" { - version = 8 + indexer, err := elasticsearch.NewIndexer(u, "gitea_codes") + if err != nil { + assert.FailNow(t, "Unable to create ES indexer Error: %v", err) } - - indexer := elasticsearch.NewIndexer(u, "gitea_codes", version) if _, err := indexer.Init(context.Background()); err != nil { if indexer != nil { indexer.Close() diff --git a/modules/indexer/internal/elasticsearch/common.go b/modules/indexer/internal/elasticsearch/common.go new file mode 100644 index 0000000000000..d3d644451c357 --- /dev/null +++ b/modules/indexer/internal/elasticsearch/common.go @@ -0,0 +1,54 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package elasticsearch + +import ( + "errors" + "io" + "net/http" + "strings" + "time" + + "code.gitea.io/gitea/modules/json" +) + +// elasticRootResponse contains a subset of the response usable for version detection. +type elasticRootResponse struct { + Version struct { + Number string `json:"number"` + } +} + +// DetectVersion detects the major version of the elasticsearch server. +// Currently only supports version 7 and 8. +func DetectVersion(url string) (int, error) { + client := &http.Client{ + Timeout: 5 * time.Second, + } + resp, err := client.Get(url) + if err != nil { + return 0, err + } + defer resp.Body.Close() + return parseElasticVersion(resp.Body) +} + +func parseElasticVersion(body io.Reader) (int, error) { + var root elasticRootResponse + if err := json.NewDecoder(body).Decode(&root); err != nil { + return 0, err + } + + majorStr, _, ok := strings.Cut(root.Version.Number, ".") + if !ok { + return 0, errors.New("invalid version number") + } + + if majorStr == "8" { + return 8, nil + } else if majorStr == "7" { + return 7, nil + } + return 0, errors.New("unsupported ElasticSearch version") +} diff --git a/modules/indexer/internal/elasticsearch/common_test.go b/modules/indexer/internal/elasticsearch/common_test.go new file mode 100644 index 0000000000000..459539a017907 --- /dev/null +++ b/modules/indexer/internal/elasticsearch/common_test.go @@ -0,0 +1,49 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package elasticsearch + +import ( + "strings" + "testing" +) + +func TestParseElasticVersion(t *testing.T) { + tests := []struct { + content string + version int + hasError bool + }{ + { + content: `{"name":"instance-0000000000","cluster_name":"my_test_cluster","version":{"number":"7.0.0"}}`, + version: 7, + }, + { + content: `{"version":{"number":"8.12.1"}}`, + version: 8, + }, + { + content: `{"version":{"number":"6.0.0"}}`, + version: 0, + hasError: true, + }, + { + content: ``, + version: 0, + hasError: true, + }, + } + + for _, test := range tests { + version, err := parseElasticVersion(strings.NewReader(test.content)) + if test.hasError && err == nil { + t.Errorf("Expected error but got nil") + } + if !test.hasError && err != nil { + t.Errorf("Expected no error but got %v", err) + } + if version != test.version { + t.Errorf("Expected version %d but got %d", test.version, version) + } + } +} diff --git a/modules/indexer/issues/elasticsearch/common.go b/modules/indexer/issues/elasticsearch/common.go index b3ea7e4c0ff1f..08e203e02845a 100644 --- a/modules/indexer/issues/elasticsearch/common.go +++ b/modules/indexer/issues/elasticsearch/common.go @@ -4,13 +4,19 @@ package elasticsearch import ( + inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" "code.gitea.io/gitea/modules/indexer/issues/internal" ) // NewIndexer creates a new elasticsearch indexer -func NewIndexer(url, indexerName string, version int) internal.Indexer { +func NewIndexer(url, indexerName string) (internal.Indexer, error) { + version, err := inner_elasticsearch.DetectVersion(url) + if err != nil { + return nil, err + } + if version == 8 { - return NewIndexerV8(url, indexerName) + return NewIndexerV8(url, indexerName), nil } - return NewIndexerV7(url, indexerName) + return NewIndexerV7(url, indexerName), nil } diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_test.go b/modules/indexer/issues/elasticsearch/elasticsearch_test.go index 9879be41c8265..e23242b21c3e3 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch_test.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "code.gitea.io/gitea/modules/indexer/issues/internal" "code.gitea.io/gitea/modules/indexer/issues/internal/tests" ) @@ -27,11 +26,6 @@ func TestElasticsearchIndexer(t *testing.T) { } } - version := 7 - if os.Getenv("TEST_ELASTICSEARCH_VERSION") == "8" { - version = 8 - } - ok := false for i := 0; i < 60; i++ { resp, err := http.Get(url) @@ -47,14 +41,9 @@ func TestElasticsearchIndexer(t *testing.T) { return } - var indexer internal.Indexer - switch version { - case 7: - indexer = NewIndexerV7(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix())) - case 8: - indexer = NewIndexerV8(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix())) - default: - t.Fatalf("Unsupported version %d", version) + indexer, err := NewIndexer(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix())) + if err != nil { + t.Fatalf("Failed to create indexer: %v", err) } defer indexer.Close() diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index de4962b689b21..0d1c196b90bcf 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -96,7 +96,10 @@ func InitIssueIndexer(syncReindex bool) { log.Fatal("Unable to initialize Bleve Issue Indexer at path: %s Error: %v", setting.Indexer.IssuePath, err) } case "elasticsearch": - issueIndexer = elasticsearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueIndexerName, setting.Indexer.ElasticSearchVersion) + issueIndexer, err = elasticsearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueIndexerName) + if err != nil { + log.Fatal("Unable to initialize ElasticSearch issue indexer at connection %s Error: %v", setting.Indexer.IssueConnStr, err) + } existed, err = issueIndexer.Init(ctx) if err != nil { log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err) diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index d39446113e1ed..18585602c3dd2 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -21,8 +21,6 @@ var Indexer = struct { IssueIndexerName string StartupTimeout time.Duration - ElasticSearchVersion int - RepoIndexerEnabled bool RepoIndexerRepoTypes []string RepoType string @@ -90,8 +88,6 @@ func loadIndexerFrom(rootCfg ConfigProvider) { Indexer.ExcludeVendored = sec.Key("REPO_INDEXER_EXCLUDE_VENDORED").MustBool(true) Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024) Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second) - - Indexer.ElasticSearchVersion = sec.Key("ELASTICSEARCH_VERSION").MustInt(7) } // IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing From facb38e4597278dda654608a5b0e547b556d16a1 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 23 Jul 2024 23:06:48 -0400 Subject: [PATCH 17/21] Use correct json tag for elastic response --- modules/indexer/internal/elasticsearch/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexer/internal/elasticsearch/common.go b/modules/indexer/internal/elasticsearch/common.go index d3d644451c357..ba5ef60df0bef 100644 --- a/modules/indexer/internal/elasticsearch/common.go +++ b/modules/indexer/internal/elasticsearch/common.go @@ -17,7 +17,7 @@ import ( type elasticRootResponse struct { Version struct { Number string `json:"number"` - } + } `json:"version"` } // DetectVersion detects the major version of the elasticsearch server. From 14633e674a974086d903efe39f638b3a1c84fa87 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 24 Jul 2024 00:00:12 -0400 Subject: [PATCH 18/21] Test version detection --- .../internal/elasticsearch/common_test.go | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/modules/indexer/internal/elasticsearch/common_test.go b/modules/indexer/internal/elasticsearch/common_test.go index 459539a017907..adc256e1974c8 100644 --- a/modules/indexer/internal/elasticsearch/common_test.go +++ b/modules/indexer/internal/elasticsearch/common_test.go @@ -4,10 +4,37 @@ package elasticsearch import ( + "io" + "net/http" + "net/http/httptest" "strings" "testing" ) +func TestDetectVersion(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, `{"version":{"number":"8.12.1"}}`) + })) + defer server.Close() + + version, err := DetectVersion(server.URL) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if version != 8 { + t.Errorf("Expected version 8 but got %d", version) + } + + // error + version, err = DetectVersion("http://not-found:1234") + if err == nil { + t.Errorf("Expected error but got nil") + } + if version != 0 { + t.Errorf("Expected version 0 but got %d", version) + } +} + func TestParseElasticVersion(t *testing.T) { tests := []struct { content string @@ -27,6 +54,11 @@ func TestParseElasticVersion(t *testing.T) { version: 0, hasError: true, }, + { + content: `{"version":{"number":"7-0-0"}}`, + version: 0, + hasError: true, + }, { content: ``, version: 0, From 7dac80c24ad4aea544065508e75f484f6c4a597b Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 24 Jul 2024 00:01:51 -0400 Subject: [PATCH 19/21] Remove unnessesary env in CI --- .github/workflows/pull-db-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 5b252a0ebb14f..1a6d6b41d552a 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -67,7 +67,6 @@ jobs: TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" - TEST_ELASTICSEARCH_VERSION: 8 test-sqlite: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' From 9ee79aa0164e2352a849d10b9e2fb131bbbb7362 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 26 Jul 2024 01:17:17 -0400 Subject: [PATCH 20/21] Fix elastic rest api connection issue --- .github/workflows/pull-db-tests.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 1a6d6b41d552a..97a5871b7292f 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -38,9 +38,10 @@ jobs: ports: - "9000:9000" elasticsearch: - image: elasticsearch:8.14.0 + image: elasticsearch:8.14.3 env: discovery.type: single-node + xpack.security.enabled: false ports: - "9200:9200" steps: @@ -50,7 +51,7 @@ jobs: go-version-file: go.mod check-latest: true - name: Add hosts to /etc/hosts - run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts' + run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio elasticsearch" | sudo tee -a /etc/hosts' - run: make deps-backend - run: make backend env: @@ -168,9 +169,11 @@ jobs: ports: - "3306:3306" elasticsearch: - image: elasticsearch:7.5.0 + image: elasticsearch:8.14.3 env: discovery.type: single-node + xpack.security.enabled: false + # ELASTIC_PASSWORD: changeme # TODO: test with tls and auth enabled ports: - "9200:9200" smtpimap: From c0c35f52dc8a78c648126961de8c30d076136b7f Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 26 Jul 2024 01:18:45 -0400 Subject: [PATCH 21/21] Prepare rest request with basic auth --- .../indexer/internal/elasticsearch/common.go | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/modules/indexer/internal/elasticsearch/common.go b/modules/indexer/internal/elasticsearch/common.go index ba5ef60df0bef..3e665a79beb10 100644 --- a/modules/indexer/internal/elasticsearch/common.go +++ b/modules/indexer/internal/elasticsearch/common.go @@ -4,9 +4,12 @@ package elasticsearch import ( + "crypto/tls" "errors" + "fmt" "io" "net/http" + "net/url" "strings" "time" @@ -22,13 +25,33 @@ type elasticRootResponse struct { // DetectVersion detects the major version of the elasticsearch server. // Currently only supports version 7 and 8. -func DetectVersion(url string) (int, error) { +func DetectVersion(connStr string) (int, error) { + u, err := url.Parse(connStr) + if err != nil { + return 0, fmt.Errorf("failed to parse url: %v", err) + } + client := &http.Client{ Timeout: 5 * time.Second, } - resp, err := client.Get(url) + if u.Scheme == "https" { + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return 0, fmt.Errorf("failed to create request: %v", err) + } + pass, ok := u.User.Password() + if ok { + req.SetBasicAuth(u.User.Username(), pass) + } + + resp, err := client.Do(req) if err != nil { - return 0, err + return 0, fmt.Errorf("failed to get response: %v", err) } defer resp.Body.Close() return parseElasticVersion(resp.Body)