mirror of
https://github.com/go-gitea/gitea
synced 2024-09-11 00:47:00 +02:00
Refactor `modules/indexer` to make it more maintainable. And it can be easier to support more features. I'm trying to solve some of issue searching, this is a precursor to making functional changes. Current supported engines and the index versions: | engines | issues | code | | - | - | - | | db | Just a wrapper for database queries, doesn't need version | - | | bleve | The version of index is **2** | The version of index is **6** | | elasticsearch | The old index has no version, will be treated as version **0** in this PR | The version of index is **1** | | meilisearch | The old index has no version, will be treated as version **0** in this PR | - | ## Changes ### Split Splited it into mutiple packages ```text indexer ├── internal │ ├── bleve │ ├── db │ ├── elasticsearch │ └── meilisearch ├── code │ ├── bleve │ ├── elasticsearch │ └── internal └── issues ├── bleve ├── db ├── elasticsearch ├── internal └── meilisearch ``` - `indexer/interanal`: Internal shared package for indexer. - `indexer/interanal/[engine]`: Internal shared package for each engine (bleve/db/elasticsearch/meilisearch). - `indexer/code`: Implementations for code indexer. - `indexer/code/internal`: Internal shared package for code indexer. - `indexer/code/[engine]`: Implementation via each engine for code indexer. - `indexer/issues`: Implementations for issues indexer. ### Deduplication - Combine `Init/Ping/Close` for code indexer and issues indexer. - ~Combine `issues.indexerHolder` and `code.wrappedIndexer` to `internal.IndexHolder`.~ Remove it, use dummy indexer instead when the indexer is not ready. - Duplicate two copies of creating ES clients. - Duplicate two copies of `indexerID()`. ### Enhancement - [x] Support index version for elasticsearch issues indexer, the old index without version will be treated as version 0. - [x] Fix spell of `elastic_search/ElasticSearch`, it should be `Elasticsearch`. - [x] Improve versioning of ES index. We don't need `Aliases`: - Gitea does't need aliases for "Zero Downtime" because it never delete old indexes. - The old code of issues indexer uses the orignal name to create issue index, so it's tricky to convert it to an alias. - [x] Support index version for meilisearch issues indexer, the old index without version will be treated as version 0. - [x] Do "ping" only when `Ping` has been called, don't ping periodically and cache the status. - [x] Support the context parameter whenever possible. - [x] Fix outdated example config. - [x] Give up the requeue logic of issues indexer: When indexing fails, call Ping to check if it was caused by the engine being unavailable, and only requeue the task if the engine is unavailable. - It is fragile and tricky, could cause data losing (It did happen when I was doing some tests for this PR). And it works for ES only. - Just always requeue the failed task, if it caused by bad data, it's a bug of Gitea which should be fixed. --------- Co-authored-by: Giteabot <teabot@gitea.io>
329 lines
10 KiB
Go
329 lines
10 KiB
Go
// Copyright 2016 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package code
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"runtime/pprof"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"code.gitea.io/gitea/models/db"
|
|
repo_model "code.gitea.io/gitea/models/repo"
|
|
"code.gitea.io/gitea/modules/graceful"
|
|
"code.gitea.io/gitea/modules/indexer/code/bleve"
|
|
"code.gitea.io/gitea/modules/indexer/code/elasticsearch"
|
|
"code.gitea.io/gitea/modules/indexer/code/internal"
|
|
"code.gitea.io/gitea/modules/log"
|
|
"code.gitea.io/gitea/modules/process"
|
|
"code.gitea.io/gitea/modules/queue"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
"code.gitea.io/gitea/modules/util"
|
|
)
|
|
|
|
var (
|
|
indexerQueue *queue.WorkerPoolQueue[*internal.IndexerData]
|
|
// globalIndexer is the global indexer, it cannot be nil.
|
|
// When the real indexer is not ready, it will be a dummy indexer which will return error to explain it's not ready.
|
|
// So it's always safe use it as *globalIndexer.Load() and call its methods.
|
|
globalIndexer atomic.Pointer[internal.Indexer]
|
|
dummyIndexer *internal.Indexer
|
|
)
|
|
|
|
func init() {
|
|
i := internal.NewDummyIndexer()
|
|
dummyIndexer = &i
|
|
globalIndexer.Store(dummyIndexer)
|
|
}
|
|
|
|
func index(ctx context.Context, indexer internal.Indexer, repoID int64) error {
|
|
repo, err := repo_model.GetRepositoryByID(ctx, repoID)
|
|
if repo_model.IsErrRepoNotExist(err) {
|
|
return indexer.Delete(ctx, repoID)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
repoTypes := setting.Indexer.RepoIndexerRepoTypes
|
|
|
|
if len(repoTypes) == 0 {
|
|
repoTypes = []string{"sources"}
|
|
}
|
|
|
|
// skip forks from being indexed if unit is not present
|
|
if !util.SliceContains(repoTypes, "forks") && repo.IsFork {
|
|
return nil
|
|
}
|
|
|
|
// skip mirrors from being indexed if unit is not present
|
|
if !util.SliceContains(repoTypes, "mirrors") && repo.IsMirror {
|
|
return nil
|
|
}
|
|
|
|
// skip templates from being indexed if unit is not present
|
|
if !util.SliceContains(repoTypes, "templates") && repo.IsTemplate {
|
|
return nil
|
|
}
|
|
|
|
// skip regular repos from being indexed if unit is not present
|
|
if !util.SliceContains(repoTypes, "sources") && !repo.IsFork && !repo.IsMirror && !repo.IsTemplate {
|
|
return nil
|
|
}
|
|
|
|
sha, err := getDefaultBranchSha(ctx, repo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
changes, err := getRepoChanges(ctx, repo, sha)
|
|
if err != nil {
|
|
return err
|
|
} else if changes == nil {
|
|
return nil
|
|
}
|
|
|
|
if err := indexer.Index(ctx, repo, sha, changes); err != nil {
|
|
return err
|
|
}
|
|
|
|
return repo_model.UpdateIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode, sha)
|
|
}
|
|
|
|
// Init initialize the repo indexer
|
|
func Init() {
|
|
if !setting.Indexer.RepoIndexerEnabled {
|
|
(*globalIndexer.Load()).Close()
|
|
return
|
|
}
|
|
|
|
ctx, cancel, finished := process.GetManager().AddTypedContext(context.Background(), "Service: CodeIndexer", process.SystemProcessType, false)
|
|
|
|
graceful.GetManager().RunAtTerminate(func() {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
cancel()
|
|
log.Debug("Closing repository indexer")
|
|
(*globalIndexer.Load()).Close()
|
|
log.Info("PID: %d Repository Indexer closed", os.Getpid())
|
|
finished()
|
|
})
|
|
|
|
waitChannel := make(chan time.Duration, 1)
|
|
|
|
// Create the Queue
|
|
switch setting.Indexer.RepoType {
|
|
case "bleve", "elasticsearch":
|
|
handler := func(items ...*internal.IndexerData) (unhandled []*internal.IndexerData) {
|
|
indexer := *globalIndexer.Load()
|
|
for _, indexerData := range items {
|
|
log.Trace("IndexerData Process Repo: %d", indexerData.RepoID)
|
|
|
|
// FIXME: it seems there is a bug in `CatFileBatch` or `nio.Pipe`, which will cause the process to hang forever in rare cases
|
|
/*
|
|
sync.(*Cond).Wait(cond.go:70)
|
|
github.com/djherbis/nio/v3.(*PipeReader).Read(sync.go:106)
|
|
bufio.(*Reader).fill(bufio.go:106)
|
|
bufio.(*Reader).ReadSlice(bufio.go:372)
|
|
bufio.(*Reader).collectFragments(bufio.go:447)
|
|
bufio.(*Reader).ReadString(bufio.go:494)
|
|
code.gitea.io/gitea/modules/git.ReadBatchLine(batch_reader.go:149)
|
|
code.gitea.io/gitea/modules/indexer/code.(*BleveIndexer).addUpdate(bleve.go:214)
|
|
code.gitea.io/gitea/modules/indexer/code.(*BleveIndexer).Index(bleve.go:296)
|
|
code.gitea.io/gitea/modules/indexer/code.(*wrappedIndexer).Index(wrapped.go:74)
|
|
code.gitea.io/gitea/modules/indexer/code.index(indexer.go:105)
|
|
*/
|
|
if err := index(ctx, indexer, indexerData.RepoID); err != nil {
|
|
unhandled = append(unhandled, indexerData)
|
|
if !setting.IsInTesting {
|
|
log.Error("Codes indexer handler: index error for repo %v: %v", indexerData.RepoID, err)
|
|
}
|
|
}
|
|
}
|
|
return unhandled
|
|
}
|
|
|
|
indexerQueue = queue.CreateUniqueQueue(ctx, "code_indexer", handler)
|
|
if indexerQueue == nil {
|
|
log.Fatal("Unable to create codes indexer queue")
|
|
}
|
|
default:
|
|
log.Fatal("Unknown codes indexer type; %s", setting.Indexer.RepoType)
|
|
}
|
|
|
|
go func() {
|
|
pprof.SetGoroutineLabels(ctx)
|
|
start := time.Now()
|
|
var (
|
|
rIndexer internal.Indexer
|
|
existed bool
|
|
err error
|
|
)
|
|
switch setting.Indexer.RepoType {
|
|
case "bleve":
|
|
log.Info("PID: %d Initializing Repository Indexer at: %s", os.Getpid(), setting.Indexer.RepoPath)
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
log.Error("PANIC whilst initializing repository indexer: %v\nStacktrace: %s", err, log.Stack(2))
|
|
log.Error("The indexer files are likely corrupted and may need to be deleted")
|
|
log.Error("You can completely remove the \"%s\" directory to make Gitea recreate the indexes", setting.Indexer.RepoPath)
|
|
}
|
|
}()
|
|
|
|
rIndexer = bleve.NewIndexer(setting.Indexer.RepoPath)
|
|
existed, err = rIndexer.Init(ctx)
|
|
if err != nil {
|
|
cancel()
|
|
(*globalIndexer.Load()).Close()
|
|
close(waitChannel)
|
|
log.Fatal("PID: %d Unable to initialize the bleve Repository Indexer at path: %s Error: %v", os.Getpid(), setting.Indexer.RepoPath, err)
|
|
}
|
|
case "elasticsearch":
|
|
log.Info("PID: %d Initializing Repository Indexer at: %s", os.Getpid(), setting.Indexer.RepoConnStr)
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
log.Error("PANIC whilst initializing repository indexer: %v\nStacktrace: %s", err, log.Stack(2))
|
|
log.Error("The indexer files are likely corrupted and may need to be deleted")
|
|
log.Error("You can completely remove the \"%s\" index to make Gitea recreate the indexes", setting.Indexer.RepoConnStr)
|
|
}
|
|
}()
|
|
|
|
rIndexer = elasticsearch.NewIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName)
|
|
if err != nil {
|
|
cancel()
|
|
(*globalIndexer.Load()).Close()
|
|
close(waitChannel)
|
|
log.Fatal("PID: %d Unable to create the elasticsearch Repository Indexer connstr: %s Error: %v", os.Getpid(), setting.Indexer.RepoConnStr, err)
|
|
}
|
|
existed, err = rIndexer.Init(ctx)
|
|
if err != nil {
|
|
cancel()
|
|
(*globalIndexer.Load()).Close()
|
|
close(waitChannel)
|
|
log.Fatal("PID: %d Unable to initialize the elasticsearch Repository Indexer connstr: %s Error: %v", os.Getpid(), setting.Indexer.RepoConnStr, err)
|
|
}
|
|
|
|
default:
|
|
log.Fatal("PID: %d Unknown Indexer type: %s", os.Getpid(), setting.Indexer.RepoType)
|
|
}
|
|
|
|
globalIndexer.Store(&rIndexer)
|
|
|
|
// Start processing the queue
|
|
go graceful.GetManager().RunWithCancel(indexerQueue)
|
|
|
|
if !existed { // populate the index because it's created for the first time
|
|
go graceful.GetManager().RunWithShutdownContext(populateRepoIndexer)
|
|
}
|
|
select {
|
|
case waitChannel <- time.Since(start):
|
|
case <-graceful.GetManager().IsShutdown():
|
|
}
|
|
|
|
close(waitChannel)
|
|
}()
|
|
|
|
if setting.Indexer.StartupTimeout > 0 {
|
|
go func() {
|
|
pprof.SetGoroutineLabels(ctx)
|
|
timeout := setting.Indexer.StartupTimeout
|
|
if graceful.GetManager().IsChild() && setting.GracefulHammerTime > 0 {
|
|
timeout += setting.GracefulHammerTime
|
|
}
|
|
select {
|
|
case <-graceful.GetManager().IsShutdown():
|
|
log.Warn("Shutdown before Repository Indexer completed initialization")
|
|
cancel()
|
|
(*globalIndexer.Load()).Close()
|
|
case duration, ok := <-waitChannel:
|
|
if !ok {
|
|
log.Warn("Repository Indexer Initialization failed")
|
|
cancel()
|
|
(*globalIndexer.Load()).Close()
|
|
return
|
|
}
|
|
log.Info("Repository Indexer Initialization took %v", duration)
|
|
case <-time.After(timeout):
|
|
cancel()
|
|
(*globalIndexer.Load()).Close()
|
|
log.Fatal("Repository Indexer Initialization Timed-Out after: %v", timeout)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
// UpdateRepoIndexer update a repository's entries in the indexer
|
|
func UpdateRepoIndexer(repo *repo_model.Repository) {
|
|
indexData := &internal.IndexerData{RepoID: repo.ID}
|
|
if err := indexerQueue.Push(indexData); err != nil {
|
|
log.Error("Update repo index data %v failed: %v", indexData, err)
|
|
}
|
|
}
|
|
|
|
// IsAvailable checks if issue indexer is available
|
|
func IsAvailable(ctx context.Context) bool {
|
|
return (*globalIndexer.Load()).Ping(ctx) == nil
|
|
}
|
|
|
|
// populateRepoIndexer populate the repo indexer with pre-existing data. This
|
|
// should only be run when the indexer is created for the first time.
|
|
func populateRepoIndexer(ctx context.Context) {
|
|
log.Info("Populating the repo indexer with existing repositories")
|
|
|
|
exist, err := db.IsTableNotEmpty("repository")
|
|
if err != nil {
|
|
log.Fatal("System error: %v", err)
|
|
} else if !exist {
|
|
return
|
|
}
|
|
|
|
// if there is any existing repo indexer metadata in the DB, delete it
|
|
// since we are starting afresh. Also, xorm requires deletes to have a
|
|
// condition, and we want to delete everything, thus 1=1.
|
|
if err := db.DeleteAllRecords("repo_indexer_status"); err != nil {
|
|
log.Fatal("System error: %v", err)
|
|
}
|
|
|
|
var maxRepoID int64
|
|
if maxRepoID, err = db.GetMaxID("repository"); err != nil {
|
|
log.Fatal("System error: %v", err)
|
|
}
|
|
|
|
// start with the maximum existing repo ID and work backwards, so that we
|
|
// don't include repos that are created after gitea starts; such repos will
|
|
// already be added to the indexer, and we don't need to add them again.
|
|
for maxRepoID > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
log.Info("Repository Indexer population shutdown before completion")
|
|
return
|
|
default:
|
|
}
|
|
ids, err := repo_model.GetUnindexedRepos(repo_model.RepoIndexerTypeCode, maxRepoID, 0, 50)
|
|
if err != nil {
|
|
log.Error("populateRepoIndexer: %v", err)
|
|
return
|
|
} else if len(ids) == 0 {
|
|
break
|
|
}
|
|
for _, id := range ids {
|
|
select {
|
|
case <-ctx.Done():
|
|
log.Info("Repository Indexer population shutdown before completion")
|
|
return
|
|
default:
|
|
}
|
|
if err := indexerQueue.Push(&internal.IndexerData{RepoID: id}); err != nil {
|
|
log.Error("indexerQueue.Push: %v", err)
|
|
return
|
|
}
|
|
maxRepoID = id - 1
|
|
}
|
|
}
|
|
log.Info("Done (re)populating the repo indexer with existing repositories")
|
|
}
|