Automatically pause queue if index service is unavailable (#15066)

* Handle keyword search error when issue indexer service is not available

* Implement automatic disabling and resume of code indexer queue

docs/content/doc/developers/hacking-on-gitea.en-us.md

@@ -187,6 +187,27 @@ make lint-frontend
 
 Note: When working on frontend code, set `USE_SERVICE_WORKER` to `false` in `app.ini` to prevent undesirable caching of frontend assets.
 
+### Configuring local ElasticSearch instance
+
+Start local ElasticSearch instance using docker:
+
+```sh
+mkdir -p $(pwd)/data/elasticsearch
+sudo chown -R 1000:1000 $(pwd)/data/elasticsearch
+docker run --rm -p 127.0.0.1:9200:9200 -p 127.0.0.1:9300:9300 -e "discovery.type=single-node" -v "$(pwd)/data/elasticsearch:/usr/share/elasticsearch/data" docker.elastic.co/elasticsearch/elasticsearch:7.16.3
+```
+
+Configure `app.ini`:
+
+```ini
+[indexer]
+ISSUE_INDEXER_TYPE = elasticsearch
+ISSUE_INDEXER_CONN_STR = http://elastic:changeme@localhost:9200
+REPO_INDEXER_ENABLED = true
+REPO_INDEXER_TYPE = elasticsearch
+REPO_INDEXER_CONN_STR = http://elastic:changeme@localhost:9200
+```
+
 ### Building and adding SVGs
 
 SVG icons are built using the `make svg` target which compiles the icon sources defined in `build/generate-svg.js` into the output directory `public/img/svg`. Custom icons can be added in the `web_src/svg` directory.

integrations/repo_search_test.go

@@ -17,10 +17,7 @@ import (
 )
 
 func resultFilenames(t testing.TB, doc *HTMLDoc) []string {
-	resultsSelection := doc.doc.Find(".repository.search")
-	assert.EqualValues(t, 1, resultsSelection.Length(),
-		"Invalid template (repo search template has changed?)")
-	filenameSelections := resultsSelection.Find(".repo-search-result").Find(".header").Find("span.file")
+	filenameSelections := doc.doc.Find(".repository.search").Find(".repo-search-result").Find(".header").Find("span.file")
 	result := make([]string, filenameSelections.Length())
 	filenameSelections.Each(func(i int, selection *goquery.Selection) {
 		result[i] = selection.Text()

models/db/engine.go

@@ -65,6 +65,7 @@ type Engine interface {
 	Query(...interface{}) ([]map[string][]byte, error)
 	Cols(...string) *xorm.Session
 	Context(ctx context.Context) *xorm.Session
+	Ping() error
 }
 
 // TableInfo returns table's information via an object

models/issue.go

@@ -1859,7 +1859,7 @@ func GetRepoIssueStats(repoID, uid int64, filterMode int, isPull bool) (numOpen,
 }
 
 // SearchIssueIDsByKeyword search issues on database
-func SearchIssueIDsByKeyword(kw string, repoIDs []int64, limit, start int) (int64, []int64, error) {
+func SearchIssueIDsByKeyword(ctx context.Context, kw string, repoIDs []int64, limit, start int) (int64, []int64, error) {
 	repoCond := builder.In("repo_id", repoIDs)
 	subQuery := builder.Select("id").From("issue").Where(repoCond)
 	kw = strings.ToUpper(kw)
@@ -1884,7 +1884,7 @@ func SearchIssueIDsByKeyword(kw string, repoIDs []int64, limit, start int) (int6
 		ID          int64
 		UpdatedUnix int64
 	}, 0, limit)
-	err := db.GetEngine(db.DefaultContext).Distinct("id", "updated_unix").Table("issue").Where(cond).
+	err := db.GetEngine(ctx).Distinct("id", "updated_unix").Table("issue").Where(cond).
 		OrderBy("`updated_unix` DESC").Limit(limit, start).
 		Find(&res)
 	if err != nil {
@@ -1894,7 +1894,7 @@ func SearchIssueIDsByKeyword(kw string, repoIDs []int64, limit, start int) (int6
 		ids = append(ids, r.ID)
 	}
 
-	total, err := db.GetEngine(db.DefaultContext).Distinct("id").Table("issue").Where(cond).Count()
+	total, err := db.GetEngine(ctx).Distinct("id").Table("issue").Where(cond).Count()
 	if err != nil {
 		return 0, nil, err
 	}

models/issue_test.go

@@ -5,6 +5,7 @@
 package models
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"sync"
@@ -303,23 +304,23 @@ func TestIssue_loadTotalTimes(t *testing.T) {
 
 func TestIssue_SearchIssueIDsByKeyword(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
-	total, ids, err := SearchIssueIDsByKeyword("issue2", []int64{1}, 10, 0)
+	total, ids, err := SearchIssueIDsByKeyword(context.TODO(), "issue2", []int64{1}, 10, 0)
 	assert.NoError(t, err)
 	assert.EqualValues(t, 1, total)
 	assert.EqualValues(t, []int64{2}, ids)
 
-	total, ids, err = SearchIssueIDsByKeyword("first", []int64{1}, 10, 0)
+	total, ids, err = SearchIssueIDsByKeyword(context.TODO(), "first", []int64{1}, 10, 0)
 	assert.NoError(t, err)
 	assert.EqualValues(t, 1, total)
 	assert.EqualValues(t, []int64{1}, ids)
 
-	total, ids, err = SearchIssueIDsByKeyword("for", []int64{1}, 10, 0)
+	total, ids, err = SearchIssueIDsByKeyword(context.TODO(), "for", []int64{1}, 10, 0)
 	assert.NoError(t, err)
 	assert.EqualValues(t, 5, total)
 	assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
 
 	// issue1's comment id 2
-	total, ids, err = SearchIssueIDsByKeyword("good", []int64{1}, 10, 0)
+	total, ids, err = SearchIssueIDsByKeyword(context.TODO(), "good", []int64{1}, 10, 0)
 	assert.NoError(t, err)
 	assert.EqualValues(t, 1, total)
 	assert.EqualValues(t, []int64{1}, ids)
@@ -464,7 +465,7 @@ func TestCorrectIssueStats(t *testing.T) {
 	wg.Wait()
 
 	// Now we will get all issueID's that match the "Bugs are nasty" query.
-	total, ids, err := SearchIssueIDsByKeyword("Bugs are nasty", []int64{1}, issueAmount, 0)
+	total, ids, err := SearchIssueIDsByKeyword(context.TODO(), "Bugs are nasty", []int64{1}, issueAmount, 0)
 
 	// Just to be sure.
 	assert.NoError(t, err)

modules/context/repo.go

@@ -21,6 +21,7 @@ import (
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/git"
+	code_indexer "code.gitea.io/gitea/modules/indexer/code"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
@@ -522,6 +523,9 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
 	ctx.Data["ExposeAnonSSH"] = setting.SSH.ExposeAnonymous
 	ctx.Data["DisableHTTP"] = setting.Repository.DisableHTTPGit
 	ctx.Data["RepoSearchEnabled"] = setting.Indexer.RepoIndexerEnabled
+	if setting.Indexer.RepoIndexerEnabled {
+		ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
+	}
 	ctx.Data["CloneLink"] = repo.CloneLink()
 	ctx.Data["WikiCloneLink"] = repo.WikiCloneLink()
 

modules/indexer/code/bleve.go

@@ -271,6 +271,15 @@ func (b *BleveIndexer) Close() {
 	log.Info("PID: %d Repository Indexer closed", os.Getpid())
 }
 
+// SetAvailabilityChangeCallback does nothing
+func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
+}
+
+// Ping does nothing
+func (b *BleveIndexer) Ping() bool {
+	return true
+}
+
 // Index indexes the data
 func (b *BleveIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
 	batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
@@ -319,7 +328,7 @@ func (b *BleveIndexer) Delete(repoID int64) error {
 
 // Search searches for files in the specified repo.
 // Returns the matching file-paths
-func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
 	var (
 		indexerQuery query.Query
 		keywordQuery query.Query
@@ -372,7 +381,7 @@ func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, p
 		searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
 	}
 
-	result, err := b.indexer.Search(searchRequest)
+	result, err := b.indexer.SearchInContext(ctx, searchRequest)
 	if err != nil {
 		return 0, nil, nil, err
 	}

modules/indexer/code/elastic_search.go

@@ -7,16 +7,20 @@ package code
 import (
 	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	"net"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/analyze"
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -39,8 +43,12 @@ var _ Indexer = &ElasticSearchIndexer{}
 
 // ElasticSearchIndexer implements Indexer interface
 type ElasticSearchIndexer struct {
-	client           *elastic.Client
-	indexerAliasName string
+	client               *elastic.Client
+	indexerAliasName     string
+	available            bool
+	availabilityCallback func(bool)
+	stopTimer            chan struct{}
+	lock                 sync.RWMutex
 }
 
 type elasticLogger struct {
@@ -78,7 +86,23 @@ func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, bo
 	indexer := &ElasticSearchIndexer{
 		client:           client,
 		indexerAliasName: indexerName,
+		available:        true,
+		stopTimer:        make(chan struct{}),
 	}
+
+	ticker := time.NewTicker(10 * time.Second)
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				indexer.checkAvailability()
+			case <-indexer.stopTimer:
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+
 	exists, err := indexer.init()
 	if err != nil {
 		indexer.Close()
@@ -123,17 +147,17 @@ func (b *ElasticSearchIndexer) realIndexerName() string {
 
 // Init will initialize the indexer
 func (b *ElasticSearchIndexer) init() (bool, error) {
-	ctx := context.Background()
+	ctx := graceful.GetManager().HammerContext()
 	exists, err := b.client.IndexExists(b.realIndexerName()).Do(ctx)
 	if err != nil {
-		return false, err
+		return false, b.checkError(err)
 	}
 	if !exists {
 		mapping := defaultMapping
 
 		createIndex, err := b.client.CreateIndex(b.realIndexerName()).BodyString(mapping).Do(ctx)
 		if err != nil {
-			return false, err
+			return false, b.checkError(err)
 		}
 		if !createIndex.Acknowledged {
 			return false, fmt.Errorf("create index %s with %s failed", b.realIndexerName(), mapping)
@@ -143,7 +167,7 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
 	// check version
 	r, err := b.client.Aliases().Do(ctx)
 	if err != nil {
-		return false, err
+		return false, b.checkError(err)
 	}
 
 	realIndexerNames := r.IndicesByAlias(b.indexerAliasName)
@@ -152,10 +176,10 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
 			Add(b.realIndexerName(), b.indexerAliasName).
 			Do(ctx)
 		if err != nil {
-			return false, err
+			return false, b.checkError(err)
 		}
 		if !res.Acknowledged {
-			return false, fmt.Errorf("")
+			return false, fmt.Errorf("create alias %s to index %s failed", b.indexerAliasName, b.realIndexerName())
 		}
 	} else if len(realIndexerNames) >= 1 && realIndexerNames[0] < b.realIndexerName() {
 		log.Warn("Found older gitea indexer named %s, but we will create a new one %s and keep the old NOT DELETED. You can delete the old version after the upgrade succeed.",
@@ -165,16 +189,30 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
 			Add(b.realIndexerName(), b.indexerAliasName).
 			Do(ctx)
 		if err != nil {
-			return false, err
+			return false, b.checkError(err)
 		}
 		if !res.Acknowledged {
-			return false, fmt.Errorf("")
+			return false, fmt.Errorf("change alias %s to index %s failed", b.indexerAliasName, b.realIndexerName())
 		}
 	}
 
 	return exists, nil
 }
 
+// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes
+func (b *ElasticSearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.availabilityCallback = callback
+}
+
+// Ping checks if elastic is available
+func (b *ElasticSearchIndexer) Ping() bool {
+	b.lock.RLock()
+	defer b.lock.RUnlock()
+	return b.available
+}
+
 func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update fileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) {
 	// Ignore vendored files in code search
 	if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
@@ -190,7 +228,7 @@ func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.Wr
 			return nil, err
 		}
 		if size, err = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64); err != nil {
-			return nil, fmt.Errorf("Misformatted git cat-file output: %v", err)
+			return nil, fmt.Errorf("misformatted git cat-file output: %v", err)
 		}
 	}
 
@@ -274,8 +312,8 @@ func (b *ElasticSearchIndexer) Index(ctx context.Context, repo *repo_model.Repos
 		_, err := b.client.Bulk().
 			Index(b.indexerAliasName).
 			Add(reqs...).
-			Do(context.Background())
-		return err
+			Do(ctx)
+		return b.checkError(err)
 	}
 	return nil
 }
@@ -284,8 +322,8 @@ func (b *ElasticSearchIndexer) Index(ctx context.Context, repo *repo_model.Repos
 func (b *ElasticSearchIndexer) Delete(repoID int64) error {
 	_, err := b.client.DeleteByQuery(b.indexerAliasName).
 		Query(elastic.NewTermsQuery("repo_id", repoID)).
-		Do(context.Background())
-	return err
+		Do(graceful.GetManager().HammerContext())
+	return b.checkError(err)
 }
 
 // indexPos find words positions for start and the following end on content. It will
@@ -366,7 +404,7 @@ func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages {
 }
 
 // Search searches for codes and language stats by given conditions.
-func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (b *ElasticSearchIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
 	searchType := esMultiMatchTypeBestFields
 	if isMatch {
 		searchType = esMultiMatchTypePhrasePrefix
@@ -407,9 +445,9 @@ func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string,
 			).
 			Sort("repo_id", true).
 			From(start).Size(pageSize).
-			Do(context.Background())
+			Do(ctx)
 		if err != nil {
-			return 0, nil, nil, err
+			return 0, nil, nil, b.checkError(err)
 		}
 
 		return convertResult(searchResult, kw, pageSize)
@@ -421,9 +459,9 @@ func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string,
 		Aggregation("language", aggregation).
 		Query(query).
 		Size(0). // We only needs stats information
-		Do(context.Background())
+		Do(ctx)
 	if err != nil {
-		return 0, nil, nil, err
+		return 0, nil, nil, b.checkError(err)
 	}
 
 	query = query.Must(langQuery)
@@ -438,9 +476,9 @@ func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string,
 		).
 		Sort("repo_id", true).
 		From(start).Size(pageSize).
-		Do(context.Background())
+		Do(ctx)
 	if err != nil {
-		return 0, nil, nil, err
+		return 0, nil, nil, b.checkError(err)
 	}
 
 	total, hits, _, err := convertResult(searchResult, kw, pageSize)
@@ -449,4 +487,51 @@ func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string,
 }
 
 // Close implements indexer
-func (b *ElasticSearchIndexer) Close() {}
+func (b *ElasticSearchIndexer) Close() {
+	select {
+	case <-b.stopTimer:
+	default:
+		close(b.stopTimer)
+	}
+}
+
+func (b *ElasticSearchIndexer) checkError(err error) error {
+	var opErr *net.OpError
+	if !(elastic.IsConnErr(err) || (errors.As(err, &opErr) && (opErr.Op == "dial" || opErr.Op == "read"))) {
+		return err
+	}
+
+	b.setAvailability(false)
+
+	return err
+}
+
+func (b *ElasticSearchIndexer) checkAvailability() {
+	if b.Ping() {
+		return
+	}
+
+	// Request cluster state to check if elastic is available again
+	_, err := b.client.ClusterState().Do(graceful.GetManager().ShutdownContext())
+	if err != nil {
+		b.setAvailability(false)
+		return
+	}
+
+	b.setAvailability(true)
+}
+
+func (b *ElasticSearchIndexer) setAvailability(available bool) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+
+	if b.available == available {
+		return
+	}
+
+	b.available = available
+	if b.availabilityCallback != nil {
+		// Call the callback from within the lock to ensure that the ordering remains correct
+		b.availabilityCallback(b.available)
+	}
+}

modules/indexer/code/indexer.go

@@ -42,9 +42,11 @@ type SearchResultLanguages struct {
 
 // Indexer defines an interface to index and search code contents
 type Indexer interface {
+	Ping() bool
+	SetAvailabilityChangeCallback(callback func(bool))
 	Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error
 	Delete(repoID int64) error
-	Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
+	Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
 	Close()
 }
 
@@ -140,6 +142,7 @@ func Init() {
 				return data
 			}
 
+			unhandled := make([]queue.Data, 0, len(data))
 			for _, datum := range data {
 				indexerData, ok := datum.(*IndexerData)
 				if !ok {
@@ -150,10 +153,14 @@ func Init() {
 
 				if err := index(ctx, indexer, indexerData.RepoID); err != nil {
 					log.Error("index: %v", err)
-					continue
+					if indexer.Ping() {
+						continue
+					}
+					// Add back to queue
+					unhandled = append(unhandled, datum)
 				}
 			}
-			return nil
+			return unhandled
 		}
 
 		indexerQueue = queue.CreateUniqueQueue("code_indexer", handler, &IndexerData{})
@@ -212,6 +219,18 @@ func Init() {
 
 		indexer.set(rIndexer)
 
+		if queue, ok := indexerQueue.(queue.Pausable); ok {
+			rIndexer.SetAvailabilityChangeCallback(func(available bool) {
+				if !available {
+					log.Info("Code index queue paused")
+					queue.Pause()
+				} else {
+					log.Info("Code index queue resumed")
+					queue.Resume()
+				}
+			})
+		}
+
 		// Start processing the queue
 		go graceful.GetManager().RunWithShutdownFns(indexerQueue.Run)
 
@@ -262,6 +281,17 @@ func UpdateRepoIndexer(repo *repo_model.Repository) {
 	}
 }
 
+// IsAvailable checks if issue indexer is available
+func IsAvailable() bool {
+	idx, err := indexer.get()
+	if err != nil {
+		log.Error("IsAvailable(): unable to get indexer: %v", err)
+		return false
+	}
+
+	return idx.Ping()
+}
+
 // populateRepoIndexer populate the repo indexer with pre-existing data. This
 // should only be run when the indexer is created for the first time.
 func populateRepoIndexer(ctx context.Context) {

modules/indexer/code/indexer_test.go

@@ -5,6 +5,7 @@
 package code
 
 import (
+	"context"
 	"path/filepath"
 	"testing"
 
@@ -65,7 +66,7 @@ func testIndexer(name string, t *testing.T, indexer Indexer) {
 
 		for _, kw := range keywords {
 			t.Run(kw.Keyword, func(t *testing.T) {
-				total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10, false)
+				total, res, langs, err := indexer.Search(context.TODO(), kw.RepoIDs, "", kw.Keyword, 1, 10, false)
 				assert.NoError(t, err)
 				assert.EqualValues(t, len(kw.IDs), total)
 				assert.Len(t, langs, kw.Langs)

modules/indexer/code/search.go

@@ -6,6 +6,7 @@ package code
 
 import (
 	"bytes"
+	"context"
 	"strings"
 
 	"code.gitea.io/gitea/modules/highlight"
@@ -106,12 +107,12 @@ func searchResult(result *SearchResult, startIndex, endIndex int) (*Result, erro
 }
 
 // PerformSearch perform a search on a repository
-func PerformSearch(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*SearchResultLanguages, error) {
+func PerformSearch(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*SearchResultLanguages, error) {
 	if len(keyword) == 0 {
 		return 0, nil, nil, nil
 	}
 
-	total, results, resultLanguages, err := indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch)
+	total, results, resultLanguages, err := indexer.Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
 	if err != nil {
 		return 0, nil, nil, err
 	}

modules/indexer/code/wrapped.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 
 	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/log"
 )
 
 var indexer = newWrappedIndexer()
@@ -56,6 +57,26 @@ func (w *wrappedIndexer) get() (Indexer, error) {
 	return w.internal, nil
 }
 
+// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes
+func (w *wrappedIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
+	indexer, err := w.get()
+	if err != nil {
+		log.Error("Failed to get indexer: %v", err)
+		return
+	}
+	indexer.SetAvailabilityChangeCallback(callback)
+}
+
+// Ping checks if elastic is available
+func (w *wrappedIndexer) Ping() bool {
+	indexer, err := w.get()
+	if err != nil {
+		log.Warn("Failed to get indexer: %v", err)
+		return false
+	}
+	return indexer.Ping()
+}
+
 func (w *wrappedIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
 	indexer, err := w.get()
 	if err != nil {
@@ -72,12 +93,12 @@ func (w *wrappedIndexer) Delete(repoID int64) error {
 	return indexer.Delete(repoID)
 }
 
-func (w *wrappedIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (w *wrappedIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
 	indexer, err := w.get()
 	if err != nil {
 		return 0, nil, nil, err
 	}
-	return indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch)
+	return indexer.Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
 }
 
 func (w *wrappedIndexer) Close() {

modules/indexer/issues/bleve.go

@@ -5,6 +5,7 @@
 package issues
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"strconv"
@@ -186,6 +187,15 @@ func (b *BleveIndexer) Init() (bool, error) {
 	return false, err
 }
 
+// SetAvailabilityChangeCallback does nothing
+func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
+}
+
+// Ping does nothing
+func (b *BleveIndexer) Ping() bool {
+	return true
+}
+
 // Close will close the bleve indexer
 func (b *BleveIndexer) Close() {
 	if b.indexer != nil {
@@ -229,7 +239,7 @@ func (b *BleveIndexer) Delete(ids ...int64) error {
 
 // Search searches for issues by given conditions.
 // Returns the matching issue IDs
-func (b *BleveIndexer) Search(keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
+func (b *BleveIndexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
 	var repoQueriesP []*query.NumericRangeQuery
 	for _, repoID := range repoIDs {
 		repoQueriesP = append(repoQueriesP, numericEqualityQuery(repoID, "RepoID"))
@@ -249,7 +259,7 @@ func (b *BleveIndexer) Search(keyword string, repoIDs []int64, limit, start int)
 	search := bleve.NewSearchRequestOptions(indexerQuery, limit, start, false)
 	search.SortBy([]string{"-_score"})
 
-	result, err := b.indexer.Search(search)
+	result, err := b.indexer.SearchInContext(ctx, search)
 	if err != nil {
 		return nil, err
 	}

modules/indexer/issues/bleve_test.go

@@ -5,6 +5,7 @@
 package issues
 
 import (
+	"context"
 	"os"
 	"testing"
 
@@ -84,7 +85,7 @@ func TestBleveIndexAndSearch(t *testing.T) {
 	}
 
 	for _, kw := range keywords {
-		res, err := indexer.Search(kw.Keyword, []int64{2}, 10, 0)
+		res, err := indexer.Search(context.TODO(), kw.Keyword, []int64{2}, 10, 0)
 		assert.NoError(t, err)
 
 		ids := make([]int64, 0, len(res.Hits))

modules/indexer/issues/db.go

@@ -4,33 +4,47 @@
 
 package issues
 
-import "code.gitea.io/gitea/models"
+import (
+	"context"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/models/db"
+)
 
 // DBIndexer implements Indexer interface to use database's like search
 type DBIndexer struct{}
 
 // Init dummy function
-func (db *DBIndexer) Init() (bool, error) {
+func (i *DBIndexer) Init() (bool, error) {
 	return false, nil
 }
 
+// SetAvailabilityChangeCallback dummy function
+func (i *DBIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
+}
+
+// Ping checks if database is available
+func (i *DBIndexer) Ping() bool {
+	return db.GetEngine(db.DefaultContext).Ping() != nil
+}
+
 // Index dummy function
-func (db *DBIndexer) Index(issue []*IndexerData) error {
+func (i *DBIndexer) Index(issue []*IndexerData) error {
 	return nil
 }
 
 // Delete dummy function
-func (db *DBIndexer) Delete(ids ...int64) error {
+func (i *DBIndexer) Delete(ids ...int64) error {
 	return nil
 }
 
 // Close dummy function
-func (db *DBIndexer) Close() {
+func (i *DBIndexer) Close() {
 }
 
 // Search dummy function
-func (db *DBIndexer) Search(kw string, repoIDs []int64, limit, start int) (*SearchResult, error) {
-	total, ids, err := models.SearchIssueIDsByKeyword(kw, repoIDs, limit, start)
+func (i *DBIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) {
+	total, ids, err := models.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start)
 	if err != nil {
 		return nil, err
 	}

modules/indexer/issues/elastic_search.go

@@ -8,9 +8,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"strconv"
+	"sync"
 	"time"
 
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 
 	"github.com/olivere/elastic/v7"
@@ -20,8 +23,12 @@ var _ Indexer = &ElasticSearchIndexer{}
 
 // ElasticSearchIndexer implements Indexer interface
 type ElasticSearchIndexer struct {
-	client      *elastic.Client
-	indexerName string
+	client               *elastic.Client
+	indexerName          string
+	available            bool
+	availabilityCallback func(bool)
+	stopTimer            chan struct{}
+	lock                 sync.RWMutex
 }
 
 type elasticLogger struct {
@@ -56,10 +63,27 @@ func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, er
 		return nil, err
 	}
 
-	return &ElasticSearchIndexer{
+	indexer := &ElasticSearchIndexer{
 		client:      client,
 		indexerName: indexerName,
-	}, nil
+		available:   true,
+		stopTimer:   make(chan struct{}),
+	}
+
+	ticker := time.NewTicker(10 * time.Second)
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				indexer.checkAvailability()
+			case <-indexer.stopTimer:
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+
+	return indexer, nil
 }
 
 const (
@@ -93,10 +117,10 @@ const (
 
 // Init will initialize the indexer
 func (b *ElasticSearchIndexer) Init() (bool, error) {
-	ctx := context.Background()
+	ctx := graceful.GetManager().HammerContext()
 	exists, err := b.client.IndexExists(b.indexerName).Do(ctx)
 	if err != nil {
-		return false, err
+		return false, b.checkError(err)
 	}
 
 	if !exists {
@@ -104,7 +128,7 @@ func (b *ElasticSearchIndexer) Init() (bool, error) {
 
 		createIndex, err := b.client.CreateIndex(b.indexerName).BodyString(mapping).Do(ctx)
 		if err != nil {
-			return false, err
+			return false, b.checkError(err)
 		}
 		if !createIndex.Acknowledged {
 			return false, errors.New("init failed")
@@ -115,6 +139,20 @@ func (b *ElasticSearchIndexer) Init() (bool, error) {
 	return true, nil
 }
 
+// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes
+func (b *ElasticSearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.availabilityCallback = callback
+}
+
+// Ping checks if elastic is available
+func (b *ElasticSearchIndexer) Ping() bool {
+	b.lock.RLock()
+	defer b.lock.RUnlock()
+	return b.available
+}
+
 // Index will save the index data
 func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
 	if len(issues) == 0 {
@@ -131,8 +169,8 @@ func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
 				"content":  issue.Content,
 				"comments": issue.Comments,
 			}).
-			Do(context.Background())
-		return err
+			Do(graceful.GetManager().HammerContext())
+		return b.checkError(err)
 	}
 
 	reqs := make([]elastic.BulkableRequest, 0)
@@ -154,8 +192,8 @@ func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
 	_, err := b.client.Bulk().
 		Index(b.indexerName).
 		Add(reqs...).
-		Do(context.Background())
-	return err
+		Do(graceful.GetManager().HammerContext())
+	return b.checkError(err)
 }
 
 // Delete deletes indexes by ids
@@ -166,8 +204,8 @@ func (b *ElasticSearchIndexer) Delete(ids ...int64) error {
 		_, err := b.client.Delete().
 			Index(b.indexerName).
 			Id(fmt.Sprintf("%d", ids[0])).
-			Do(context.Background())
-		return err
+			Do(graceful.GetManager().HammerContext())
+		return b.checkError(err)
 	}
 
 	reqs := make([]elastic.BulkableRequest, 0)
@@ -182,13 +220,13 @@ func (b *ElasticSearchIndexer) Delete(ids ...int64) error {
 	_, err := b.client.Bulk().
 		Index(b.indexerName).
 		Add(reqs...).
-		Do(context.Background())
-	return err
+		Do(graceful.GetManager().HammerContext())
+	return b.checkError(err)
 }
 
 // Search searches for issues by given conditions.
 // Returns the matching issue IDs
-func (b *ElasticSearchIndexer) Search(keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
+func (b *ElasticSearchIndexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
 	kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
 	query := elastic.NewBoolQuery()
 	query = query.Must(kwQuery)
@@ -205,9 +243,9 @@ func (b *ElasticSearchIndexer) Search(keyword string, repoIDs []int64, limit, st
 		Query(query).
 		Sort("_score", false).
 		From(start).Size(limit).
-		Do(context.Background())
+		Do(ctx)
 	if err != nil {
-		return nil, err
+		return nil, b.checkError(err)
 	}
 
 	hits := make([]Match, 0, limit)
@@ -225,4 +263,51 @@ func (b *ElasticSearchIndexer) Search(keyword string, repoIDs []int64, limit, st
 }
 
 // Close implements indexer
-func (b *ElasticSearchIndexer) Close() {}
+func (b *ElasticSearchIndexer) Close() {
+	select {
+	case <-b.stopTimer:
+	default:
+		close(b.stopTimer)
+	}
+}
+
+func (b *ElasticSearchIndexer) checkError(err error) error {
+	var opErr *net.OpError
+	if !(elastic.IsConnErr(err) || (errors.As(err, &opErr) && (opErr.Op == "dial" || opErr.Op == "read"))) {
+		return err
+	}
+
+	b.setAvailability(false)
+
+	return err
+}
+
+func (b *ElasticSearchIndexer) checkAvailability() {
+	if b.Ping() {
+		return
+	}
+
+	// Request cluster state to check if elastic is available again
+	_, err := b.client.ClusterState().Do(graceful.GetManager().ShutdownContext())
+	if err != nil {
+		b.setAvailability(false)
+		return
+	}
+
+	b.setAvailability(true)
+}
+
+func (b *ElasticSearchIndexer) setAvailability(available bool) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+
+	if b.available == available {
+		return
+	}
+
+	b.available = available
+	if b.availabilityCallback != nil {
+		// Call the callback from within the lock to ensure that the ordering remains correct
+		b.availabilityCallback(b.available)
+	}
+}

modules/indexer/issues/indexer.go

@@ -47,9 +47,11 @@ type SearchResult struct {
 // Indexer defines an interface to indexer issues contents
 type Indexer interface {
 	Init() (bool, error)
+	Ping() bool
+	SetAvailabilityChangeCallback(callback func(bool))
 	Index(issue []*IndexerData) error
 	Delete(ids ...int64) error
-	Search(kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
+	Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
 	Close()
 }
 
@@ -111,6 +113,7 @@ func InitIssueIndexer(syncReindex bool) {
 			}
 
 			iData := make([]*IndexerData, 0, len(data))
+			unhandled := make([]queue.Data, 0, len(data))
 			for _, datum := range data {
 				indexerData, ok := datum.(*IndexerData)
 				if !ok {
@@ -119,13 +122,34 @@ func InitIssueIndexer(syncReindex bool) {
 				}
 				log.Trace("IndexerData Process: %d %v %t", indexerData.ID, indexerData.IDs, indexerData.IsDelete)
 				if indexerData.IsDelete {
-					_ = indexer.Delete(indexerData.IDs...)
+					if err := indexer.Delete(indexerData.IDs...); err != nil {
+						log.Error("Error whilst deleting from index: %v Error: %v", indexerData.IDs, err)
+						if indexer.Ping() {
+							continue
+						}
+						// Add back to queue
+						unhandled = append(unhandled, datum)
+					}
 					continue
 				}
 				iData = append(iData, indexerData)
 			}
+			if len(unhandled) > 0 {
+				for _, indexerData := range iData {
+					unhandled = append(unhandled, indexerData)
+				}
+				return unhandled
+			}
 			if err := indexer.Index(iData); err != nil {
 				log.Error("Error whilst indexing: %v Error: %v", iData, err)
+				if indexer.Ping() {
+					return nil
+				}
+				// Add back to queue
+				for _, indexerData := range iData {
+					unhandled = append(unhandled, indexerData)
+				}
+				return unhandled
 			}
 			return nil
 		}
@@ -193,6 +217,18 @@ func InitIssueIndexer(syncReindex bool) {
 			log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType)
 		}
 
+		if queue, ok := issueIndexerQueue.(queue.Pausable); ok {
+			holder.get().SetAvailabilityChangeCallback(func(available bool) {
+				if !available {
+					log.Info("Issue index queue paused")
+					queue.Pause()
+				} else {
+					log.Info("Issue index queue resumed")
+					queue.Resume()
+				}
+			})
+		}
+
 		// Start processing the queue
 		go graceful.GetManager().RunWithShutdownFns(issueIndexerQueue.Run)
 
@@ -334,7 +370,7 @@ func DeleteRepoIssueIndexer(repo *repo_model.Repository) {
 
 // SearchIssuesByKeyword search issue ids by keywords and repo id
 // WARNNING: You have to ensure user have permission to visit repoIDs' issues
-func SearchIssuesByKeyword(repoIDs []int64, keyword string) ([]int64, error) {
+func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) ([]int64, error) {
 	var issueIDs []int64
 	indexer := holder.get()
 
@@ -342,7 +378,7 @@ func SearchIssuesByKeyword(repoIDs []int64, keyword string) ([]int64, error) {
 		log.Error("SearchIssuesByKeyword(): unable to get indexer!")
 		return nil, fmt.Errorf("unable to get issue indexer")
 	}
-	res, err := indexer.Search(keyword, repoIDs, 50, 0)
+	res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0)
 	if err != nil {
 		return nil, err
 	}
@@ -351,3 +387,14 @@ func SearchIssuesByKeyword(repoIDs []int64, keyword string) ([]int64, error) {
 	}
 	return issueIDs, nil
 }
+
+// IsAvailable checks if issue indexer is available
+func IsAvailable() bool {
+	indexer := holder.get()
+	if indexer == nil {
+		log.Error("IsAvailable(): unable to get indexer!")
+		return false
+	}
+
+	return indexer.Ping()
+}

modules/indexer/issues/indexer_test.go

@@ -5,6 +5,7 @@
 package issues
 
 import (
+	"context"
 	"os"
 	"path"
 	"path/filepath"
@@ -56,19 +57,19 @@ func TestBleveSearchIssues(t *testing.T) {
 
 	time.Sleep(5 * time.Second)
 
-	ids, err := SearchIssuesByKeyword([]int64{1}, "issue2")
+	ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2")
 	assert.NoError(t, err)
 	assert.EqualValues(t, []int64{2}, ids)
 
-	ids, err = SearchIssuesByKeyword([]int64{1}, "first")
+	ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first")
 	assert.NoError(t, err)
 	assert.EqualValues(t, []int64{1}, ids)
 
-	ids, err = SearchIssuesByKeyword([]int64{1}, "for")
+	ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for")
 	assert.NoError(t, err)
 	assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
 
-	ids, err = SearchIssuesByKeyword([]int64{1}, "good")
+	ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good")
 	assert.NoError(t, err)
 	assert.EqualValues(t, []int64{1}, ids)
 }
@@ -79,19 +80,19 @@ func TestDBSearchIssues(t *testing.T) {
 	setting.Indexer.IssueType = "db"
 	InitIssueIndexer(true)
 
-	ids, err := SearchIssuesByKeyword([]int64{1}, "issue2")
+	ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2")
 	assert.NoError(t, err)
 	assert.EqualValues(t, []int64{2}, ids)
 
-	ids, err = SearchIssuesByKeyword([]int64{1}, "first")
+	ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first")
 	assert.NoError(t, err)
 	assert.EqualValues(t, []int64{1}, ids)
 
-	ids, err = SearchIssuesByKeyword([]int64{1}, "for")
+	ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for")
 	assert.NoError(t, err)
 	assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
 
-	ids, err = SearchIssuesByKeyword([]int64{1}, "good")
+	ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good")
 	assert.NoError(t, err)
 	assert.EqualValues(t, []int64{1}, ids)
 }

options/locale/locale_en-US.ini

@@ -268,6 +268,7 @@ search = Search
 code = Code
 search.fuzzy = Fuzzy
 search.match = Match
+code_search_unavailable = Currently code search is not available. Please contact your site administrator.
 repo_no_results = No matching repositories found.
 user_no_results = No matching users found.
 org_no_results = No matching organizations found.
@@ -1262,6 +1263,7 @@ issues.filter_sort.moststars = Most stars
 issues.filter_sort.feweststars = Fewest stars
 issues.filter_sort.mostforks = Most forks
 issues.filter_sort.fewestforks = Fewest forks
+issues.keyword_search_unavailable = Currently searhing by keyword is not available. Please contact your site administrator.
 issues.action_open = Open
 issues.action_close = Close
 issues.action_label = Label
@@ -1707,6 +1709,8 @@ search.search_repo = Search repository
 search.fuzzy = Fuzzy
 search.match = Match
 search.results = Search results for "%s" in <a href="%s">%s</a>
+search.code_no_results = No source code matching your search term found.
+search.code_search_unavailable = Currently code search is not available. Please contact your site administrator.
 
 settings = Settings
 settings.desc = Settings is where you can manage the settings for the repository

routers/api/v1/repo/issue.go

@@ -188,7 +188,7 @@ func SearchIssues(ctx *context.APIContext) {
 	}
 	var issueIDs []int64
 	if len(keyword) > 0 && len(repoIDs) > 0 {
-		if issueIDs, err = issue_indexer.SearchIssuesByKeyword(repoIDs, keyword); err != nil {
+		if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil {
 			ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
 			return
 		}
@@ -379,7 +379,7 @@ func ListIssues(ctx *context.APIContext) {
 	var issueIDs []int64
 	var labelIDs []int64
 	if len(keyword) > 0 {
-		issueIDs, err = issue_indexer.SearchIssuesByKeyword([]int64{ctx.Repo.Repository.ID}, keyword)
+		issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword)
 		if err != nil {
 			ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
 			return

routers/web/explore/code.go

@@ -87,17 +87,27 @@ func Code(ctx *context.Context) {
 
 		ctx.Data["RepoMaps"] = rightRepoMap
 
-		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
+		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
 		if err != nil {
-			ctx.ServerError("SearchResults", err)
-			return
+			if code_indexer.IsAvailable() {
+				ctx.ServerError("SearchResults", err)
+				return
+			}
+			ctx.Data["CodeIndexerUnavailable"] = true
+		} else {
+			ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
 		}
 		// if non-login user or isAdmin, no need to check UnitTypeCode
 	} else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
-		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
+		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
 		if err != nil {
-			ctx.ServerError("SearchResults", err)
-			return
+			if code_indexer.IsAvailable() {
+				ctx.ServerError("SearchResults", err)
+				return
+			}
+			ctx.Data["CodeIndexerUnavailable"] = true
+		} else {
+			ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
 		}
 
 		loadRepoIDs := make([]int64, 0, len(searchResults))

routers/web/repo/issue.go

@@ -161,10 +161,13 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
 
 	var issueIDs []int64
 	if len(keyword) > 0 {
-		issueIDs, err = issue_indexer.SearchIssuesByKeyword([]int64{repo.ID}, keyword)
+		issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword)
 		if err != nil {
-			ctx.ServerError("issueIndexer.Search", err)
-			return
+			if issue_indexer.IsAvailable() {
+				ctx.ServerError("issueIndexer.Search", err)
+				return
+			}
+			ctx.Data["IssueIndexerUnavailable"] = true
 		}
 		if len(issueIDs) == 0 {
 			forceEmpty = true

routers/web/repo/search.go

@@ -30,11 +30,16 @@ func Search(ctx *context.Context) {
 	queryType := ctx.FormTrim("t")
 	isMatch := queryType == "match"
 
-	total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch([]int64{ctx.Repo.Repository.ID},
+	total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch(ctx, []int64{ctx.Repo.Repository.ID},
 		language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
 	if err != nil {
-		ctx.ServerError("SearchResults", err)
-		return
+		if code_indexer.IsAvailable() {
+			ctx.ServerError("SearchResults", err)
+			return
+		}
+		ctx.Data["CodeIndexerUnavailable"] = true
+	} else {
+		ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
 	}
 	ctx.Data["Keyword"] = keyword
 	ctx.Data["Language"] = language

routers/web/user/home.go

@@ -438,7 +438,7 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
 
 	// Execute keyword search for issues.
 	// USING NON-FINAL STATE OF opts FOR A QUERY.
-	issueIDsFromSearch, err := issueIDsFromSearch(ctxUser, keyword, opts)
+	issueIDsFromSearch, err := issueIDsFromSearch(ctx, ctxUser, keyword, opts)
 	if err != nil {
 		ctx.ServerError("issueIDsFromSearch", err)
 		return
@@ -673,7 +673,7 @@ func getRepoIDs(reposQuery string) []int64 {
 	return repoIDs
 }
 
-func issueIDsFromSearch(ctxUser *user_model.User, keyword string, opts *models.IssuesOptions) ([]int64, error) {
+func issueIDsFromSearch(ctx *context.Context, ctxUser *user_model.User, keyword string, opts *models.IssuesOptions) ([]int64, error) {
 	if len(keyword) == 0 {
 		return []int64{}, nil
 	}
@@ -682,7 +682,7 @@ func issueIDsFromSearch(ctxUser *user_model.User, keyword string, opts *models.I
 	if err != nil {
 		return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %v", err)
 	}
-	issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(searchRepoIDs, keyword)
+	issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword)
 	if err != nil {
 		return nil, fmt.Errorf("SearchIssuesByKeyword: %v", err)
 	}

templates/explore/code.tmpl

@@ -5,21 +5,25 @@
 		<form class="ui form ignore-dirty" style="max-width: 100%">
 			<input type="hidden" name="tab" value="{{$.TabName}}">
 			<div class="ui fluid action input">
-				<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
-				<div class="ui dropdown selection">
-					<input name="t" type="hidden" value="{{.queryType}}">{{svg "octicon-triangle-down" 14 "dropdown icon"}}
+				<input name="q" value="{{.Keyword}}"{{if .CodeIndexerUnavailable }} disabled{{end}} placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
+				<div class="ui dropdown selection{{if .CodeIndexerUnavailable }} disabled{{end}}">
+					<input name="t" type="hidden" value="{{.queryType}}"{{if .CodeIndexerUnavailable }} disabled{{end}}>{{svg "octicon-triangle-down" 14 "dropdown icon"}}
 					<div class="text">{{.i18n.Tr (printf "explore.search.%s" (or .queryType "fuzzy"))}}</div>
 					<div class="menu transition hidden" tabindex="-1" style="display: block !important;">
 						<div class="item" data-value="">{{.i18n.Tr "explore.search.fuzzy"}}</div>
 						<div class="item" data-value="match">{{.i18n.Tr "explore.search.match"}}</div>
 					</div>
 				</div>
-				<button class="ui blue button">{{.i18n.Tr "explore.search"}}</button>
+				<button class="ui blue button"{{if .CodeIndexerUnavailable }} disabled{{end}}>{{.i18n.Tr "explore.search"}}</button>
 			</div>
 		</form>
 		<div class="ui divider"></div>
 		<div class="ui user list">
-			{{if .SearchResults}}
+			{{if .CodeIndexerUnavailable }}
+				<div class="ui error message">
+					<p>{{$.i18n.Tr "explore.code_search_unavailable"}}</p>
+				</div>
+			{{else if .SearchResults}}
 				<h3>
 					{{.i18n.Tr "explore.code_search_results" (.Keyword|Escape) | Str2html }}
 				</h3>

templates/repo/home.tmpl

@@ -13,9 +13,12 @@
 				<div class="ui repo-search">
 					<form class="ui form ignore-dirty" action="{{.RepoLink}}/search" method="get">
 						<div class="field">
-							<div class="ui action input">
-								<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
-								<button class="ui icon button" type="submit">
+							<div class="ui action input{{if .CodeIndexerUnavailable }} disabled left icon tooltip{{end}}"{{if .CodeIndexerUnavailable }} data-content="{{.i18n.Tr "repo.search.code_search_unavailable"}}"{{end}}>
+								<input name="q" value="{{.Keyword}}"{{if .CodeIndexerUnavailable }} disabled{{end}} placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
+								{{if .CodeIndexerUnavailable }}
+									<i class="icon df ac jc">{{svg "octicon-alert"}}</i>
+								{{end}}
+								<button class="ui icon button"{{if .CodeIndexerUnavailable }} disabled{{end}} type="submit">
 									{{svg "octicon-search"}}
 								</button>
 							</div>

templates/repo/search.tmpl

@@ -5,60 +5,68 @@
 		<div class="ui repo-search">
 			<form class="ui form ignore-dirty" method="get">
 				<div class="ui fluid action input">
-					<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
-					<div class="ui dropdown selection">
-						<input name="t" type="hidden" value="{{.queryType}}">{{svg "octicon-triangle-down" 14 "dropdown icon"}}
+					<input name="q" value="{{.Keyword}}"{{if .CodeIndexerUnavailable }} disabled{{end}} placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
+					<div class="ui dropdown selection{{if .CodeIndexerUnavailable }} disabled{{end}}">
+						<input name="t" type="hidden"{{if .CodeIndexerUnavailable }} disabled{{end}} value="{{.queryType}}">{{svg "octicon-triangle-down" 14 "dropdown icon"}}
 						<div class="text">{{.i18n.Tr (printf "repo.search.%s" (or .queryType "fuzzy"))}}</div>
 						<div class="menu transition hidden" tabindex="-1" style="display: block !important;">
 							<div class="item" data-value="">{{.i18n.Tr "repo.search.fuzzy"}}</div>
 							<div class="item" data-value="match">{{.i18n.Tr "repo.search.match"}}</div>
 						</div>
 					</div>
-					<button class="ui icon button" type="submit">{{svg "octicon-search" 16}}</button>
+					<button class="ui icon button"{{if .CodeIndexerUnavailable }} disabled{{end}} type="submit">{{svg "octicon-search" 16}}</button>
 				</div>
 			</form>
 		</div>
-		{{if .Keyword}}
+		{{if .CodeIndexerUnavailable }}
+			<div class="ui error message">
+				<p>{{$.i18n.Tr "repo.search.code_search_unavailable"}}</p>
+			</div>
+		{{else if .Keyword}}
 			<h3>
 				{{.i18n.Tr "repo.search.results" (.Keyword|Escape) (.RepoLink|Escape) (.RepoName|Escape) | Str2html }}
 			</h3>
-			<div class="df ac fw">
-				{{range $term := .SearchResultLanguages}}
-				<a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{$.SourcePath}}/search?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if ne $.queryType ""}}&t={{$.queryType}}{{end}}">
-					<i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i>
-					{{$term.Language}}
-					<div class="detail">{{$term.Count}}</div>
-				</a>
-				{{end}}
-			</div>
-			<div class="repository search">
-				{{range $result := .SearchResults}}
-					<div class="diff-file-box diff-box file-content non-diff-file-content repo-search-result">
-						<h4 class="ui top attached normal header">
-							<span class="file">{{.Filename}}</span>
-							<a class="ui basic tiny button" rel="nofollow" href="{{$.SourcePath}}/src/commit/{{PathEscape $result.CommitID}}/{{PathEscapeSegments .Filename}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
-						</h4>
-						<div class="ui attached table segment">
-							<div class="file-body file-code code-view">
-								<table>
-									<tbody>
-										<tr>
-											<td class="lines-num">
-												{{range .LineNumbers}}
-													<a href="{{$.SourcePath}}/src/commit/{{PathEscape $result.CommitID}}/{{PathEscapeSegments $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
-												{{end}}
-											</td>
-											<td class="lines-code chroma"><code class="code-inner">{{.FormattedLines | Safe}}</code></td>
-										</tr>
-									</tbody>
-								</table>
+			{{if .SearchResults}}
+				<div class="df ac fw">
+					{{range $term := .SearchResultLanguages}}
+					<a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{$.SourcePath}}/search?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if ne $.queryType ""}}&t={{$.queryType}}{{end}}">
+						<i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i>
+						{{$term.Language}}
+						<div class="detail">{{$term.Count}}</div>
+					</a>
+					{{end}}
+				</div>
+				<div class="repository search">
+					{{range $result := .SearchResults}}
+						<div class="diff-file-box diff-box file-content non-diff-file-content repo-search-result">
+							<h4 class="ui top attached normal header">
+								<span class="file">{{.Filename}}</span>
+								<a class="ui basic tiny button" rel="nofollow" href="{{$.SourcePath}}/src/commit/{{PathEscape $result.CommitID}}/{{PathEscapeSegments .Filename}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
+							</h4>
+							<div class="ui attached table segment">
+								<div class="file-body file-code code-view">
+									<table>
+										<tbody>
+											<tr>
+												<td class="lines-num">
+													{{range .LineNumbers}}
+														<a href="{{$.SourcePath}}/src/commit/{{PathEscape $result.CommitID}}/{{PathEscapeSegments $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
+													{{end}}
+												</td>
+												<td class="lines-code chroma"><code class="code-inner">{{.FormattedLines | Safe}}</code></td>
+											</tr>
+										</tbody>
+									</table>
+								</div>
 							</div>
+							{{template "shared/searchbottom" dict "root" $ "result" .}}
 						</div>
-						{{template "shared/searchbottom" dict "root" $ "result" .}}
-					</div>
-				{{end}}
-			</div>
-			{{template "base/paginate" .}}
+					{{end}}
+				</div>
+				{{template "base/paginate" .}}
+			{{else}}
+				<div>{{$.i18n.Tr "repo.search.code_no_results"}}</div>
+			{{end}}
 		{{end}}
 	</div>
 </div>

templates/shared/issuelist.tmpl

@@ -139,5 +139,10 @@
 			</div>
 		</li>
 	{{end}}
+	{{if .IssueIndexerUnavailable}}
+		<div class="ui error message">
+			<p>{{$.i18n.Tr "repo.issues.keyword_search_unavailable"}}</p>
+		</div>
+	{{end}}
 </div>
 {{template "base/paginate" .}}