From bccf4eeed2e98edada4828cb28d993924e8be70c Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Fri, 7 Jul 2023 17:19:47 +0100 Subject: [PATCH] Add the ability to reload search analyzers during shard recovery (#97421) This change adds the ability for reloadable search analysers to adapt their loading based on the index creation context. It is useful for reloadable search analysers that need to load expensive resources from indices or disk. In such case they can defer the loading of the resource during the shard recovery and avoid blocking a master or a create index thread. --------- Co-authored-by: Mayya Sharipova --- .../analysis/common/CommonAnalysisPlugin.java | 17 ++--------- .../common/MultiplexerTokenFilterFactory.java | 6 ++-- .../ScriptedConditionTokenFilterFactory.java | 4 ++- .../SynonymGraphTokenFilterFactory.java | 10 +++---- .../common/SynonymTokenFilterFactory.java | 23 +++++++------- .../common/CompoundAnalysisTests.java | 3 +- .../common/EdgeNGramTokenizerTests.java | 3 +- .../common/MultiplexerTokenFilterTests.java | 5 ++-- .../PredicateTokenScriptFilterTests.java | 3 +- .../ScriptedConditionTokenFilterTests.java | 3 +- .../common/SynonymsAnalysisTests.java | 4 +-- ...DelimiterGraphTokenFilterFactoryTests.java | 5 ++-- .../70_synonyms_from_index.yml | 19 ++++++++++++ .../analyze/TransportAnalyzeAction.java | 3 ++ .../org/elasticsearch/index/IndexModule.java | 7 +++-- .../org/elasticsearch/index/IndexService.java | 5 +++- .../index/analysis/Analysis.java | 20 ++----------- .../index/analysis/AnalysisRegistry.java | 27 +++++++++++++---- .../index/analysis/AnalyzerComponents.java | 10 ++++++- .../analysis/CustomAnalyzerProvider.java | 7 +++-- .../analysis/ReloadableCustomAnalyzer.java | 10 ++++++- .../index/analysis/TokenFilterFactory.java | 10 +++++++ .../index/mapper/MapperService.java | 11 ++++++- .../elasticsearch/indices/IndicesService.java | 28 +++++++++++++++++ .../support/AggregationContext.java | 10 ++++++- .../indices/TransportAnalyzeActionTests.java | 2 +- .../index/analysis/AnalysisRegistryTests.java | 30 ++++++++++++++----- .../ReloadableCustomAnalyzerTests.java | 4 +++ .../indices/analysis/AnalysisModuleTests.java | 3 +- .../IncorrectSetupStablePluginsTests.java | 3 +- .../StableAnalysisPluginsNoSettingsTests.java | 3 +- ...tableAnalysisPluginsWithSettingsTests.java | 3 +- .../index/analysis/AnalysisTestsHelper.java | 3 +- .../test/AbstractBuilderTestCase.java | 3 +- .../org/elasticsearch/test/ESTestCase.java | 3 +- .../CategorizationAnalyzer.java | 10 ++++++- 36 files changed, 225 insertions(+), 95 deletions(-) diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index fa8103225c36..c4f8915811ae 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -154,7 +154,6 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, Scri private final SetOnce scriptServiceHolder = new SetOnce<>(); private final SetOnce synonymsManagementServiceHolder = new SetOnce<>(); - private final SetOnce threadPoolHolder = new SetOnce<>(); @Override public Collection createComponents( @@ -175,7 +174,6 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, Scri ) { this.scriptServiceHolder.set(scriptService); this.synonymsManagementServiceHolder.set(new SynonymsManagementAPIService(client)); - this.threadPoolHolder.set(threadPool); return Collections.emptyList(); } @@ -341,22 +339,11 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, Scri filters.put("stemmer", StemmerTokenFilterFactory::new); filters.put( "synonym", - requiresAnalysisSettings( - (i, e, n, s) -> new SynonymTokenFilterFactory(i, e, n, s, synonymsManagementServiceHolder.get(), threadPoolHolder.get()) - ) + requiresAnalysisSettings((i, e, n, s) -> new SynonymTokenFilterFactory(i, e, n, s, synonymsManagementServiceHolder.get())) ); filters.put( "synonym_graph", - requiresAnalysisSettings( - (i, e, n, s) -> new SynonymGraphTokenFilterFactory( - i, - e, - n, - s, - synonymsManagementServiceHolder.get(), - threadPoolHolder.get() - ) - ) + requiresAnalysisSettings((i, e, n, s) -> new SynonymGraphTokenFilterFactory(i, e, n, s, synonymsManagementServiceHolder.get())) ); filters.put("trim", TrimTokenFilterFactory::new); filters.put("truncate", requiresAnalysisSettings(TruncateTokenFilterFactory::new)); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java index ad0881b300b5..fba6b78a23cf 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java @@ -16,6 +16,7 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.AnalysisMode; @@ -51,6 +52,7 @@ public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory { @Override public TokenFilterFactory getChainAwareTokenFilterFactory( + IndexCreationContext context, TokenizerFactory tokenizer, List charFilters, List previousTokenFilters, @@ -66,7 +68,7 @@ public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory { String[] parts = Strings.tokenizeToStringArray(filter, ","); if (parts.length == 1) { TokenFilterFactory factory = resolveFilterFactory(allFilters, parts[0]); - factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, previousTokenFilters, allFilters); + factory = factory.getChainAwareTokenFilterFactory(context, tokenizer, charFilters, previousTokenFilters, allFilters); filters.add(factory); mode = mode.merge(factory.getAnalysisMode()); } else { @@ -74,7 +76,7 @@ public class MultiplexerTokenFilterFactory extends AbstractTokenFilterFactory { List chain = new ArrayList<>(); for (String subfilter : parts) { TokenFilterFactory factory = resolveFilterFactory(allFilters, subfilter); - factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, existingChain, allFilters); + factory = factory.getChainAwareTokenFilterFactory(context, tokenizer, charFilters, existingChain, allFilters); chain.add(factory); existingChain.add(factory); mode = mode.merge(factory.getAnalysisMode()); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java index d758bcb9987a..b41e52835ebe 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java @@ -11,6 +11,7 @@ package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.CharFilterFactory; @@ -57,6 +58,7 @@ public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFact @Override public TokenFilterFactory getChainAwareTokenFilterFactory( + IndexCreationContext context, TokenizerFactory tokenizer, List charFilters, List previousTokenFilters, @@ -71,7 +73,7 @@ public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFact "ScriptedConditionTokenFilter [" + name() + "] refers to undefined token filter [" + filter + "]" ); } - tff = tff.getChainAwareTokenFilterFactory(tokenizer, charFilters, existingChain, allFilters); + tff = tff.getChainAwareTokenFilterFactory(context, tokenizer, charFilters, existingChain, allFilters); filters.add(tff); existingChain.add(tff); } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java index 5ceee45e606d..9626e44a74bd 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java @@ -14,13 +14,13 @@ import org.apache.lucene.analysis.synonym.SynonymGraphFilter; import org.apache.lucene.analysis.synonym.SynonymMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisMode; import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.synonyms.SynonymsManagementAPIService; -import org.elasticsearch.threadpool.ThreadPool; import java.util.List; import java.util.function.Function; @@ -32,10 +32,9 @@ public class SynonymGraphTokenFilterFactory extends SynonymTokenFilterFactory { Environment env, String name, Settings settings, - SynonymsManagementAPIService synonymsManagementAPIService, - ThreadPool threadPool + SynonymsManagementAPIService synonymsManagementAPIService ) { - super(indexSettings, env, name, settings, synonymsManagementAPIService, threadPool); + super(indexSettings, env, name, settings, synonymsManagementAPIService); } @Override @@ -45,13 +44,14 @@ public class SynonymGraphTokenFilterFactory extends SynonymTokenFilterFactory { @Override public TokenFilterFactory getChainAwareTokenFilterFactory( + IndexCreationContext context, TokenizerFactory tokenizer, List charFilters, List previousTokenFilters, Function allFilters ) { final Analyzer analyzer = buildSynonymAnalyzer(tokenizer, charFilters, previousTokenFilters); - ReaderWithOrigin rulesFromSettings = getRulesFromSettings(environment); + ReaderWithOrigin rulesFromSettings = getRulesFromSettings(environment, context); final SynonymMap synonyms = buildSynonyms(analyzer, rulesFromSettings); final String name = name(); return new TokenFilterFactory() { diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java index 8d37a84a9fae..a66c657b7b59 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java @@ -12,11 +12,11 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.synonym.SynonymFilter; import org.apache.lucene.analysis.synonym.SynonymMap; -import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.Analysis; @@ -27,7 +27,6 @@ import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.synonyms.SynonymsAPI; import org.elasticsearch.synonyms.SynonymsManagementAPIService; -import org.elasticsearch.threadpool.ThreadPool; import java.io.Reader; import java.io.StringReader; @@ -45,15 +44,13 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { protected final Environment environment; protected final AnalysisMode analysisMode; private final SynonymsManagementAPIService synonymsManagementAPIService; - private final ThreadPool threadPool; SynonymTokenFilterFactory( IndexSettings indexSettings, Environment env, String name, Settings settings, - SynonymsManagementAPIService synonymsManagementAPIService, - ThreadPool threadPool + SynonymsManagementAPIService synonymsManagementAPIService ) { super(name, settings); this.settings = settings; @@ -73,7 +70,6 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { this.analysisMode = updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL; this.environment = env; this.synonymsManagementAPIService = synonymsManagementAPIService; - this.threadPool = threadPool; } @Override @@ -88,13 +84,14 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { @Override public TokenFilterFactory getChainAwareTokenFilterFactory( + IndexCreationContext context, TokenizerFactory tokenizer, List charFilters, List previousTokenFilters, Function allFilters ) { final Analyzer analyzer = buildSynonymAnalyzer(tokenizer, charFilters, previousTokenFilters); - ReaderWithOrigin rulesFromSettings = getRulesFromSettings(environment); + ReaderWithOrigin rulesFromSettings = getRulesFromSettings(environment, context); final SynonymMap synonyms = buildSynonyms(analyzer, rulesFromSettings); final String name = name(); return new TokenFilterFactory() { @@ -156,7 +153,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { } } - protected ReaderWithOrigin getRulesFromSettings(Environment env) { + protected ReaderWithOrigin getRulesFromSettings(Environment env, IndexCreationContext context) { if (settings.getAsList("synonyms", null) != null) { List rulesList = Analysis.getWordList(env, settings, "synonyms"); StringBuilder sb = new StringBuilder(); @@ -171,15 +168,17 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { ); } String synonymsSet = settings.get("synonyms_set", null); - // provide fake synonyms on master thread, as on Master an analyzer is built for validation only - if (MasterService.isMasterUpdateThread()) { + // provide fake synonyms on index creation and index metadata checks to ensure that we + // don't block a master thread + if (context != IndexCreationContext.RELOAD_ANALYZERS) { return new ReaderWithOrigin( new StringReader("fake rule => fake"), - "fake [" + synonymsSet + "] synonyms_set in .synonyms index" + "fake [" + synonymsSet + "] synonyms_set in .synonyms index", + synonymsSet ); } return new ReaderWithOrigin( - Analysis.getReaderFromIndex(synonymsSet, threadPool, synonymsManagementAPIService), + Analysis.getReaderFromIndex(synonymsSet, synonymsManagementAPIService), "[" + synonymsSet + "] synonyms_set in .synonyms index", synonymsSet ); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java index 2b7421d95576..79a083073213 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.MyFilterTokenFilterFactory; @@ -64,7 +65,7 @@ public class CompoundAnalysisTests extends ESTestCase { private List analyze(Settings settings, String analyzerName, String text) throws IOException { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); AnalysisModule analysisModule = createAnalysisModule(settings); - IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(idxSettings); + IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); Analyzer analyzer = indexAnalyzers.get(analyzerName).analyzer(); TokenStream stream = analyzer.tokenStream("", text); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java index 1f2ac2078e2a..766fc444d887 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -41,7 +42,7 @@ public class EdgeNGramTokenizerTests extends ESTokenStreamTestCase { TestEnvironment.newEnvironment(settings), Collections.singletonList(new CommonAnalysisPlugin()), new StablePluginsRegistry() - ).getAnalysisRegistry().build(idxSettings); + ).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); } public void testPreConfiguredTokenizer() throws IOException { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterTests.java index b145c8fee256..1228be64130a 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -44,7 +45,7 @@ public class MultiplexerTokenFilterTests extends ESTokenStreamTestCase { TestEnvironment.newEnvironment(settings), Collections.singletonList(new CommonAnalysisPlugin()), new StablePluginsRegistry() - ).getAnalysisRegistry().build(idxSettings); + ).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); try (NamedAnalyzer analyzer = indexAnalyzers.get("myAnalyzer")) { assertNotNull(analyzer); @@ -79,7 +80,7 @@ public class MultiplexerTokenFilterTests extends ESTokenStreamTestCase { TestEnvironment.newEnvironment(settings), Collections.singletonList(new CommonAnalysisPlugin()), new StablePluginsRegistry() - ).getAnalysisRegistry().build(idxSettings); + ).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); try (NamedAnalyzer analyzer = indexAnalyzers.get("myAnalyzer")) { assertNotNull(analyzer); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java index c55f8d36ee4f..8837695e876f 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -74,7 +75,7 @@ public class PredicateTokenScriptFilterTests extends ESTokenStreamTestCase { new StablePluginsRegistry() ); - IndexAnalyzers analyzers = module.getAnalysisRegistry().build(idxSettings); + IndexAnalyzers analyzers = module.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); try (NamedAnalyzer analyzer = analyzers.get("myAnalyzer")) { assertNotNull(analyzer); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java index dc4887413092..ab644f769436 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -74,7 +75,7 @@ public class ScriptedConditionTokenFilterTests extends ESTokenStreamTestCase { new StablePluginsRegistry() ); - IndexAnalyzers analyzers = module.getAnalysisRegistry().build(idxSettings); + IndexAnalyzers analyzers = module.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); try (NamedAnalyzer analyzer = analyzers.get("myAnalyzer")) { assertNotNull(analyzer); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java index caec5b204669..d4d6b9c55083 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java @@ -268,7 +268,7 @@ public class SynonymsAnalysisTests extends ESTestCase { for (String factory : bypassingFactories) { TokenFilterFactory tff = plugin.getTokenFilters().get(factory).get(idxSettings, null, factory, settings); TokenizerFactory tok = new KeywordTokenizerFactory(idxSettings, null, "keyword", settings); - SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings, null, null); + SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings, null); Analyzer analyzer = SynonymTokenFilterFactory.buildSynonymAnalyzer( tok, Collections.emptyList(), @@ -338,7 +338,7 @@ public class SynonymsAnalysisTests extends ESTestCase { for (String factory : disallowedFactories) { TokenFilterFactory tff = plugin.getTokenFilters().get(factory).get(idxSettings, null, factory, settings); TokenizerFactory tok = new KeywordTokenizerFactory(idxSettings, null, "keyword", settings); - SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings, null, null); + SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings, null); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java index 22a1868785e4..31550ab0945f 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisTestsHelper; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -196,7 +197,7 @@ public class WordDelimiterGraphTokenFilterFactoryTests extends BaseWordDelimiter TestEnvironment.newEnvironment(settings), Collections.singletonList(new CommonAnalysisPlugin()), new StablePluginsRegistry() - ).getAnalysisRegistry().build(idxSettings) + ).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings) ) { NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); @@ -221,7 +222,7 @@ public class WordDelimiterGraphTokenFilterFactoryTests extends BaseWordDelimiter TestEnvironment.newEnvironment(settings), Collections.singletonList(new CommonAnalysisPlugin()), new StablePluginsRegistry() - ).getAnalysisRegistry().build(idxSettings) + ).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings) ) { NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); diff --git a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/70_synonyms_from_index.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/70_synonyms_from_index.yml index fd8089d98d64..a0aca1b1d1f1 100644 --- a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/70_synonyms_from_index.yml +++ b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/70_synonyms_from_index.yml @@ -141,6 +141,25 @@ setup: query: bye - match: { hits.total.value: 1 } + - do: + indices.close: + index: my_index + + - do: + indices.open: + index: my_index + + # Confirm that the index analyzers are reloaded + - do: + search: + index: my_index + body: + query: + match: + my_field: + query: hola + - match: { hits.total.value: 1 } + --- "Delete the synonyms set and confirm failed reload analyzers details": - do: diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 05131081af15..60bece582e5c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.IOUtils; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalyzerComponents; @@ -212,6 +213,7 @@ public class TransportAnalyzeAction extends TransportSingleShardAction 0) || (request.charFilters() != null && request.charFilters().size() > 0))) { return analysisRegistry.buildCustomAnalyzer( + IndexCreationContext.RELOAD_ANALYZERS, indexSettings, true, new NameOrDefinition("keyword"), diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 11f7b07f6904..c4e44fb615f1 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -35,6 +35,7 @@ import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.cache.query.DisabledQueryCache; @@ -456,7 +457,7 @@ public final class IndexModule { } public IndexService newIndexService( - IndexService.IndexCreationContext indexCreationContext, + IndexCreationContext indexCreationContext, NodeEnvironment environment, XContentParserConfiguration parserConfiguration, IndexService.ShardStoreDeleter shardStoreDeleter, @@ -501,7 +502,7 @@ public final class IndexModule { queryCache = DisabledQueryCache.INSTANCE; } if (IndexService.needsMapperService(indexSettings, indexCreationContext)) { - indexAnalyzers = analysisRegistry.build(indexSettings); + indexAnalyzers = analysisRegistry.build(indexCreationContext, indexSettings); } final IndexService indexService = new IndexService( indexSettings, @@ -636,7 +637,7 @@ public final class IndexModule { return new MapperService( clusterService, indexSettings, - analysisRegistry.build(indexSettings), + analysisRegistry.build(IndexCreationContext.METADATA_VERIFICATION, indexSettings), parserConfiguration, new SimilarityService(indexSettings, scriptService, similarities), mapperRegistry, diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 9e2125c783cf..7dacb05166db 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -190,6 +190,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust Engine.IndexCommitListener indexCommitListener ) { super(indexSettings); + assert indexCreationContext != IndexCreationContext.RELOAD_ANALYZERS + : "IndexCreationContext.RELOAD_ANALYZERS should only be used when reloading analysers"; this.allowExpensiveQueries = allowExpensiveQueries; this.indexSettings = indexSettings; this.parserConfiguration = parserConfiguration; @@ -274,7 +276,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust public enum IndexCreationContext { CREATE_INDEX, - METADATA_VERIFICATION + METADATA_VERIFICATION, + RELOAD_ANALYZERS } public int numberOfShards() { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java b/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java index 31cd312998f7..7dd605c4c8a7 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java @@ -52,7 +52,6 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.synonyms.PagedResult; import org.elasticsearch.synonyms.SynonymRule; import org.elasticsearch.synonyms.SynonymsManagementAPIService; -import org.elasticsearch.threadpool.ThreadPool; import java.io.BufferedReader; import java.io.IOException; @@ -298,22 +297,9 @@ public class Analysis { } } - public static Reader getReaderFromIndex( - String synonymsSet, - ThreadPool threadPool, - SynonymsManagementAPIService synonymsManagementAPIService - ) { - // TODO: this is a temporary solution for loading synonyms under feature flag, to be redesigned for GA - final PlainActionFuture> synonymsLoadingFuture = new PlainActionFuture<>() { - @Override - protected boolean blockingAllowed() { - // allow blocking while loading synonyms under feature flag - return true; - } - }; - threadPool.executor(ThreadPool.Names.SYSTEM_READ).execute(() -> { - synonymsManagementAPIService.getSynonymSetRules(synonymsSet, 0, 10_000, synonymsLoadingFuture); - }); + public static Reader getReaderFromIndex(String synonymsSet, SynonymsManagementAPIService synonymsManagementAPIService) { + final PlainActionFuture> synonymsLoadingFuture = new PlainActionFuture<>(); + synonymsManagementAPIService.getSynonymSetRules(synonymsSet, 0, 10_000, synonymsLoadingFuture); PagedResult results = synonymsLoadingFuture.actionGet(); SynonymRule[] synonymRules = results.pageResults(); diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 863b1d6c9e83..4ef5ed4afb61 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -41,7 +42,7 @@ import static java.util.Collections.unmodifiableMap; /** * An internal registry for tokenizer, token filter, char filter and analyzer. - * This class exists per node and allows to create per-index {@link IndexAnalyzers} via {@link #build(IndexSettings)} + * This class exists per node and allows to create per-index {@link IndexAnalyzers} via {@link #build} */ public final class AnalysisRegistry implements Closeable { public static final String INDEX_ANALYSIS_CHAR_FILTER = "index.analysis.char_filter"; @@ -201,14 +202,23 @@ public final class AnalysisRegistry implements Closeable { /** * Creates an index-level {@link IndexAnalyzers} from this registry using the given index settings + * and {@link IndexCreationContext}. */ - public IndexAnalyzers build(IndexSettings indexSettings) throws IOException { + public IndexAnalyzers build(IndexCreationContext context, IndexSettings indexSettings) throws IOException { final Map charFilterFactories = buildCharFilterFactories(indexSettings); final Map tokenizerFactories = buildTokenizerFactories(indexSettings); final Map tokenFilterFactories = buildTokenFilterFactories(indexSettings); final Map> analyzerFactories = buildAnalyzerFactories(indexSettings); final Map> normalizerFactories = buildNormalizerFactories(indexSettings); - return build(indexSettings, analyzerFactories, normalizerFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories); + return build( + context, + indexSettings, + analyzerFactories, + normalizerFactories, + tokenizerFactories, + charFilterFactories, + tokenFilterFactories + ); } /** @@ -217,6 +227,7 @@ public final class AnalysisRegistry implements Closeable { * Callers are responsible for closing the returned Analyzer */ public NamedAnalyzer buildCustomAnalyzer( + IndexCreationContext context, IndexSettings indexSettings, boolean normalizer, NameOrDefinition tokenizer, @@ -259,7 +270,7 @@ public final class AnalysisRegistry implements Closeable { if (normalizer && tff instanceof NormalizingTokenFilterFactory == false) { throw new IllegalArgumentException("Custom normalizer may not use filter [" + tff.name() + "]"); } - tff = tff.getChainAwareTokenFilterFactory(tokenizerFactory, charFilterFactories, tokenFilterFactories, name -> { + tff = tff.getChainAwareTokenFilterFactory(context, tokenizerFactory, charFilterFactories, tokenFilterFactories, name -> { try { return getComponentFactory( indexSettings, @@ -281,7 +292,7 @@ public final class AnalysisRegistry implements Closeable { charFilterFactories.toArray(new CharFilterFactory[] {}), tokenFilterFactories.toArray(new TokenFilterFactory[] {}) ); - return produceAnalyzer("__custom__", new AnalyzerProvider<>() { + return produceAnalyzer(context, "__custom__", new AnalyzerProvider<>() { @Override public String name() { return "__custom__"; @@ -589,6 +600,7 @@ public final class AnalysisRegistry implements Closeable { } public static IndexAnalyzers build( + IndexCreationContext context, IndexSettings indexSettings, Map> analyzerProviders, Map> normalizerProviders, @@ -603,6 +615,7 @@ public final class AnalysisRegistry implements Closeable { analyzers.merge( entry.getKey(), produceAnalyzer( + context, entry.getKey(), entry.getValue(), tokenFilterFactoryFactories, @@ -641,6 +654,7 @@ public final class AnalysisRegistry implements Closeable { analyzers.put( DEFAULT_ANALYZER_NAME, produceAnalyzer( + context, DEFAULT_ANALYZER_NAME, new StandardAnalyzerProvider(indexSettings, null, DEFAULT_ANALYZER_NAME, Settings.EMPTY), tokenFilterFactoryFactories, @@ -677,6 +691,7 @@ public final class AnalysisRegistry implements Closeable { } private static NamedAnalyzer produceAnalyzer( + IndexCreationContext context, String name, AnalyzerProvider analyzerFactory, Map tokenFilters, @@ -691,7 +706,7 @@ public final class AnalysisRegistry implements Closeable { */ int overridePositionIncrementGap = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP; if (analyzerFactory instanceof CustomAnalyzerProvider) { - ((CustomAnalyzerProvider) analyzerFactory).build(tokenizers, charFilters, tokenFilters); + ((CustomAnalyzerProvider) analyzerFactory).build(context, tokenizers, charFilters, tokenFilters); /* * Custom analyzers already default to the correct, version * dependent positionIncrementGap and the user is be able to diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponents.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponents.java index 71e2175dacc6..0839145202ec 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponents.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponents.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.analysis; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexService.IndexCreationContext; import java.util.ArrayList; import java.util.List; @@ -38,6 +39,7 @@ public final class AnalyzerComponents { } static AnalyzerComponents createComponents( + IndexCreationContext context, String name, Settings analyzerSettings, final Map tokenizers, @@ -77,7 +79,13 @@ public final class AnalyzerComponents { "Custom Analyzer [" + name + "] failed to find filter under name " + "[" + tokenFilterName + "]" ); } - tokenFilter = tokenFilter.getChainAwareTokenFilterFactory(tokenizer, charFiltersList, tokenFilterList, tokenFilters::get); + tokenFilter = tokenFilter.getChainAwareTokenFilterFactory( + context, + tokenizer, + charFiltersList, + tokenFilterList, + tokenFilters::get + ); tokenFilterList.add(tokenFilter); } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java index 58a35ab22ef3..7aedd4c742c9 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java @@ -10,6 +10,7 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -33,11 +34,12 @@ public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider tokenizers, final Map charFilters, final Map tokenFilters ) { - customAnalyzer = create(name(), analyzerSettings, tokenizers, charFilters, tokenFilters); + customAnalyzer = create(context, name(), analyzerSettings, tokenizers, charFilters, tokenFilters); } /** @@ -45,6 +47,7 @@ public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider tokenizers, @@ -54,7 +57,7 @@ public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider charFilters, final Map tokenFilters ) { - AnalyzerComponents components = AnalyzerComponents.createComponents(name, settings, tokenizers, charFilters, tokenFilters); + AnalyzerComponents components = AnalyzerComponents.createComponents( + IndexCreationContext.RELOAD_ANALYZERS, + name, + settings, + tokenizers, + charFilters, + tokenFilters + ); this.components = components; } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java index 53da8bbd7641..b8cea068f85d 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java @@ -9,7 +9,9 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter; import java.util.List; @@ -41,12 +43,20 @@ public interface TokenFilterFactory { /** * Rewrite the TokenFilterFactory to take into account the preceding analysis chain, or refer * to other TokenFilterFactories + * If the token filter is part of the definition of a {@link ReloadableCustomAnalyzer}, + * this function is called twice, once at index creation with {@link IndexCreationContext#CREATE_INDEX} + * and then later with {@link IndexCreationContext#RELOAD_ANALYZERS} on shard recovery. + * The {@link IndexCreationContext#RELOAD_ANALYZERS} context should be used to load expensive resources + * on a generic thread pool. See {@link SynonymGraphFilterFactory} for an example of how this context + * is used. + * @param context the IndexCreationContext for the underlying index * @param tokenizer the TokenizerFactory for the preceding chain * @param charFilters any CharFilterFactories for the preceding chain * @param previousTokenFilters a list of TokenFilterFactories in the preceding chain * @param allFilters access to previously defined TokenFilterFactories */ default TokenFilterFactory getChainAwareTokenFilterFactory( + IndexCreationContext context, TokenizerFactory tokenizer, List charFilters, List previousTokenFilters, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 3556843a301e..5fcb84a119a8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -525,7 +526,15 @@ public class MapperService extends AbstractIndexComponent implements Closeable { return mappingLookup().isMultiField(field); } - public synchronized List reloadSearchAnalyzers(AnalysisRegistry registry, String resource) throws IOException { + /** + * Reload any search analyzers that have reloadable components if resource is {@code null}, + * otherwise only the provided resource is reloaded. + * @param registry the analysis registry + * @param resource the name of the reloadable resource or {@code null} if all resources should be reloaded. + * @return The names of reloaded resources. + * @throws IOException + */ + public synchronized List reloadSearchAnalyzers(AnalysisRegistry registry, @Nullable String resource) throws IOException { logger.info("reloading search analyzers"); // TODO this should bust the cache somehow. Tracked in https://github.com/elastic/elasticsearch/issues/66722 return indexAnalyzers.reload(registry, indexSettings, resource); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index de47fb04bf08..9ffcf174b708 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -18,6 +18,7 @@ import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.mapping.put.AutoPutMappingAction; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; @@ -626,8 +627,35 @@ public class IndicesService extends AbstractLifecycleComponent } } }; + final IndexEventListener beforeIndexShardRecovery = new IndexEventListener() { + volatile boolean reloaded; + + @Override + public void beforeIndexShardRecovery(IndexShard indexShard, IndexSettings indexSettings, ActionListener listener) { + try { + if (indexShard.mapperService() != null) { + // we need to reload once, not on every shard recovery in case multiple shards are on the same node + if (reloaded == false) { + synchronized (indexShard.mapperService()) { + if (reloaded == false) { + // we finish loading analyzers from resources here + // during shard recovery in the generic thread pool, + // as this may require longer running operations and blocking calls + indexShard.mapperService().reloadSearchAnalyzers(getAnalysis(), null); + } + reloaded = true; + } + } + } + listener.onResponse(null); + } catch (Exception e) { + listener.onFailure(e); + } + } + }; finalListeners.add(onStoreClose); finalListeners.add(oldShardsStats); + finalListeners.add(beforeIndexShardRecovery); IndexService indexService; try (var ignored = threadPool.getThreadContext().newStoredContext()) { indexService = createIndexService( diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java index bd4ce4691265..653aff4935c9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.NameOrDefinition; @@ -443,7 +444,14 @@ public abstract class AggregationContext implements Releasable { List charFilters, List tokenFilters ) throws IOException { - return analysisRegistry.buildCustomAnalyzer(indexSettings, normalizer, tokenizer, charFilters, tokenFilters); + return analysisRegistry.buildCustomAnalyzer( + IndexService.IndexCreationContext.RELOAD_ANALYZERS, + indexSettings, + normalizer, + tokenizer, + charFilters, + tokenFilters + ); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java index dfcc03b6fa65..395a228e6db4 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java @@ -140,7 +140,7 @@ public class TransportAnalyzeActionTests extends ESTestCase { } }; registry = new AnalysisModule(environment, singletonList(plugin), new StablePluginsRegistry()).getAnalysisRegistry(); - indexAnalyzers = registry.build(this.indexSettings); + indexAnalyzers = registry.build(IndexService.IndexCreationContext.RELOAD_ANALYZERS, this.indexSettings); maxTokenCount = IndexSettings.MAX_TOKEN_COUNT_SETTING.getDefault(settings); idxMaxTokenCount = this.indexSettings.getMaxTokenCount(); } diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index da7f0e4dc606..5b7af56b7e6e 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -107,7 +108,7 @@ public class AnalysisRegistryTests extends ESTestCase { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - IndexAnalyzers indexAnalyzers = emptyRegistry.build(idxSettings); + IndexAnalyzers indexAnalyzers = emptyRegistry.build(IndexCreationContext.CREATE_INDEX, idxSettings); assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class)); @@ -118,6 +119,7 @@ public class AnalysisRegistryTests extends ESTestCase { Version version = VersionUtils.randomVersion(random()); Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); IndexAnalyzers indexAnalyzers = AnalysisRegistry.build( + IndexCreationContext.CREATE_INDEX, IndexSettingsModule.newIndexSettings("index", settings), singletonMap("default", analyzerProvider("default")), emptyMap(), @@ -155,6 +157,7 @@ public class AnalysisRegistryTests extends ESTestCase { MapperException ex = expectThrows( MapperException.class, () -> AnalysisRegistry.build( + IndexCreationContext.CREATE_INDEX, IndexSettingsModule.newIndexSettings("index", settings), singletonMap("default", new PreBuiltAnalyzerProvider("default", AnalyzerScope.INDEX, analyzer)), emptyMap(), @@ -169,7 +172,10 @@ public class AnalysisRegistryTests extends ESTestCase { public void testNameClashNormalizer() throws IOException { // Test out-of-the-box normalizer works OK. - IndexAnalyzers indexAnalyzers = nonEmptyRegistry.build(IndexSettingsModule.newIndexSettings("index", Settings.EMPTY)); + IndexAnalyzers indexAnalyzers = nonEmptyRegistry.build( + IndexCreationContext.CREATE_INDEX, + IndexSettingsModule.newIndexSettings("index", Settings.EMPTY) + ); assertNotNull(indexAnalyzers.getNormalizer("lowercase")); assertThat(indexAnalyzers.getNormalizer("lowercase").normalize("field", "AbC").utf8ToString(), equalTo("abc")); @@ -181,7 +187,7 @@ public class AnalysisRegistryTests extends ESTestCase { .putList("index.analysis.normalizer.lowercase.filter", "reverse") .build(); - indexAnalyzers = nonEmptyRegistry.build(IndexSettingsModule.newIndexSettings("index", settings)); + indexAnalyzers = nonEmptyRegistry.build(IndexCreationContext.CREATE_INDEX, IndexSettingsModule.newIndexSettings("index", settings)); assertNotNull(indexAnalyzers.getNormalizer("lowercase")); assertThat(indexAnalyzers.getNormalizer("lowercase").normalize("field", "AbC").utf8ToString(), equalTo("CbA")); } @@ -193,6 +199,7 @@ public class AnalysisRegistryTests extends ESTestCase { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> AnalysisRegistry.build( + IndexCreationContext.CREATE_INDEX, IndexSettingsModule.newIndexSettings("index", settings), singletonMap("default_index", defaultIndex), emptyMap(), @@ -208,6 +215,7 @@ public class AnalysisRegistryTests extends ESTestCase { Version version = VersionUtils.randomVersion(random()); Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); IndexAnalyzers indexAnalyzers = AnalysisRegistry.build( + IndexCreationContext.CREATE_INDEX, IndexSettingsModule.newIndexSettings("index", settings), singletonMap("default_search", analyzerProvider("default_search")), emptyMap(), @@ -263,7 +271,7 @@ public class AnalysisRegistryTests extends ESTestCase { TestEnvironment.newEnvironment(settings), singletonList(plugin), new StablePluginsRegistry() - ).getAnalysisRegistry().build(idxSettings); + ).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); // This shouldn't contain English stopwords try (NamedAnalyzer custom_analyser = indexAnalyzers.get("custom_analyzer_with_camel_case")) { @@ -298,8 +306,8 @@ public class AnalysisRegistryTests extends ESTestCase { Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); - IndexAnalyzers indexAnalyzers = emptyAnalysisRegistry(settings).build(idxSettings); - IndexAnalyzers otherIndexAnalyzers = emptyAnalysisRegistry(settings).build(idxSettings); + IndexAnalyzers indexAnalyzers = emptyAnalysisRegistry(settings).build(IndexCreationContext.CREATE_INDEX, idxSettings); + IndexAnalyzers otherIndexAnalyzers = emptyAnalysisRegistry(settings).build(IndexCreationContext.CREATE_INDEX, idxSettings); final int numIters = randomIntBetween(5, 20); for (int i = 0; i < numIters; i++) { PreBuiltAnalyzers preBuiltAnalyzers = RandomPicks.randomFrom(random(), PreBuiltAnalyzers.values()); @@ -317,12 +325,18 @@ public class AnalysisRegistryTests extends ESTestCase { .build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> emptyAnalysisRegistry(settings).build(idxSettings)); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> emptyAnalysisRegistry(settings).build(IndexCreationContext.CREATE_INDEX, idxSettings) + ); assertThat(e.getMessage(), equalTo("analyzer [test_analyzer] must specify either an analyzer type, or a tokenizer")); } public void testCloseIndexAnalyzersMultipleTimes() throws IOException { - IndexAnalyzers indexAnalyzers = emptyRegistry.build(indexSettingsOfCurrentVersion(Settings.builder())); + IndexAnalyzers indexAnalyzers = emptyRegistry.build( + IndexCreationContext.CREATE_INDEX, + indexSettingsOfCurrentVersion(Settings.builder()) + ); indexAnalyzers.close(); indexAnalyzers.close(); } diff --git a/server/src/test/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzerTests.java b/server/src/test/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzerTests.java index d93445ca2b81..1b9fccb46ec0 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzerTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzerTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; @@ -72,6 +73,7 @@ public class ReloadableCustomAnalyzerTests extends ESTestCase { Settings analyzerSettings = Settings.builder().put("tokenizer", "standard").putList("filter", "my_filter").build(); AnalyzerComponents components = createComponents( + IndexCreationContext.CREATE_INDEX, "my_analyzer", analyzerSettings, testAnalysis.tokenizer, @@ -92,6 +94,7 @@ public class ReloadableCustomAnalyzerTests extends ESTestCase { // check that when using regular non-search time filters only, we get an exception final Settings indexAnalyzerSettings = Settings.builder().put("tokenizer", "standard").putList("filter", "lowercase").build(); AnalyzerComponents indexAnalyzerComponents = createComponents( + IndexCreationContext.CREATE_INDEX, "my_analyzer", indexAnalyzerSettings, testAnalysis.tokenizer, @@ -115,6 +118,7 @@ public class ReloadableCustomAnalyzerTests extends ESTestCase { Settings analyzerSettings = Settings.builder().put("tokenizer", "standard").putList("filter", "my_filter").build(); AnalyzerComponents components = createComponents( + IndexCreationContext.RELOAD_ANALYZERS, "my_analyzer", analyzerSettings, testAnalysis.tokenizer, diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index b28d330e8355..2d5cc7f6e5bf 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.Analysis; @@ -75,7 +76,7 @@ public class AnalysisModuleTests extends ESTestCase { public IndexAnalyzers getIndexAnalyzers(AnalysisRegistry registry, Settings settings) throws IOException { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - return registry.build(idxSettings); + return registry.build(IndexCreationContext.CREATE_INDEX, idxSettings); } public AnalysisRegistry getNewRegistry(Settings settings) { diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java index 68d879686269..fe6dfbd14a6f 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -122,7 +123,7 @@ public class IncorrectSetupStablePluginsTests extends ESTestCase { AnalysisRegistry registry = setupRegistry(mapOfCharFilters); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - return registry.build(idxSettings); + return registry.build(IndexCreationContext.CREATE_INDEX, idxSettings); } private AnalysisRegistry setupRegistry(Map mapOfCharFilters) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java index 76d3ff14c9f4..3ea2f3ec5144 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -55,7 +56,7 @@ public class StableAnalysisPluginsNoSettingsTests extends ESTestCase { AnalysisRegistry registry = setupRegistry(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - return registry.build(idxSettings); + return registry.build(IndexCreationContext.CREATE_INDEX, idxSettings); } public void testStablePlugins() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java index 0a8d38b32c09..dcc70313e95b 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -132,7 +133,7 @@ public class StableAnalysisPluginsWithSettingsTests extends ESTestCase { AnalysisRegistry registry = setupRegistry(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); - return registry.build(idxSettings); + return registry.build(IndexCreationContext.CREATE_INDEX, idxSettings); } private AnalysisRegistry setupRegistry() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java b/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java index ee9f363a49fb..aef85a415ee0 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java @@ -12,6 +12,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugins.AnalysisPlugin; @@ -61,7 +62,7 @@ public class AnalysisTestsHelper { new StablePluginsRegistry() ).getAnalysisRegistry(); return new ESTestCase.TestAnalysis( - analysisRegistry.build(indexSettings), + analysisRegistry.build(IndexCreationContext.CREATE_INDEX, indexSettings), analysisRegistry.buildTokenFilterFactories(indexSettings), analysisRegistry.buildTokenizerFactories(indexSettings), analysisRegistry.buildCharFilterFactories(indexSettings) diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 1c23447a1a37..f334777a9903 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -37,6 +37,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -455,7 +456,7 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { emptyList(), new StablePluginsRegistry() ); - IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(idxSettings); + IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings); scriptService = new MockScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts); similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap()); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 4872163b0083..f382e162ba08 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -83,6 +83,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalysisRegistry; @@ -1749,7 +1750,7 @@ public abstract class ESTestCase extends LuceneTestCase { AnalysisModule analysisModule = new AnalysisModule(env, Arrays.asList(analysisPlugins), new StablePluginsRegistry()); AnalysisRegistry analysisRegistry = analysisModule.getAnalysisRegistry(); return new TestAnalysis( - analysisRegistry.build(indexSettings), + analysisRegistry.build(IndexCreationContext.CREATE_INDEX, indexSettings), analysisRegistry.buildTokenFilterFactories(indexSettings), analysisRegistry.buildTokenizerFactories(indexSettings), analysisRegistry.buildCharFilterFactories(indexSettings) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java index fb7b30b8ffd9..04dcda7f160e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java @@ -12,6 +12,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig; @@ -112,7 +113,14 @@ public class CategorizationAnalyzer implements Releasable { return new Tuple<>(globalAnalyzer, Boolean.FALSE); } else { return new Tuple<>( - analysisRegistry.buildCustomAnalyzer(null, false, config.getTokenizer(), config.getCharFilters(), config.getTokenFilters()), + analysisRegistry.buildCustomAnalyzer( + IndexCreationContext.RELOAD_ANALYZERS, + null, + false, + config.getTokenizer(), + config.getCharFilters(), + config.getTokenFilters() + ), Boolean.TRUE ); }