From ddb97cc0422e5eb5d8f266853b86aefe4636bd9e Mon Sep 17 00:00:00 2001 From: Navarone Feekery <13634519+navarone-feekery@users.noreply.github.com> Date: Tue, 28 Jan 2025 08:11:51 +0100 Subject: [PATCH] Revert "[Search] Add system index descriptors to Connector indices (#118991)" (#120951) This reverts commit 385e1fdf21fb30c20adf94c29f35703b344b97f5. --- .../xpack/core/ClientHelper.java | 3 - .../elastic-connectors-mappings.json} | 51 +++---- .../elastic-connectors-settings.json | 14 ++ ...lastic-connectors-sync-jobs-mappings.json} | 30 ++-- .../elastic-connectors-sync-jobs.json | 14 ++ .../connector/elastic-connectors.json | 14 ++ .../xpack/application/EnterpriseSearch.java | 9 +- .../connector/ConnectorIndexService.java | 50 +----- .../connector/ConnectorTemplateRegistry.java | 81 +++++++++- .../syncjob/ConnectorSyncJobIndexService.java | 47 +----- .../connector/ConnectorIndexServiceTests.java | 25 +-- .../ConnectorTemplateRegistryTests.java | 144 +++++++++++++++++- .../connector/ConnectorTestUtils.java | 53 ++++++- .../ConnectorSyncJobIndexServiceTests.java | 44 ++---- .../syncjob/ConnectorSyncJobTestUtils.java | 26 ++-- 15 files changed, 375 insertions(+), 230 deletions(-) rename x-pack/plugin/core/template-resources/src/main/resources/{elastic-connectors.json => entsearch/connector/elastic-connectors-mappings.json} (92%) create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-settings.json rename x-pack/plugin/core/template-resources/src/main/resources/{elastic-connectors-sync-jobs.json => entsearch/connector/elastic-connectors-sync-jobs-mappings.json} (88%) create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs.json create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors.json diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java index 680b72cb970c..9a0d1a58a30a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java @@ -196,9 +196,6 @@ public final class ClientHelper { public static final String APM_ORIGIN = "apm"; public static final String OTEL_ORIGIN = "otel"; public static final String REINDEX_DATA_STREAM_ORIGIN = "reindex_data_stream"; - // TODO consolidate the Kibana origin with the one defined in org/elasticsearch/kibana/KibanaPlugin.java - public static final String KIBANA_ORIGIN = "kibana"; - public static final String CLOUD_ORIGIN = "cloud"; private ClientHelper() {} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json similarity index 92% rename from x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors.json rename to x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json index a98018e76f0e..25409dbf8460 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json @@ -1,35 +1,29 @@ { - "settings": { - "index": { - "number_of_shards": "1", - "auto_expand_replicas": "0-1" - } - }, - "mappings": { - "_doc": { - "dynamic": "strict", + "template": { + "aliases": { + ".elastic-connectors": {} + }, + "mappings": { + "dynamic": "false", "_meta": { - "version": "${elastic-connectors.version}", - "managed_index_mappings_version": ${elastic-connectors.managed.index.version} + "pipeline": { + "default_name": "search-default-ingestion", + "default_extract_binary_content": true, + "default_run_ml_inference": true, + "default_reduce_whitespace": true + }, + "version": ${xpack.application.connector.template.version} }, "properties": { "api_key_id": { "type": "keyword" }, - "api_key_secret_id": { - "type": "keyword" - }, "configuration": { - "dynamic": "false", "type": "object" }, "custom_scheduling": { - "dynamic": "false", "type": "object" }, - "deleted": { - "type": "boolean" - }, "description": { "type": "text" }, @@ -37,7 +31,6 @@ "type": "keyword" }, "features": { - "dynamic": "false", "properties": { "filtering_advanced_config": { "type": "boolean" @@ -73,7 +66,6 @@ } }, "filtering": { - "dynamic": "false", "properties": { "active": { "properties": { @@ -86,7 +78,6 @@ "type": "date" }, "value": { - "dynamic": "false", "type": "object" } } @@ -152,7 +143,6 @@ "type": "date" }, "value": { - "dynamic": "false", "type": "object" } } @@ -252,7 +242,6 @@ "type": "keyword" }, "pipeline": { - "dynamic": "false", "properties": { "extract_binary_content": { "type": "boolean" @@ -269,7 +258,6 @@ } }, "scheduling": { - "dynamic": "false", "properties": { "access_control": { "properties": { @@ -310,13 +298,22 @@ "type": "keyword" }, "sync_cursor": { - "dynamic": "false", "type": "object" }, "sync_now": { "type": "boolean" + }, + "deleted": { + "type": "boolean" } } } - } + }, + "_meta": { + "description": "Built-in mappings applied by default to elastic-connectors indices", + "managed": true + }, + "version": ${xpack.application.connector.template.version} } + + diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-settings.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-settings.json new file mode 100644 index 000000000000..6ff951057428 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-settings.json @@ -0,0 +1,14 @@ +{ + "template": { + "settings": { + "hidden": true, + "number_of_shards": "1", + "auto_expand_replicas": "0-1" + } + }, + "_meta": { + "description": "Built-in settings applied by default to connector management indices", + "managed": true + }, + "version": ${xpack.application.connector.template.version} +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors-sync-jobs.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs-mappings.json similarity index 88% rename from x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors-sync-jobs.json rename to x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs-mappings.json index 7d1e7fa3a041..4dd6e0681c7c 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors-sync-jobs.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs-mappings.json @@ -1,16 +1,12 @@ { - "settings": { - "index": { - "number_of_shards": "1", - "auto_expand_replicas": "0-1" - } - }, - "mappings": { - "_doc": { - "dynamic": "strict", + "template": { + "aliases": { + ".elastic-connectors-sync-jobs": {} + }, + "mappings": { + "dynamic": "false", "_meta": { - "version": "${elastic-connectors-sync-jobs.version}", - "managed_index_mappings_version": ${elastic-connectors-sync-jobs.managed.index.version} + "version": ${xpack.application.connector.template.version} }, "properties": { "cancelation_requested_at": { @@ -25,11 +21,9 @@ "connector": { "properties": { "configuration": { - "dynamic": "false", "type": "object" }, "filtering": { - "dynamic": "false", "properties": { "advanced_snippet": { "properties": { @@ -97,7 +91,6 @@ "type": "keyword" }, "pipeline": { - "dynamic": "false", "properties": { "extract_binary_content": { "type": "boolean" @@ -117,7 +110,6 @@ "type": "keyword" }, "sync_cursor": { - "dynamic": "false", "type": "object" } } @@ -144,7 +136,6 @@ "type": "date" }, "metadata": { - "dynamic": "false", "type": "object" }, "started_at": { @@ -164,5 +155,10 @@ } } } - } + }, + "_meta": { + "description": "Built-in mappings applied by default to elastic-connectors indices", + "managed": true + }, + "version": ${xpack.application.connector.template.version} } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs.json new file mode 100644 index 000000000000..db5404a30c6e --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs.json @@ -0,0 +1,14 @@ +{ + "index_patterns": ["${connectors-sync-jobs.index_pattern}"], + "priority": 100, + "composed_of": [ + "elastic-connectors-settings", + "elastic-connectors-sync-jobs-mappings" + ], + "allow_auto_create": true, + "_meta": { + "description": "Built-in template for elastic-connectors-sync-jobs", + "managed": true + }, + "version": ${xpack.application.connector.template.version} +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors.json new file mode 100644 index 000000000000..17c0b1eef061 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors.json @@ -0,0 +1,14 @@ +{ + "index_patterns": ["${connectors.index_pattern}"], + "priority": 100, + "composed_of": [ + "elastic-connectors-settings", + "elastic-connectors-mappings" + ], + "allow_auto_create": true, + "_meta": { + "description": "Built-in template for elastic-connectors", + "managed": true + }, + "version": ${xpack.application.connector.template.version} +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index 4142d907d0c5..df1c76ccf770 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -46,7 +46,6 @@ import org.elasticsearch.xpack.application.analytics.action.TransportPostAnalyti import org.elasticsearch.xpack.application.analytics.action.TransportPutAnalyticsCollectionAction; import org.elasticsearch.xpack.application.analytics.ingest.AnalyticsEventIngestConfig; import org.elasticsearch.xpack.application.connector.ConnectorAPIFeature; -import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import org.elasticsearch.xpack.application.connector.action.DeleteConnectorAction; import org.elasticsearch.xpack.application.connector.action.GetConnectorAction; @@ -125,7 +124,6 @@ import org.elasticsearch.xpack.application.connector.secrets.action.TransportDel import org.elasticsearch.xpack.application.connector.secrets.action.TransportGetConnectorSecretAction; import org.elasticsearch.xpack.application.connector.secrets.action.TransportPostConnectorSecretAction; import org.elasticsearch.xpack.application.connector.secrets.action.TransportPutConnectorSecretAction; -import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; import org.elasticsearch.xpack.application.connector.syncjob.action.CancelConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.CheckInConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.ClaimConnectorSyncJobAction; @@ -479,12 +477,7 @@ public class EnterpriseSearch extends Plugin implements ActionPlugin, SystemInde @Override public Collection getSystemIndexDescriptors(Settings settings) { Collection systemIndices = new ArrayList<>( - List.of( - SearchApplicationIndexService.getSystemIndexDescriptor(), - QueryRulesIndexService.getSystemIndexDescriptor(), - ConnectorSyncJobIndexService.getSystemIndexDescriptor(), - ConnectorIndexService.getSystemIndexDescriptor() - ) + List.of(SearchApplicationIndexService.getSystemIndexDescriptor(), QueryRulesIndexService.getSystemIndexDescriptor()) ); if (ConnectorSecretsFeature.isEnabled()) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index a9ca8552feee..3120124c1752 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -10,12 +10,10 @@ package org.elasticsearch.xpack.application.connector; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; @@ -35,7 +33,6 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -62,7 +59,6 @@ import org.elasticsearch.xpack.application.connector.filtering.FilteringValidati import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationState; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; -import org.elasticsearch.xpack.core.template.TemplateUtils; import java.time.Instant; import java.util.ArrayList; @@ -80,7 +76,6 @@ import java.util.stream.Collectors; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder; -import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTORS_ALLOWED_PRODUCT_ORIGINS; import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX; import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; @@ -92,20 +87,7 @@ public class ConnectorIndexService { // The client to interact with the system index (internal user). private final Client clientWithOrigin; - // TODO use proper version IDs (see org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java) - // TODO if this version is updated, a test should be added to - // javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java - private static final int CONNECTORS_INDEX_VERSION = 1; - // TODO rename to CONNECTOR_ALIAS_NAME - public static final String CONNECTOR_INDEX_NAME = ".elastic-connectors"; - public static final String CONNECTOR_INDEX_PREFIX = ".elastic-connectors-v"; - public static final String CONNECTOR_CONCRETE_INDEX_NAME = CONNECTOR_INDEX_PREFIX + CONNECTORS_INDEX_VERSION; - // The index pattern needs a stricter regex to prevent conflicts with .elastic-connectors-sync-jobs - - public static final String CONNECTOR_INDEX_NAME_PATTERN = CONNECTOR_INDEX_PREFIX + "*"; - - private static final String CONNECTORS_MAPPING_VERSION_VARIABLE = "elastic-connectors.version"; - private static final String CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE = "elastic-connectors.managed.index.version"; + public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; /** * @param client A client for executing actions on the connector index @@ -114,36 +96,6 @@ public class ConnectorIndexService { this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } - /** - * Returns the {@link SystemIndexDescriptor} for the Connector system index. - * - * @return The {@link SystemIndexDescriptor} for the Connector system index. - */ - public static SystemIndexDescriptor getSystemIndexDescriptor() { - PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - String templateSource = TemplateUtils.loadTemplate( - "/elastic-connectors.json", - Version.CURRENT.toString(), - CONNECTORS_MAPPING_VERSION_VARIABLE, - Map.of(CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE, Integer.toString(CONNECTORS_INDEX_VERSION)) - ); - request.source(templateSource, XContentType.JSON); - - // The index pattern needs a stricter regex to prevent conflicts with .elastic-connectors-sync-jobs - return SystemIndexDescriptor.builder() - .setIndexPattern(CONNECTOR_INDEX_NAME_PATTERN) - .setPrimaryIndex(CONNECTOR_CONCRETE_INDEX_NAME) - .setAliasName(CONNECTOR_INDEX_NAME) - .setDescription("Search connectors") - .setMappings(request.mappings()) - .setSettings(request.settings()) - .setOrigin(CONNECTORS_ORIGIN) - .setType(SystemIndexDescriptor.Type.EXTERNAL_MANAGED) - .setAllowedElasticProductOrigins(CONNECTORS_ALLOWED_PRODUCT_ORIGINS) - .setNetNew() - .build(); - } - /** * Creates or updates the {@link Connector} in the underlying index with a specific doc ID * if connectorId is provided. Otherwise, the connector doc is indexed with auto-generated doc ID. diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java index 97ac05c443ad..fd35acc89db5 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java @@ -8,23 +8,25 @@ package org.elasticsearch.xpack.application.connector; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; import org.elasticsearch.xpack.core.template.IngestPipelineConfig; import org.elasticsearch.xpack.core.template.JsonIngestPipelineConfig; +import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.elasticsearch.xpack.core.ClientHelper.CLOUD_ORIGIN; -import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; -import static org.elasticsearch.xpack.core.ClientHelper.KIBANA_ORIGIN; public class ConnectorTemplateRegistry extends IndexTemplateRegistry { @@ -32,6 +34,13 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { static final int REGISTRY_VERSION = 3; // Connector indices constants + + public static final String CONNECTOR_INDEX_NAME_PATTERN = ".elastic-connectors-v1"; + public static final String CONNECTOR_TEMPLATE_NAME = "elastic-connectors"; + + public static final String CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN = ".elastic-connectors-sync-jobs-v1"; + public static final String CONNECTOR_SYNC_JOBS_TEMPLATE_NAME = "elastic-connectors-sync-jobs"; + public static final String ACCESS_CONTROL_INDEX_PREFIX = ".search-acl-filter-"; public static final String ACCESS_CONTROL_INDEX_NAME_PATTERN = ".search-acl-filter-*"; public static final String ACCESS_CONTROL_TEMPLATE_NAME = "search-acl-filter"; @@ -49,8 +58,51 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { // Variable used to replace template version in index templates public static final String TEMPLATE_VERSION_VARIABLE = "xpack.application.connector.template.version"; - // Sources allowed to access system indices using X-elastic-product-origin header - public static final List CONNECTORS_ALLOWED_PRODUCT_ORIGINS = List.of(KIBANA_ORIGIN, CONNECTORS_ORIGIN, CLOUD_ORIGIN); + private static final String MAPPINGS_SUFFIX = "-mappings"; + + private static final String SETTINGS_SUFFIX = "-settings"; + + private static final String JSON_EXTENSION = ".json"; + + static final Map COMPONENT_TEMPLATES; + + static { + final Map componentTemplates = new HashMap<>(); + for (IndexTemplateConfig config : List.of( + new IndexTemplateConfig( + CONNECTOR_TEMPLATE_NAME + MAPPINGS_SUFFIX, + ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + MAPPINGS_SUFFIX + JSON_EXTENSION, + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + CONNECTOR_TEMPLATE_NAME + SETTINGS_SUFFIX, + ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + SETTINGS_SUFFIX + JSON_EXTENSION, + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + MAPPINGS_SUFFIX, + ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + MAPPINGS_SUFFIX + JSON_EXTENSION, + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + SETTINGS_SUFFIX, + ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + SETTINGS_SUFFIX + JSON_EXTENSION, + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ) + )) { + + try (var parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) { + componentTemplates.put(config.getTemplateName(), ComponentTemplate.parse(parser)); + } catch (IOException e) { + throw new AssertionError(e); + } + } + COMPONENT_TEMPLATES = Map.copyOf(componentTemplates); + } @Override protected List getIngestPipelines() { @@ -65,6 +117,20 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { } static final Map COMPOSABLE_INDEX_TEMPLATES = parseComposableTemplates( + new IndexTemplateConfig( + CONNECTOR_TEMPLATE_NAME, + ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + ".json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + Map.of("connectors.index_pattern", CONNECTOR_INDEX_NAME_PATTERN) + ), + new IndexTemplateConfig( + CONNECTOR_SYNC_JOBS_TEMPLATE_NAME, + ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + ".json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + Map.of("connectors-sync-jobs.index_pattern", CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN) + ), new IndexTemplateConfig( ACCESS_CONTROL_TEMPLATE_NAME, ROOT_TEMPLATE_RESOURCE_PATH + ACCESS_CONTROL_TEMPLATE_NAME + ".json", @@ -88,6 +154,11 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { return ENT_SEARCH_ORIGIN; } + @Override + protected Map getComponentTemplateConfigs() { + return COMPONENT_TEMPLATES; + } + @Override protected Map getComposableTemplateConfigs() { return COMPOSABLE_INDEX_TEMPLATES; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java index 85de2f900ddf..f46d915a7123 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java @@ -11,12 +11,10 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; @@ -42,7 +40,6 @@ import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.index.reindex.DeleteByQueryRequest; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -52,10 +49,10 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.ConnectorFiltering; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; +import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import org.elasticsearch.xpack.application.connector.filtering.FilteringRules; import org.elasticsearch.xpack.application.connector.syncjob.action.PostConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.UpdateConnectorSyncJobIngestionStatsAction; -import org.elasticsearch.xpack.core.template.TemplateUtils; import java.io.IOException; import java.time.Instant; @@ -72,7 +69,6 @@ import java.util.stream.Stream; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorIndexService.CONNECTOR_INDEX_NAME; -import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTORS_ALLOWED_PRODUCT_ORIGINS; import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** @@ -85,17 +81,7 @@ public class ConnectorSyncJobIndexService { // The client to interact with the system index (internal user). private final Client clientWithOrigin; - // TODO use proper version IDs (see org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java) - // TODO if this version is updated, a test should be added to - // javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java - private static final int CONNECTOR_SYNC_JOB_INDEX_VERSION = 1; - public static final String CONNECTOR_SYNC_JOB_INDEX_NAME = ".elastic-connectors-sync-jobs"; - public static final String CONNECTOR_SYNC_JOB_INDEX_PREFIX = ".elastic-connectors-sync-jobs-v"; - public static final String CONNECTOR_SYNC_JOB_CONCRETE_INDEX_NAME = CONNECTOR_SYNC_JOB_INDEX_PREFIX + CONNECTOR_SYNC_JOB_INDEX_VERSION; - public static final String CONNECTOR_SYNC_JOB_INDEX_NAME_PATTERN = CONNECTOR_SYNC_JOB_INDEX_NAME + "*"; - - private static final String CONNECTOR_SYNC_JOB_MAPPING_VERSION_VARIABLE = "elastic-connectors-sync-jobs.version"; - private static final String CONNECTOR_SYNC_JOB_MAPPING_MANAGED_VERSION_VARIABLE = "elastic-connectors-sync-jobs.managed.index.version"; + public static final String CONNECTOR_SYNC_JOB_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN; /** * @param client A client for executing actions on the connectors sync jobs index. @@ -104,35 +90,6 @@ public class ConnectorSyncJobIndexService { this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } - /** - * Returns the {@link SystemIndexDescriptor} for the Connector system index. - * - * @return The {@link SystemIndexDescriptor} for the Connector system index. - */ - public static SystemIndexDescriptor getSystemIndexDescriptor() { - PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - String templateSource = TemplateUtils.loadTemplate( - "/elastic-connectors-sync-jobs.json", - Version.CURRENT.toString(), - CONNECTOR_SYNC_JOB_MAPPING_VERSION_VARIABLE, - Map.of(CONNECTOR_SYNC_JOB_MAPPING_MANAGED_VERSION_VARIABLE, Integer.toString(CONNECTOR_SYNC_JOB_INDEX_VERSION)) - ); - request.source(templateSource, XContentType.JSON); - - return SystemIndexDescriptor.builder() - .setIndexPattern(CONNECTOR_SYNC_JOB_INDEX_NAME_PATTERN) - .setPrimaryIndex(CONNECTOR_SYNC_JOB_CONCRETE_INDEX_NAME) - .setAliasName(CONNECTOR_SYNC_JOB_INDEX_NAME) - .setDescription("Search connectors sync jobs") - .setMappings(request.mappings()) - .setSettings(request.settings()) - .setOrigin(CONNECTORS_ORIGIN) - .setType(SystemIndexDescriptor.Type.EXTERNAL_MANAGED) - .setAllowedElasticProductOrigins(CONNECTORS_ALLOWED_PRODUCT_ORIGINS) - .setNetNew() - .build(); - } - /** * @param request Request for creating a connector sync job. * @param listener Listener to respond to a successful response or an error. diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java index 53a8c7ac9694..7b6d9c9b14df 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java @@ -14,9 +14,7 @@ import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptPlugin; @@ -61,6 +59,7 @@ import static org.elasticsearch.xpack.application.connector.ConnectorTemplateReg import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomConnectorFeatures; import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomCronExpression; import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.randomConnectorFeatureEnabled; +import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.registerSimplifiedConnectorIndexTemplates; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.not; @@ -73,6 +72,7 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { @Before public void setup() { + registerSimplifiedConnectorIndexTemplates(indicesAdmin()); this.connectorIndexService = new ConnectorIndexService(client()); } @@ -80,7 +80,6 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { protected Collection> getPlugins() { List> plugins = new ArrayList<>(super.getPlugins()); plugins.add(MockPainlessScriptEngine.TestPlugin.class); - plugins.add(ConnectorIndexServiceTests.TestPlugin.class); return plugins; } @@ -1613,24 +1612,4 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { } } - /** - * Test plugin to register the {@link ConnectorIndexService} system index descriptor. - */ - public static class TestPlugin extends Plugin implements SystemIndexPlugin { - @Override - public Collection getSystemIndexDescriptors(Settings settings) { - return List.of(ConnectorIndexService.getSystemIndexDescriptor()); - } - - @Override - public String getFeatureName() { - return this.getClass().getSimpleName(); - } - - @Override - public String getFeatureDescription() { - return this.getClass().getCanonicalName(); - } - } - } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java index 89bdabe78300..068b99626af9 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java @@ -55,13 +55,15 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.ACCESS_CONTROL_INDEX_NAME_PATTERN; +import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; +import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.oneOf; import static org.hamcrest.Matchers.sameInstance; -import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -90,6 +92,14 @@ public class ConnectorTemplateRegistryTests extends ESTestCase { DiscoveryNode node = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); Map existingComponentTemplates = Map.of( + ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", + ConnectorTemplateRegistry.REGISTRY_VERSION, + ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings", + ConnectorTemplateRegistry.REGISTRY_VERSION, + ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-mappings", + ConnectorTemplateRegistry.REGISTRY_VERSION, + ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-settings", + ConnectorTemplateRegistry.REGISTRY_VERSION, ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION ); @@ -115,6 +125,131 @@ public class ConnectorTemplateRegistryTests extends ESTestCase { }); } + public void testThatNonExistingComponentTemplatesAreAddedImmediately() throws Exception { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.emptyMap(), + Collections.emptyMap(), + Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION), + Collections.emptyMap(), + nodes + ); + + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); + + calledTimes.set(0); + + // attempting to register the event multiple times as a race condition can yield this test flaky, namely: + // when calling registry.clusterChanged(newEvent) the templateCreationsInProgress state that the IndexTemplateRegistry maintains + // might've not yet been updated to reflect that the first template registration was complete, so a second template registration + // will not be issued anymore, leaving calledTimes to 0 + assertBusy(() -> { + // now delete all templates from the cluster state and let's retry + ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), nodes); + registry.clusterChanged(newEvent); + assertThat(calledTimes.get(), greaterThan(4)); + }); + } + + public void testThatVersionedOldComponentTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.emptyMap(), + Collections.singletonMap( + ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings", + ConnectorTemplateRegistry.REGISTRY_VERSION - 1 + ), + Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION), + Collections.emptyMap(), + nodes + ); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); + } + + public void testThatUnversionedOldComponentTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.emptyMap(), + Collections.singletonMap(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", null), + Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION), + Collections.emptyMap(), + nodes + ); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); + } + + public void testSameOrHigherVersionComponentTemplateNotUpgraded() { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map versions = new HashMap<>(); + versions.put(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", ConnectorTemplateRegistry.REGISTRY_VERSION); + versions.put(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings", ConnectorTemplateRegistry.REGISTRY_VERSION); + versions.put(ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-mappings", ConnectorTemplateRegistry.REGISTRY_VERSION); + versions.put(ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-settings", ConnectorTemplateRegistry.REGISTRY_VERSION); + versions.put(ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION); + ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(Collections.emptyMap(), versions, nodes); + client.setVerifier((action, request, listener) -> { + if (action == PutPipelineTransportAction.TYPE) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } + if (action instanceof PutComponentTemplateAction) { + fail("template should not have been re-installed"); + return null; + } else if (action == ILMActions.PUT) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action == TransportPutComposableIndexTemplateAction.TYPE) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + }); + registry.clusterChanged(sameVersionEvent); + + versions.clear(); + versions.put( + ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", + ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000) + ); + versions.put( + ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings", + ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000) + ); + versions.put( + ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-mappings", + ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000) + ); + versions.put( + ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-settings", + ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000) + ); + versions.put( + ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME, + ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000) + ); + ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(Collections.emptyMap(), versions, nodes); + registry.clusterChanged(higherVersionEvent); + } + public void testThatMissingMasterNodeDoesNothing() { DiscoveryNode localNode = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); @@ -125,7 +260,7 @@ public class ConnectorTemplateRegistryTests extends ESTestCase { }); ClusterChangedEvent event = createClusterChangedEvent( - Collections.singletonMap(ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME, null), + Collections.singletonMap(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME, null), Collections.emptyMap(), nodes ); @@ -222,7 +357,10 @@ public class ConnectorTemplateRegistryTests extends ESTestCase { assertThat(putRequest.indexTemplate().version(), equalTo((long) ConnectorTemplateRegistry.REGISTRY_VERSION)); final List indexPatterns = putRequest.indexTemplate().indexPatterns(); assertThat(indexPatterns, hasSize(1)); - assertThat(indexPatterns, contains(ACCESS_CONTROL_INDEX_NAME_PATTERN)); + assertThat( + indexPatterns, + contains(oneOf(ACCESS_CONTROL_INDEX_NAME_PATTERN, CONNECTOR_INDEX_NAME_PATTERN, CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN)) + ); assertNotNull(listener); return new TestPutIndexTemplateResponse(true); } else { diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java index 3f2f47e19088..c563bc0a14ee 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.application.connector; +import org.elasticsearch.client.internal.IndicesAdminClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.XContentType; @@ -26,6 +27,7 @@ import org.elasticsearch.xpack.application.connector.filtering.FilteringRules; import org.elasticsearch.xpack.application.connector.filtering.FilteringValidation; import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationInfo; import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationState; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobType; import org.elasticsearch.xpack.core.scheduler.Cron; @@ -45,14 +47,55 @@ import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomInt; import static org.elasticsearch.test.ESTestCase.randomList; +import static org.elasticsearch.test.ESTestCase.randomLong; import static org.elasticsearch.test.ESTestCase.randomLongBetween; -import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong; -import static org.elasticsearch.test.ESTestCase.randomShort; +import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; +import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN; +import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME; +import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME; public final class ConnectorTestUtils { public static final String NULL_STRING = null; + /** + * Registers index templates for instances of {@link Connector} and {@link ConnectorSyncJob} with essential field mappings. This method + * only includes mappings for fields relevant to test cases, specifying field types to ensure correct ES query logic behavior. + * + * @param indicesAdminClient The Elasticsearch indices admin client used for template registration. + */ + + public static void registerSimplifiedConnectorIndexTemplates(IndicesAdminClient indicesAdminClient) { + + indicesAdminClient.preparePutTemplate(CONNECTOR_TEMPLATE_NAME) + .setPatterns(List.of(CONNECTOR_INDEX_NAME_PATTERN)) + .setVersion(0) + .setMapping( + "service_type", + "type=keyword,store=true", + "status", + "type=keyword,store=true", + "index_name", + "type=keyword,store=true", + "configuration", + "type=object" + ) + .get(); + + indicesAdminClient.preparePutTemplate(CONNECTOR_SYNC_JOBS_TEMPLATE_NAME) + .setPatterns(List.of(CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN)) + .setVersion(0) + .setMapping( + "job_type", + "type=keyword,store=true", + "connector.id", + "type=keyword,store=true", + "status", + "type=keyword,store=true" + ) + .get(); + } + public static PutConnectorAction.Request getRandomPutConnectorActionRequest() { return new PutConnectorAction.Request( randomAlphaOfLengthBetween(5, 15), @@ -101,9 +144,9 @@ public final class ConnectorTestUtils { return new ConnectorSyncInfo.Builder().setLastAccessControlSyncError(randomFrom(new String[] { null, randomAlphaOfLength(10) })) .setLastAccessControlSyncScheduledAt(randomFrom(new Instant[] { null, ConnectorTestUtils.randomInstant() })) .setLastAccessControlSyncStatus(randomFrom(new ConnectorSyncStatus[] { null, getRandomSyncStatus() })) - .setLastDeletedDocumentCount(randomNonNegativeLong()) + .setLastDeletedDocumentCount(randomLong()) .setLastIncrementalSyncScheduledAt(randomFrom(new Instant[] { null, ConnectorTestUtils.randomInstant() })) - .setLastIndexedDocumentCount(randomNonNegativeLong()) + .setLastIndexedDocumentCount(randomLong()) .setLastSyncError(randomFrom(new String[] { null, randomAlphaOfLength(10) })) .setLastSyncScheduledAt(randomFrom(new Instant[] { null, ConnectorTestUtils.randomInstant() })) .setLastSyncStatus(randomFrom(new ConnectorSyncStatus[] { null, getRandomSyncStatus() })) @@ -154,7 +197,7 @@ public final class ConnectorTestUtils { public static ConnectorFiltering getRandomConnectorFiltering() { Instant currentTimestamp = Instant.now(); - int order = randomShort(); + int order = randomInt(); return new ConnectorFiltering.Builder().setActive( new FilteringRules.Builder().setAdvancedSnippet( diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java index fe6d97a871e0..f6c0a54f107b 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java @@ -20,11 +20,8 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -61,6 +58,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.ACCESS_CONTROL_INDEX_PREFIX; +import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.registerSimplifiedConnectorIndexTemplates; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -88,12 +86,14 @@ public class ConnectorSyncJobIndexServiceTests extends ESSingleNodeTestCase { List> plugins = new ArrayList<>(super.getPlugins()); // Reindex plugin is required for testDeleteAllSyncJobsByConnectorId (supports delete_by_query) plugins.add(ReindexPlugin.class); - plugins.add(TestPlugin.class); return plugins; } @Before public void setup() throws Exception { + + registerSimplifiedConnectorIndexTemplates(indicesAdmin()); + connectorOneId = createConnector(ConnectorTestUtils.getRandomConnector()); connectorTwoId = createConnector(ConnectorTestUtils.getRandomConnector()); connectorThreeId = createConnector(ConnectorTestUtils.getRandomConnectorWithDetachedIndex()); @@ -805,18 +805,18 @@ public class ConnectorSyncJobIndexServiceTests extends ESSingleNodeTestCase { Instant requestLastSeen = request.getLastSeen(); Map metadata = request.getMetadata(); - Long deletedDocumentCountAfterUpdate = ((Number) syncJobSourceAfterUpdate.get( + Long deletedDocumentCountAfterUpdate = (Long) syncJobSourceAfterUpdate.get( ConnectorSyncJob.DELETED_DOCUMENT_COUNT_FIELD.getPreferredName() - )).longValue(); - Long indexedDocumentCountAfterUpdate = ((Number) syncJobSourceAfterUpdate.get( + ); + Long indexedDocumentCountAfterUpdate = (Long) syncJobSourceAfterUpdate.get( ConnectorSyncJob.INDEXED_DOCUMENT_COUNT_FIELD.getPreferredName() - )).longValue(); - Long indexedDocumentVolumeAfterUpdate = ((Number) syncJobSourceAfterUpdate.get( + ); + Long indexedDocumentVolumeAfterUpdate = (Long) syncJobSourceAfterUpdate.get( ConnectorSyncJob.INDEXED_DOCUMENT_VOLUME_FIELD.getPreferredName() - )).longValue(); - Long totalDocumentCountAfterUpdate = ((Number) syncJobSourceAfterUpdate.get( + ); + Long totalDocumentCountAfterUpdate = (Long) syncJobSourceAfterUpdate.get( ConnectorSyncJob.TOTAL_DOCUMENT_COUNT_FIELD.getPreferredName() - )).longValue(); + ); Instant lastSeenAfterUpdate = Instant.parse( (String) syncJobSourceAfterUpdate.get(ConnectorSyncJob.LAST_SEEN_FIELD.getPreferredName()) ); @@ -1411,24 +1411,4 @@ public class ConnectorSyncJobIndexServiceTests extends ESSingleNodeTestCase { // wait 10 seconds for connector creation return index.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).getId(); } - - /** - * Test plugin to register the {@link ConnectorSyncJobIndexService} system index descriptor. - */ - public static class TestPlugin extends Plugin implements SystemIndexPlugin { - @Override - public Collection getSystemIndexDescriptors(Settings settings) { - return List.of(ConnectorSyncJobIndexService.getSystemIndexDescriptor()); - } - - @Override - public String getFeatureName() { - return this.getClass().getSimpleName(); - } - - @Override - public String getFeatureDescription() { - return this.getClass().getCanonicalName(); - } - } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java index 1e6426051e04..e72bf04fb7e5 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java @@ -36,7 +36,7 @@ import static org.elasticsearch.test.ESTestCase.randomInstantBetween; import static org.elasticsearch.test.ESTestCase.randomInt; import static org.elasticsearch.test.ESTestCase.randomLong; import static org.elasticsearch.test.ESTestCase.randomMap; -import static org.elasticsearch.test.ESTestCase.randomNonNegativeInt; +import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong; public class ConnectorSyncJobTestUtils { @@ -51,11 +51,11 @@ public class ConnectorSyncJobTestUtils { .setCompletedAt(randomFrom(new Instant[] { null, randomInstantBetween(lowerBoundInstant, upperBoundInstant) })) .setConnector(ConnectorTestUtils.getRandomSyncJobConnectorInfo()) .setCreatedAt(randomInstantBetween(lowerBoundInstant, upperBoundInstant)) - .setDeletedDocumentCount(randomNonNegativeInt()) + .setDeletedDocumentCount(randomLong()) .setError(randomFrom(new String[] { null, randomAlphaOfLength(10) })) .setId(randomAlphaOfLength(10)) - .setIndexedDocumentCount(randomNonNegativeInt()) - .setIndexedDocumentVolume(randomNonNegativeInt()) + .setIndexedDocumentCount(randomLong()) + .setIndexedDocumentVolume(randomLong()) .setJobType(getRandomConnectorJobType()) .setLastSeen(randomFrom(new Instant[] { null, randomInstantBetween(lowerBoundInstant, upperBoundInstant) })) .setMetadata( @@ -67,7 +67,7 @@ public class ConnectorSyncJobTestUtils { ) .setStartedAt(randomFrom(new Instant[] { null, randomInstantBetween(lowerBoundInstant, upperBoundInstant) })) .setStatus(ConnectorTestUtils.getRandomSyncStatus()) - .setTotalDocumentCount(randomNonNegativeInt()) + .setTotalDocumentCount(randomLong()) .setTriggerMethod(getRandomConnectorSyncJobTriggerMethod()) .setWorkerHostname(randomAlphaOfLength(10)) .build(); @@ -156,10 +156,10 @@ public class ConnectorSyncJobTestUtils { return new UpdateConnectorSyncJobIngestionStatsAction.Request( randomAlphaOfLength(10), - (long) randomNonNegativeInt(), - (long) randomNonNegativeInt(), - (long) randomNonNegativeInt(), - (long) randomNonNegativeInt(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), randomInstantBetween(lowerBoundInstant, upperBoundInstant), randomMap(2, 3, () -> new Tuple<>(randomAlphaOfLength(4), randomAlphaOfLength(4))) ); @@ -173,10 +173,10 @@ public class ConnectorSyncJobTestUtils { return new UpdateConnectorSyncJobIngestionStatsAction.Request( syncJobId, - (long) randomNonNegativeInt(), - (long) randomNonNegativeInt(), - (long) randomNonNegativeInt(), - (long) randomNonNegativeInt(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), randomInstantBetween(lowerBoundInstant, upperBoundInstant), randomMap(2, 3, () -> new Tuple<>(randomAlphaOfLength(4), randomAlphaOfLength(4))) );