From 385e1fdf21fb30c20adf94c29f35703b344b97f5 Mon Sep 17 00:00:00 2001 From: Navarone Feekery <13634519+navarone-feekery@users.noreply.github.com> Date: Fri, 24 Jan 2025 13:10:38 +0100 Subject: [PATCH] [Search] Add system index descriptors to Connector indices (#118991) Update the .elastic-connectors and .elastic-connectors-sync-jobs indices into system indices --- .../xpack/core/ClientHelper.java | 3 + ...json => elastic-connectors-sync-jobs.json} | 30 ++-- ...-mappings.json => elastic-connectors.json} | 51 ++++--- .../elastic-connectors-settings.json | 14 -- .../elastic-connectors-sync-jobs.json | 14 -- .../connector/elastic-connectors.json | 14 -- .../xpack/application/EnterpriseSearch.java | 9 +- .../connector/ConnectorIndexService.java | 50 +++++- .../connector/ConnectorTemplateRegistry.java | 81 +--------- .../syncjob/ConnectorSyncJobIndexService.java | 47 +++++- .../connector/ConnectorIndexServiceTests.java | 25 ++- .../ConnectorTemplateRegistryTests.java | 144 +----------------- .../connector/ConnectorTestUtils.java | 53 +------ .../ConnectorSyncJobIndexServiceTests.java | 44 ++++-- .../syncjob/ConnectorSyncJobTestUtils.java | 26 ++-- 15 files changed, 230 insertions(+), 375 deletions(-) rename x-pack/plugin/core/template-resources/src/main/resources/{entsearch/connector/elastic-connectors-sync-jobs-mappings.json => elastic-connectors-sync-jobs.json} (88%) rename x-pack/plugin/core/template-resources/src/main/resources/{entsearch/connector/elastic-connectors-mappings.json => elastic-connectors.json} (92%) delete mode 100644 x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-settings.json delete mode 100644 x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs.json delete mode 100644 x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors.json diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java index 9a0d1a58a30a..680b72cb970c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java @@ -196,6 +196,9 @@ public final class ClientHelper { public static final String APM_ORIGIN = "apm"; public static final String OTEL_ORIGIN = "otel"; public static final String REINDEX_DATA_STREAM_ORIGIN = "reindex_data_stream"; + // TODO consolidate the Kibana origin with the one defined in org/elasticsearch/kibana/KibanaPlugin.java + public static final String KIBANA_ORIGIN = "kibana"; + public static final String CLOUD_ORIGIN = "cloud"; private ClientHelper() {} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs-mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors-sync-jobs.json similarity index 88% rename from x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs-mappings.json rename to x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors-sync-jobs.json index 4dd6e0681c7c..7d1e7fa3a041 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs-mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors-sync-jobs.json @@ -1,12 +1,16 @@ { - "template": { - "aliases": { - ".elastic-connectors-sync-jobs": {} - }, - "mappings": { - "dynamic": "false", + "settings": { + "index": { + "number_of_shards": "1", + "auto_expand_replicas": "0-1" + } + }, + "mappings": { + "_doc": { + "dynamic": "strict", "_meta": { - "version": ${xpack.application.connector.template.version} + "version": "${elastic-connectors-sync-jobs.version}", + "managed_index_mappings_version": ${elastic-connectors-sync-jobs.managed.index.version} }, "properties": { "cancelation_requested_at": { @@ -21,9 +25,11 @@ "connector": { "properties": { "configuration": { + "dynamic": "false", "type": "object" }, "filtering": { + "dynamic": "false", "properties": { "advanced_snippet": { "properties": { @@ -91,6 +97,7 @@ "type": "keyword" }, "pipeline": { + "dynamic": "false", "properties": { "extract_binary_content": { "type": "boolean" @@ -110,6 +117,7 @@ "type": "keyword" }, "sync_cursor": { + "dynamic": "false", "type": "object" } } @@ -136,6 +144,7 @@ "type": "date" }, "metadata": { + "dynamic": "false", "type": "object" }, "started_at": { @@ -155,10 +164,5 @@ } } } - }, - "_meta": { - "description": "Built-in mappings applied by default to elastic-connectors indices", - "managed": true - }, - "version": ${xpack.application.connector.template.version} + } } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors.json similarity index 92% rename from x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json rename to x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors.json index 25409dbf8460..a98018e76f0e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors.json @@ -1,29 +1,35 @@ { - "template": { - "aliases": { - ".elastic-connectors": {} - }, - "mappings": { - "dynamic": "false", + "settings": { + "index": { + "number_of_shards": "1", + "auto_expand_replicas": "0-1" + } + }, + "mappings": { + "_doc": { + "dynamic": "strict", "_meta": { - "pipeline": { - "default_name": "search-default-ingestion", - "default_extract_binary_content": true, - "default_run_ml_inference": true, - "default_reduce_whitespace": true - }, - "version": ${xpack.application.connector.template.version} + "version": "${elastic-connectors.version}", + "managed_index_mappings_version": ${elastic-connectors.managed.index.version} }, "properties": { "api_key_id": { "type": "keyword" }, + "api_key_secret_id": { + "type": "keyword" + }, "configuration": { + "dynamic": "false", "type": "object" }, "custom_scheduling": { + "dynamic": "false", "type": "object" }, + "deleted": { + "type": "boolean" + }, "description": { "type": "text" }, @@ -31,6 +37,7 @@ "type": "keyword" }, "features": { + "dynamic": "false", "properties": { "filtering_advanced_config": { "type": "boolean" @@ -66,6 +73,7 @@ } }, "filtering": { + "dynamic": "false", "properties": { "active": { "properties": { @@ -78,6 +86,7 @@ "type": "date" }, "value": { + "dynamic": "false", "type": "object" } } @@ -143,6 +152,7 @@ "type": "date" }, "value": { + "dynamic": "false", "type": "object" } } @@ -242,6 +252,7 @@ "type": "keyword" }, "pipeline": { + "dynamic": "false", "properties": { "extract_binary_content": { "type": "boolean" @@ -258,6 +269,7 @@ } }, "scheduling": { + "dynamic": "false", "properties": { "access_control": { "properties": { @@ -298,22 +310,13 @@ "type": "keyword" }, "sync_cursor": { + "dynamic": "false", "type": "object" }, "sync_now": { "type": "boolean" - }, - "deleted": { - "type": "boolean" } } } - }, - "_meta": { - "description": "Built-in mappings applied by default to elastic-connectors indices", - "managed": true - }, - "version": ${xpack.application.connector.template.version} + } } - - diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-settings.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-settings.json deleted file mode 100644 index 6ff951057428..000000000000 --- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-settings.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "template": { - "settings": { - "hidden": true, - "number_of_shards": "1", - "auto_expand_replicas": "0-1" - } - }, - "_meta": { - "description": "Built-in settings applied by default to connector management indices", - "managed": true - }, - "version": ${xpack.application.connector.template.version} -} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs.json deleted file mode 100644 index db5404a30c6e..000000000000 --- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "index_patterns": ["${connectors-sync-jobs.index_pattern}"], - "priority": 100, - "composed_of": [ - "elastic-connectors-settings", - "elastic-connectors-sync-jobs-mappings" - ], - "allow_auto_create": true, - "_meta": { - "description": "Built-in template for elastic-connectors-sync-jobs", - "managed": true - }, - "version": ${xpack.application.connector.template.version} -} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors.json deleted file mode 100644 index 17c0b1eef061..000000000000 --- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "index_patterns": ["${connectors.index_pattern}"], - "priority": 100, - "composed_of": [ - "elastic-connectors-settings", - "elastic-connectors-mappings" - ], - "allow_auto_create": true, - "_meta": { - "description": "Built-in template for elastic-connectors", - "managed": true - }, - "version": ${xpack.application.connector.template.version} -} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index df1c76ccf770..4142d907d0c5 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -46,6 +46,7 @@ import org.elasticsearch.xpack.application.analytics.action.TransportPostAnalyti import org.elasticsearch.xpack.application.analytics.action.TransportPutAnalyticsCollectionAction; import org.elasticsearch.xpack.application.analytics.ingest.AnalyticsEventIngestConfig; import org.elasticsearch.xpack.application.connector.ConnectorAPIFeature; +import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import org.elasticsearch.xpack.application.connector.action.DeleteConnectorAction; import org.elasticsearch.xpack.application.connector.action.GetConnectorAction; @@ -124,6 +125,7 @@ import org.elasticsearch.xpack.application.connector.secrets.action.TransportDel import org.elasticsearch.xpack.application.connector.secrets.action.TransportGetConnectorSecretAction; import org.elasticsearch.xpack.application.connector.secrets.action.TransportPostConnectorSecretAction; import org.elasticsearch.xpack.application.connector.secrets.action.TransportPutConnectorSecretAction; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; import org.elasticsearch.xpack.application.connector.syncjob.action.CancelConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.CheckInConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.ClaimConnectorSyncJobAction; @@ -477,7 +479,12 @@ public class EnterpriseSearch extends Plugin implements ActionPlugin, SystemInde @Override public Collection getSystemIndexDescriptors(Settings settings) { Collection systemIndices = new ArrayList<>( - List.of(SearchApplicationIndexService.getSystemIndexDescriptor(), QueryRulesIndexService.getSystemIndexDescriptor()) + List.of( + SearchApplicationIndexService.getSystemIndexDescriptor(), + QueryRulesIndexService.getSystemIndexDescriptor(), + ConnectorSyncJobIndexService.getSystemIndexDescriptor(), + ConnectorIndexService.getSystemIndexDescriptor() + ) ); if (ConnectorSecretsFeature.isEnabled()) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 3120124c1752..a9ca8552feee 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -10,10 +10,12 @@ package org.elasticsearch.xpack.application.connector; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; @@ -33,6 +35,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -59,6 +62,7 @@ import org.elasticsearch.xpack.application.connector.filtering.FilteringValidati import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationState; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; +import org.elasticsearch.xpack.core.template.TemplateUtils; import java.time.Instant; import java.util.ArrayList; @@ -76,6 +80,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder; +import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTORS_ALLOWED_PRODUCT_ORIGINS; import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX; import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; @@ -87,7 +92,20 @@ public class ConnectorIndexService { // The client to interact with the system index (internal user). private final Client clientWithOrigin; - public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; + // TODO use proper version IDs (see org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java) + // TODO if this version is updated, a test should be added to + // javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java + private static final int CONNECTORS_INDEX_VERSION = 1; + // TODO rename to CONNECTOR_ALIAS_NAME + public static final String CONNECTOR_INDEX_NAME = ".elastic-connectors"; + public static final String CONNECTOR_INDEX_PREFIX = ".elastic-connectors-v"; + public static final String CONNECTOR_CONCRETE_INDEX_NAME = CONNECTOR_INDEX_PREFIX + CONNECTORS_INDEX_VERSION; + // The index pattern needs a stricter regex to prevent conflicts with .elastic-connectors-sync-jobs + + public static final String CONNECTOR_INDEX_NAME_PATTERN = CONNECTOR_INDEX_PREFIX + "*"; + + private static final String CONNECTORS_MAPPING_VERSION_VARIABLE = "elastic-connectors.version"; + private static final String CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE = "elastic-connectors.managed.index.version"; /** * @param client A client for executing actions on the connector index @@ -96,6 +114,36 @@ public class ConnectorIndexService { this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } + /** + * Returns the {@link SystemIndexDescriptor} for the Connector system index. + * + * @return The {@link SystemIndexDescriptor} for the Connector system index. + */ + public static SystemIndexDescriptor getSystemIndexDescriptor() { + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + String templateSource = TemplateUtils.loadTemplate( + "/elastic-connectors.json", + Version.CURRENT.toString(), + CONNECTORS_MAPPING_VERSION_VARIABLE, + Map.of(CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE, Integer.toString(CONNECTORS_INDEX_VERSION)) + ); + request.source(templateSource, XContentType.JSON); + + // The index pattern needs a stricter regex to prevent conflicts with .elastic-connectors-sync-jobs + return SystemIndexDescriptor.builder() + .setIndexPattern(CONNECTOR_INDEX_NAME_PATTERN) + .setPrimaryIndex(CONNECTOR_CONCRETE_INDEX_NAME) + .setAliasName(CONNECTOR_INDEX_NAME) + .setDescription("Search connectors") + .setMappings(request.mappings()) + .setSettings(request.settings()) + .setOrigin(CONNECTORS_ORIGIN) + .setType(SystemIndexDescriptor.Type.EXTERNAL_MANAGED) + .setAllowedElasticProductOrigins(CONNECTORS_ALLOWED_PRODUCT_ORIGINS) + .setNetNew() + .build(); + } + /** * Creates or updates the {@link Connector} in the underlying index with a specific doc ID * if connectorId is provided. Otherwise, the connector doc is indexed with auto-generated doc ID. diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java index fd35acc89db5..97ac05c443ad 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java @@ -8,25 +8,23 @@ package org.elasticsearch.xpack.application.connector; import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; import org.elasticsearch.xpack.core.template.IngestPipelineConfig; import org.elasticsearch.xpack.core.template.JsonIngestPipelineConfig; -import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.core.ClientHelper.CLOUD_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.KIBANA_ORIGIN; public class ConnectorTemplateRegistry extends IndexTemplateRegistry { @@ -34,13 +32,6 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { static final int REGISTRY_VERSION = 3; // Connector indices constants - - public static final String CONNECTOR_INDEX_NAME_PATTERN = ".elastic-connectors-v1"; - public static final String CONNECTOR_TEMPLATE_NAME = "elastic-connectors"; - - public static final String CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN = ".elastic-connectors-sync-jobs-v1"; - public static final String CONNECTOR_SYNC_JOBS_TEMPLATE_NAME = "elastic-connectors-sync-jobs"; - public static final String ACCESS_CONTROL_INDEX_PREFIX = ".search-acl-filter-"; public static final String ACCESS_CONTROL_INDEX_NAME_PATTERN = ".search-acl-filter-*"; public static final String ACCESS_CONTROL_TEMPLATE_NAME = "search-acl-filter"; @@ -58,51 +49,8 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { // Variable used to replace template version in index templates public static final String TEMPLATE_VERSION_VARIABLE = "xpack.application.connector.template.version"; - private static final String MAPPINGS_SUFFIX = "-mappings"; - - private static final String SETTINGS_SUFFIX = "-settings"; - - private static final String JSON_EXTENSION = ".json"; - - static final Map COMPONENT_TEMPLATES; - - static { - final Map componentTemplates = new HashMap<>(); - for (IndexTemplateConfig config : List.of( - new IndexTemplateConfig( - CONNECTOR_TEMPLATE_NAME + MAPPINGS_SUFFIX, - ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + MAPPINGS_SUFFIX + JSON_EXTENSION, - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE - ), - new IndexTemplateConfig( - CONNECTOR_TEMPLATE_NAME + SETTINGS_SUFFIX, - ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + SETTINGS_SUFFIX + JSON_EXTENSION, - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE - ), - new IndexTemplateConfig( - CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + MAPPINGS_SUFFIX, - ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + MAPPINGS_SUFFIX + JSON_EXTENSION, - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE - ), - new IndexTemplateConfig( - CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + SETTINGS_SUFFIX, - ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + SETTINGS_SUFFIX + JSON_EXTENSION, - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE - ) - )) { - - try (var parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) { - componentTemplates.put(config.getTemplateName(), ComponentTemplate.parse(parser)); - } catch (IOException e) { - throw new AssertionError(e); - } - } - COMPONENT_TEMPLATES = Map.copyOf(componentTemplates); - } + // Sources allowed to access system indices using X-elastic-product-origin header + public static final List CONNECTORS_ALLOWED_PRODUCT_ORIGINS = List.of(KIBANA_ORIGIN, CONNECTORS_ORIGIN, CLOUD_ORIGIN); @Override protected List getIngestPipelines() { @@ -117,20 +65,6 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { } static final Map COMPOSABLE_INDEX_TEMPLATES = parseComposableTemplates( - new IndexTemplateConfig( - CONNECTOR_TEMPLATE_NAME, - ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + ".json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - Map.of("connectors.index_pattern", CONNECTOR_INDEX_NAME_PATTERN) - ), - new IndexTemplateConfig( - CONNECTOR_SYNC_JOBS_TEMPLATE_NAME, - ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + ".json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - Map.of("connectors-sync-jobs.index_pattern", CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN) - ), new IndexTemplateConfig( ACCESS_CONTROL_TEMPLATE_NAME, ROOT_TEMPLATE_RESOURCE_PATH + ACCESS_CONTROL_TEMPLATE_NAME + ".json", @@ -154,11 +88,6 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { return ENT_SEARCH_ORIGIN; } - @Override - protected Map getComponentTemplateConfigs() { - return COMPONENT_TEMPLATES; - } - @Override protected Map getComposableTemplateConfigs() { return COMPOSABLE_INDEX_TEMPLATES; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java index f46d915a7123..85de2f900ddf 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java @@ -11,10 +11,12 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; @@ -40,6 +42,7 @@ import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -49,10 +52,10 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.ConnectorFiltering; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; -import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import org.elasticsearch.xpack.application.connector.filtering.FilteringRules; import org.elasticsearch.xpack.application.connector.syncjob.action.PostConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.UpdateConnectorSyncJobIngestionStatsAction; +import org.elasticsearch.xpack.core.template.TemplateUtils; import java.io.IOException; import java.time.Instant; @@ -69,6 +72,7 @@ import java.util.stream.Stream; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorIndexService.CONNECTOR_INDEX_NAME; +import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTORS_ALLOWED_PRODUCT_ORIGINS; import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** @@ -81,7 +85,17 @@ public class ConnectorSyncJobIndexService { // The client to interact with the system index (internal user). private final Client clientWithOrigin; - public static final String CONNECTOR_SYNC_JOB_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN; + // TODO use proper version IDs (see org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java) + // TODO if this version is updated, a test should be added to + // javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java + private static final int CONNECTOR_SYNC_JOB_INDEX_VERSION = 1; + public static final String CONNECTOR_SYNC_JOB_INDEX_NAME = ".elastic-connectors-sync-jobs"; + public static final String CONNECTOR_SYNC_JOB_INDEX_PREFIX = ".elastic-connectors-sync-jobs-v"; + public static final String CONNECTOR_SYNC_JOB_CONCRETE_INDEX_NAME = CONNECTOR_SYNC_JOB_INDEX_PREFIX + CONNECTOR_SYNC_JOB_INDEX_VERSION; + public static final String CONNECTOR_SYNC_JOB_INDEX_NAME_PATTERN = CONNECTOR_SYNC_JOB_INDEX_NAME + "*"; + + private static final String CONNECTOR_SYNC_JOB_MAPPING_VERSION_VARIABLE = "elastic-connectors-sync-jobs.version"; + private static final String CONNECTOR_SYNC_JOB_MAPPING_MANAGED_VERSION_VARIABLE = "elastic-connectors-sync-jobs.managed.index.version"; /** * @param client A client for executing actions on the connectors sync jobs index. @@ -90,6 +104,35 @@ public class ConnectorSyncJobIndexService { this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } + /** + * Returns the {@link SystemIndexDescriptor} for the Connector system index. + * + * @return The {@link SystemIndexDescriptor} for the Connector system index. + */ + public static SystemIndexDescriptor getSystemIndexDescriptor() { + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + String templateSource = TemplateUtils.loadTemplate( + "/elastic-connectors-sync-jobs.json", + Version.CURRENT.toString(), + CONNECTOR_SYNC_JOB_MAPPING_VERSION_VARIABLE, + Map.of(CONNECTOR_SYNC_JOB_MAPPING_MANAGED_VERSION_VARIABLE, Integer.toString(CONNECTOR_SYNC_JOB_INDEX_VERSION)) + ); + request.source(templateSource, XContentType.JSON); + + return SystemIndexDescriptor.builder() + .setIndexPattern(CONNECTOR_SYNC_JOB_INDEX_NAME_PATTERN) + .setPrimaryIndex(CONNECTOR_SYNC_JOB_CONCRETE_INDEX_NAME) + .setAliasName(CONNECTOR_SYNC_JOB_INDEX_NAME) + .setDescription("Search connectors sync jobs") + .setMappings(request.mappings()) + .setSettings(request.settings()) + .setOrigin(CONNECTORS_ORIGIN) + .setType(SystemIndexDescriptor.Type.EXTERNAL_MANAGED) + .setAllowedElasticProductOrigins(CONNECTORS_ALLOWED_PRODUCT_ORIGINS) + .setNetNew() + .build(); + } + /** * @param request Request for creating a connector sync job. * @param listener Listener to respond to a successful response or an error. diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java index 7b6d9c9b14df..53a8c7ac9694 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java @@ -14,7 +14,9 @@ import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptPlugin; @@ -59,7 +61,6 @@ import static org.elasticsearch.xpack.application.connector.ConnectorTemplateReg import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomConnectorFeatures; import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomCronExpression; import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.randomConnectorFeatureEnabled; -import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.registerSimplifiedConnectorIndexTemplates; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.not; @@ -72,7 +73,6 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { @Before public void setup() { - registerSimplifiedConnectorIndexTemplates(indicesAdmin()); this.connectorIndexService = new ConnectorIndexService(client()); } @@ -80,6 +80,7 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { protected Collection> getPlugins() { List> plugins = new ArrayList<>(super.getPlugins()); plugins.add(MockPainlessScriptEngine.TestPlugin.class); + plugins.add(ConnectorIndexServiceTests.TestPlugin.class); return plugins; } @@ -1612,4 +1613,24 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { } } + /** + * Test plugin to register the {@link ConnectorIndexService} system index descriptor. + */ + public static class TestPlugin extends Plugin implements SystemIndexPlugin { + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return List.of(ConnectorIndexService.getSystemIndexDescriptor()); + } + + @Override + public String getFeatureName() { + return this.getClass().getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return this.getClass().getCanonicalName(); + } + } + } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java index 068b99626af9..89bdabe78300 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java @@ -55,15 +55,13 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.ACCESS_CONTROL_INDEX_NAME_PATTERN; -import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; -import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.oneOf; import static org.hamcrest.Matchers.sameInstance; +import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -92,14 +90,6 @@ public class ConnectorTemplateRegistryTests extends ESTestCase { DiscoveryNode node = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); Map existingComponentTemplates = Map.of( - ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", - ConnectorTemplateRegistry.REGISTRY_VERSION, - ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings", - ConnectorTemplateRegistry.REGISTRY_VERSION, - ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-mappings", - ConnectorTemplateRegistry.REGISTRY_VERSION, - ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-settings", - ConnectorTemplateRegistry.REGISTRY_VERSION, ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION ); @@ -125,131 +115,6 @@ public class ConnectorTemplateRegistryTests extends ESTestCase { }); } - public void testThatNonExistingComponentTemplatesAreAddedImmediately() throws Exception { - DiscoveryNode node = DiscoveryNodeUtils.create("node"); - DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - - ClusterChangedEvent event = createClusterChangedEvent( - Collections.emptyMap(), - Collections.emptyMap(), - Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION), - Collections.emptyMap(), - nodes - ); - - AtomicInteger calledTimes = new AtomicInteger(0); - client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); - registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); - - calledTimes.set(0); - - // attempting to register the event multiple times as a race condition can yield this test flaky, namely: - // when calling registry.clusterChanged(newEvent) the templateCreationsInProgress state that the IndexTemplateRegistry maintains - // might've not yet been updated to reflect that the first template registration was complete, so a second template registration - // will not be issued anymore, leaving calledTimes to 0 - assertBusy(() -> { - // now delete all templates from the cluster state and let's retry - ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), nodes); - registry.clusterChanged(newEvent); - assertThat(calledTimes.get(), greaterThan(4)); - }); - } - - public void testThatVersionedOldComponentTemplatesAreUpgraded() throws Exception { - DiscoveryNode node = DiscoveryNodeUtils.create("node"); - DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - - ClusterChangedEvent event = createClusterChangedEvent( - Collections.emptyMap(), - Collections.singletonMap( - ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings", - ConnectorTemplateRegistry.REGISTRY_VERSION - 1 - ), - Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION), - Collections.emptyMap(), - nodes - ); - AtomicInteger calledTimes = new AtomicInteger(0); - client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); - registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); - } - - public void testThatUnversionedOldComponentTemplatesAreUpgraded() throws Exception { - DiscoveryNode node = DiscoveryNodeUtils.create("node"); - DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - - ClusterChangedEvent event = createClusterChangedEvent( - Collections.emptyMap(), - Collections.singletonMap(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", null), - Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION), - Collections.emptyMap(), - nodes - ); - AtomicInteger calledTimes = new AtomicInteger(0); - client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener)); - registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size()))); - } - - public void testSameOrHigherVersionComponentTemplateNotUpgraded() { - DiscoveryNode node = DiscoveryNodeUtils.create("node"); - DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - - Map versions = new HashMap<>(); - versions.put(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", ConnectorTemplateRegistry.REGISTRY_VERSION); - versions.put(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings", ConnectorTemplateRegistry.REGISTRY_VERSION); - versions.put(ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-mappings", ConnectorTemplateRegistry.REGISTRY_VERSION); - versions.put(ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-settings", ConnectorTemplateRegistry.REGISTRY_VERSION); - versions.put(ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION); - ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(Collections.emptyMap(), versions, nodes); - client.setVerifier((action, request, listener) -> { - if (action == PutPipelineTransportAction.TYPE) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; - } - if (action instanceof PutComponentTemplateAction) { - fail("template should not have been re-installed"); - return null; - } else if (action == ILMActions.PUT) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; - } else if (action == TransportPutComposableIndexTemplateAction.TYPE) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; - } else { - fail("client called with unexpected request:" + request.toString()); - return null; - } - }); - registry.clusterChanged(sameVersionEvent); - - versions.clear(); - versions.put( - ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", - ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000) - ); - versions.put( - ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings", - ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000) - ); - versions.put( - ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-mappings", - ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000) - ); - versions.put( - ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-settings", - ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000) - ); - versions.put( - ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME, - ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000) - ); - ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(Collections.emptyMap(), versions, nodes); - registry.clusterChanged(higherVersionEvent); - } - public void testThatMissingMasterNodeDoesNothing() { DiscoveryNode localNode = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); @@ -260,7 +125,7 @@ public class ConnectorTemplateRegistryTests extends ESTestCase { }); ClusterChangedEvent event = createClusterChangedEvent( - Collections.singletonMap(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME, null), + Collections.singletonMap(ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME, null), Collections.emptyMap(), nodes ); @@ -357,10 +222,7 @@ public class ConnectorTemplateRegistryTests extends ESTestCase { assertThat(putRequest.indexTemplate().version(), equalTo((long) ConnectorTemplateRegistry.REGISTRY_VERSION)); final List indexPatterns = putRequest.indexTemplate().indexPatterns(); assertThat(indexPatterns, hasSize(1)); - assertThat( - indexPatterns, - contains(oneOf(ACCESS_CONTROL_INDEX_NAME_PATTERN, CONNECTOR_INDEX_NAME_PATTERN, CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN)) - ); + assertThat(indexPatterns, contains(ACCESS_CONTROL_INDEX_NAME_PATTERN)); assertNotNull(listener); return new TestPutIndexTemplateResponse(true); } else { diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java index c563bc0a14ee..3f2f47e19088 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.application.connector; -import org.elasticsearch.client.internal.IndicesAdminClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.XContentType; @@ -27,7 +26,6 @@ import org.elasticsearch.xpack.application.connector.filtering.FilteringRules; import org.elasticsearch.xpack.application.connector.filtering.FilteringValidation; import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationInfo; import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationState; -import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobType; import org.elasticsearch.xpack.core.scheduler.Cron; @@ -47,55 +45,14 @@ import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomInt; import static org.elasticsearch.test.ESTestCase.randomList; -import static org.elasticsearch.test.ESTestCase.randomLong; import static org.elasticsearch.test.ESTestCase.randomLongBetween; -import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; -import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN; -import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME; -import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME; +import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong; +import static org.elasticsearch.test.ESTestCase.randomShort; public final class ConnectorTestUtils { public static final String NULL_STRING = null; - /** - * Registers index templates for instances of {@link Connector} and {@link ConnectorSyncJob} with essential field mappings. This method - * only includes mappings for fields relevant to test cases, specifying field types to ensure correct ES query logic behavior. - * - * @param indicesAdminClient The Elasticsearch indices admin client used for template registration. - */ - - public static void registerSimplifiedConnectorIndexTemplates(IndicesAdminClient indicesAdminClient) { - - indicesAdminClient.preparePutTemplate(CONNECTOR_TEMPLATE_NAME) - .setPatterns(List.of(CONNECTOR_INDEX_NAME_PATTERN)) - .setVersion(0) - .setMapping( - "service_type", - "type=keyword,store=true", - "status", - "type=keyword,store=true", - "index_name", - "type=keyword,store=true", - "configuration", - "type=object" - ) - .get(); - - indicesAdminClient.preparePutTemplate(CONNECTOR_SYNC_JOBS_TEMPLATE_NAME) - .setPatterns(List.of(CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN)) - .setVersion(0) - .setMapping( - "job_type", - "type=keyword,store=true", - "connector.id", - "type=keyword,store=true", - "status", - "type=keyword,store=true" - ) - .get(); - } - public static PutConnectorAction.Request getRandomPutConnectorActionRequest() { return new PutConnectorAction.Request( randomAlphaOfLengthBetween(5, 15), @@ -144,9 +101,9 @@ public final class ConnectorTestUtils { return new ConnectorSyncInfo.Builder().setLastAccessControlSyncError(randomFrom(new String[] { null, randomAlphaOfLength(10) })) .setLastAccessControlSyncScheduledAt(randomFrom(new Instant[] { null, ConnectorTestUtils.randomInstant() })) .setLastAccessControlSyncStatus(randomFrom(new ConnectorSyncStatus[] { null, getRandomSyncStatus() })) - .setLastDeletedDocumentCount(randomLong()) + .setLastDeletedDocumentCount(randomNonNegativeLong()) .setLastIncrementalSyncScheduledAt(randomFrom(new Instant[] { null, ConnectorTestUtils.randomInstant() })) - .setLastIndexedDocumentCount(randomLong()) + .setLastIndexedDocumentCount(randomNonNegativeLong()) .setLastSyncError(randomFrom(new String[] { null, randomAlphaOfLength(10) })) .setLastSyncScheduledAt(randomFrom(new Instant[] { null, ConnectorTestUtils.randomInstant() })) .setLastSyncStatus(randomFrom(new ConnectorSyncStatus[] { null, getRandomSyncStatus() })) @@ -197,7 +154,7 @@ public final class ConnectorTestUtils { public static ConnectorFiltering getRandomConnectorFiltering() { Instant currentTimestamp = Instant.now(); - int order = randomInt(); + int order = randomShort(); return new ConnectorFiltering.Builder().setActive( new FilteringRules.Builder().setAdvancedSnippet( diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java index f6c0a54f107b..fe6d97a871e0 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java @@ -20,8 +20,11 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -58,7 +61,6 @@ import java.util.stream.Collectors; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.ACCESS_CONTROL_INDEX_PREFIX; -import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.registerSimplifiedConnectorIndexTemplates; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -86,14 +88,12 @@ public class ConnectorSyncJobIndexServiceTests extends ESSingleNodeTestCase { List> plugins = new ArrayList<>(super.getPlugins()); // Reindex plugin is required for testDeleteAllSyncJobsByConnectorId (supports delete_by_query) plugins.add(ReindexPlugin.class); + plugins.add(TestPlugin.class); return plugins; } @Before public void setup() throws Exception { - - registerSimplifiedConnectorIndexTemplates(indicesAdmin()); - connectorOneId = createConnector(ConnectorTestUtils.getRandomConnector()); connectorTwoId = createConnector(ConnectorTestUtils.getRandomConnector()); connectorThreeId = createConnector(ConnectorTestUtils.getRandomConnectorWithDetachedIndex()); @@ -805,18 +805,18 @@ public class ConnectorSyncJobIndexServiceTests extends ESSingleNodeTestCase { Instant requestLastSeen = request.getLastSeen(); Map metadata = request.getMetadata(); - Long deletedDocumentCountAfterUpdate = (Long) syncJobSourceAfterUpdate.get( + Long deletedDocumentCountAfterUpdate = ((Number) syncJobSourceAfterUpdate.get( ConnectorSyncJob.DELETED_DOCUMENT_COUNT_FIELD.getPreferredName() - ); - Long indexedDocumentCountAfterUpdate = (Long) syncJobSourceAfterUpdate.get( + )).longValue(); + Long indexedDocumentCountAfterUpdate = ((Number) syncJobSourceAfterUpdate.get( ConnectorSyncJob.INDEXED_DOCUMENT_COUNT_FIELD.getPreferredName() - ); - Long indexedDocumentVolumeAfterUpdate = (Long) syncJobSourceAfterUpdate.get( + )).longValue(); + Long indexedDocumentVolumeAfterUpdate = ((Number) syncJobSourceAfterUpdate.get( ConnectorSyncJob.INDEXED_DOCUMENT_VOLUME_FIELD.getPreferredName() - ); - Long totalDocumentCountAfterUpdate = (Long) syncJobSourceAfterUpdate.get( + )).longValue(); + Long totalDocumentCountAfterUpdate = ((Number) syncJobSourceAfterUpdate.get( ConnectorSyncJob.TOTAL_DOCUMENT_COUNT_FIELD.getPreferredName() - ); + )).longValue(); Instant lastSeenAfterUpdate = Instant.parse( (String) syncJobSourceAfterUpdate.get(ConnectorSyncJob.LAST_SEEN_FIELD.getPreferredName()) ); @@ -1411,4 +1411,24 @@ public class ConnectorSyncJobIndexServiceTests extends ESSingleNodeTestCase { // wait 10 seconds for connector creation return index.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).getId(); } + + /** + * Test plugin to register the {@link ConnectorSyncJobIndexService} system index descriptor. + */ + public static class TestPlugin extends Plugin implements SystemIndexPlugin { + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return List.of(ConnectorSyncJobIndexService.getSystemIndexDescriptor()); + } + + @Override + public String getFeatureName() { + return this.getClass().getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return this.getClass().getCanonicalName(); + } + } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java index e72bf04fb7e5..1e6426051e04 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java @@ -36,7 +36,7 @@ import static org.elasticsearch.test.ESTestCase.randomInstantBetween; import static org.elasticsearch.test.ESTestCase.randomInt; import static org.elasticsearch.test.ESTestCase.randomLong; import static org.elasticsearch.test.ESTestCase.randomMap; -import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong; +import static org.elasticsearch.test.ESTestCase.randomNonNegativeInt; public class ConnectorSyncJobTestUtils { @@ -51,11 +51,11 @@ public class ConnectorSyncJobTestUtils { .setCompletedAt(randomFrom(new Instant[] { null, randomInstantBetween(lowerBoundInstant, upperBoundInstant) })) .setConnector(ConnectorTestUtils.getRandomSyncJobConnectorInfo()) .setCreatedAt(randomInstantBetween(lowerBoundInstant, upperBoundInstant)) - .setDeletedDocumentCount(randomLong()) + .setDeletedDocumentCount(randomNonNegativeInt()) .setError(randomFrom(new String[] { null, randomAlphaOfLength(10) })) .setId(randomAlphaOfLength(10)) - .setIndexedDocumentCount(randomLong()) - .setIndexedDocumentVolume(randomLong()) + .setIndexedDocumentCount(randomNonNegativeInt()) + .setIndexedDocumentVolume(randomNonNegativeInt()) .setJobType(getRandomConnectorJobType()) .setLastSeen(randomFrom(new Instant[] { null, randomInstantBetween(lowerBoundInstant, upperBoundInstant) })) .setMetadata( @@ -67,7 +67,7 @@ public class ConnectorSyncJobTestUtils { ) .setStartedAt(randomFrom(new Instant[] { null, randomInstantBetween(lowerBoundInstant, upperBoundInstant) })) .setStatus(ConnectorTestUtils.getRandomSyncStatus()) - .setTotalDocumentCount(randomLong()) + .setTotalDocumentCount(randomNonNegativeInt()) .setTriggerMethod(getRandomConnectorSyncJobTriggerMethod()) .setWorkerHostname(randomAlphaOfLength(10)) .build(); @@ -156,10 +156,10 @@ public class ConnectorSyncJobTestUtils { return new UpdateConnectorSyncJobIngestionStatsAction.Request( randomAlphaOfLength(10), - randomNonNegativeLong(), - randomNonNegativeLong(), - randomNonNegativeLong(), - randomNonNegativeLong(), + (long) randomNonNegativeInt(), + (long) randomNonNegativeInt(), + (long) randomNonNegativeInt(), + (long) randomNonNegativeInt(), randomInstantBetween(lowerBoundInstant, upperBoundInstant), randomMap(2, 3, () -> new Tuple<>(randomAlphaOfLength(4), randomAlphaOfLength(4))) ); @@ -173,10 +173,10 @@ public class ConnectorSyncJobTestUtils { return new UpdateConnectorSyncJobIngestionStatsAction.Request( syncJobId, - randomNonNegativeLong(), - randomNonNegativeLong(), - randomNonNegativeLong(), - randomNonNegativeLong(), + (long) randomNonNegativeInt(), + (long) randomNonNegativeInt(), + (long) randomNonNegativeInt(), + (long) randomNonNegativeInt(), randomInstantBetween(lowerBoundInstant, upperBoundInstant), randomMap(2, 3, () -> new Tuple<>(randomAlphaOfLength(4), randomAlphaOfLength(4))) );