Revert "[Search] Add system index descriptors to Connector indices (#118991)" (#120951)

This reverts commit 385e1fdf21.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java

@@ -196,9 +196,6 @@ public final class ClientHelper {
     public static final String APM_ORIGIN = "apm";
     public static final String OTEL_ORIGIN = "otel";
     public static final String REINDEX_DATA_STREAM_ORIGIN = "reindex_data_stream";
-    // TODO consolidate the Kibana origin with the one defined in org/elasticsearch/kibana/KibanaPlugin.java
-    public static final String KIBANA_ORIGIN = "kibana";
-    public static final String CLOUD_ORIGIN = "cloud";
 
     private ClientHelper() {}
 

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json

@@ -1,35 +1,29 @@
 {
-  "settings": {
+  "template": {
-    "index": {
+    "aliases": {
-      "number_of_shards": "1",
+      ".elastic-connectors":  {}
-      "auto_expand_replicas": "0-1"
-    }
     },
     "mappings": {
-    "_doc": {
+      "dynamic": "false",
-      "dynamic": "strict",
       "_meta": {
-        "version": "${elastic-connectors.version}",
+        "pipeline": {
-        "managed_index_mappings_version": ${elastic-connectors.managed.index.version}
+          "default_name": "search-default-ingestion",
+          "default_extract_binary_content": true,
+          "default_run_ml_inference": true,
+          "default_reduce_whitespace": true
+        },
+        "version": ${xpack.application.connector.template.version}
       },
       "properties": {
         "api_key_id": {
           "type": "keyword"
         },
-        "api_key_secret_id": {
-          "type": "keyword"
-        },
         "configuration": {
-          "dynamic": "false",
           "type": "object"
         },
         "custom_scheduling": {
-          "dynamic": "false",
           "type": "object"
         },
-        "deleted": {
-          "type": "boolean"
-        },
         "description": {
           "type": "text"
         },
@@ -37,7 +31,6 @@
           "type": "keyword"
         },
         "features": {
-          "dynamic": "false",
           "properties": {
             "filtering_advanced_config": {
               "type": "boolean"
@@ -73,7 +66,6 @@
           }
         },
         "filtering": {
-          "dynamic": "false",
           "properties": {
             "active": {
               "properties": {
@@ -86,7 +78,6 @@
                       "type": "date"
                     },
                     "value": {
-                      "dynamic": "false",
                       "type": "object"
                     }
                   }
@@ -152,7 +143,6 @@
                       "type": "date"
                     },
                     "value": {
-                      "dynamic": "false",
                       "type": "object"
                     }
                   }
@@ -252,7 +242,6 @@
           "type": "keyword"
         },
         "pipeline": {
-          "dynamic": "false",
           "properties": {
             "extract_binary_content": {
               "type": "boolean"
@@ -269,7 +258,6 @@
           }
         },
         "scheduling": {
-          "dynamic": "false",
           "properties": {
             "access_control": {
               "properties": {
@@ -310,13 +298,22 @@
           "type": "keyword"
         },
         "sync_cursor": {
-          "dynamic": "false",
           "type": "object"
         },
         "sync_now": {
           "type": "boolean"
+        },
+        "deleted": {
+          "type": "boolean"
         }
       }
     }
-  }
+  },
+  "_meta": {
+    "description": "Built-in mappings applied by default to elastic-connectors indices",
+    "managed": true
+  },
+  "version": ${xpack.application.connector.template.version}
 }
+
+

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-settings.json

@@ -0,0 +1,14 @@
+{
+  "template": {
+    "settings": {
+      "hidden": true,
+      "number_of_shards": "1",
+      "auto_expand_replicas": "0-1"
+    }
+  },
+  "_meta": {
+    "description": "Built-in settings applied by default to connector management indices",
+    "managed": true
+  },
+  "version": ${xpack.application.connector.template.version}
+}

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs-mappings.json

@@ -1,16 +1,12 @@
 {
-  "settings": {
+  "template": {
-    "index": {
+    "aliases": {
-      "number_of_shards": "1",
+      ".elastic-connectors-sync-jobs":  {}
-      "auto_expand_replicas": "0-1"
-    }
     },
     "mappings": {
-    "_doc": {
+      "dynamic": "false",
-      "dynamic": "strict",
       "_meta": {
-        "version": "${elastic-connectors-sync-jobs.version}",
+        "version": ${xpack.application.connector.template.version}
-        "managed_index_mappings_version": ${elastic-connectors-sync-jobs.managed.index.version}
       },
       "properties": {
         "cancelation_requested_at": {
@@ -25,11 +21,9 @@
         "connector": {
           "properties": {
             "configuration": {
-              "dynamic": "false",
               "type": "object"
             },
             "filtering": {
-              "dynamic": "false",
               "properties": {
                 "advanced_snippet": {
                   "properties": {
@@ -97,7 +91,6 @@
               "type": "keyword"
             },
             "pipeline": {
-              "dynamic": "false",
               "properties": {
                 "extract_binary_content": {
                   "type": "boolean"
@@ -117,7 +110,6 @@
               "type": "keyword"
             },
             "sync_cursor": {
-              "dynamic": "false",
               "type": "object"
             }
           }
@@ -144,7 +136,6 @@
           "type": "date"
         },
         "metadata": {
-          "dynamic": "false",
           "type": "object"
         },
         "started_at": {
@@ -164,5 +155,10 @@
         }
       }
     }
-  }
+  },
+  "_meta": {
+    "description": "Built-in mappings applied by default to elastic-connectors indices",
+    "managed": true
+  },
+  "version": ${xpack.application.connector.template.version}
 }

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs.json

@@ -0,0 +1,14 @@
+{
+  "index_patterns": ["${connectors-sync-jobs.index_pattern}"],
+  "priority": 100,
+  "composed_of": [
+    "elastic-connectors-settings",
+    "elastic-connectors-sync-jobs-mappings"
+  ],
+  "allow_auto_create": true,
+  "_meta": {
+    "description": "Built-in template for elastic-connectors-sync-jobs",
+    "managed": true
+  },
+  "version": ${xpack.application.connector.template.version}
+}

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors.json

@@ -0,0 +1,14 @@
+{
+  "index_patterns": ["${connectors.index_pattern}"],
+  "priority": 100,
+  "composed_of": [
+    "elastic-connectors-settings",
+    "elastic-connectors-mappings"
+  ],
+  "allow_auto_create": true,
+  "_meta": {
+    "description": "Built-in template for elastic-connectors",
+    "managed": true
+  },
+  "version": ${xpack.application.connector.template.version}
+}

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java

@@ -46,7 +46,6 @@ import org.elasticsearch.xpack.application.analytics.action.TransportPostAnalyti
 import org.elasticsearch.xpack.application.analytics.action.TransportPutAnalyticsCollectionAction;
 import org.elasticsearch.xpack.application.analytics.ingest.AnalyticsEventIngestConfig;
 import org.elasticsearch.xpack.application.connector.ConnectorAPIFeature;
-import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
 import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
 import org.elasticsearch.xpack.application.connector.action.DeleteConnectorAction;
 import org.elasticsearch.xpack.application.connector.action.GetConnectorAction;
@@ -125,7 +124,6 @@ import org.elasticsearch.xpack.application.connector.secrets.action.TransportDel
 import org.elasticsearch.xpack.application.connector.secrets.action.TransportGetConnectorSecretAction;
 import org.elasticsearch.xpack.application.connector.secrets.action.TransportPostConnectorSecretAction;
 import org.elasticsearch.xpack.application.connector.secrets.action.TransportPutConnectorSecretAction;
-import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService;
 import org.elasticsearch.xpack.application.connector.syncjob.action.CancelConnectorSyncJobAction;
 import org.elasticsearch.xpack.application.connector.syncjob.action.CheckInConnectorSyncJobAction;
 import org.elasticsearch.xpack.application.connector.syncjob.action.ClaimConnectorSyncJobAction;
@@ -479,12 +477,7 @@ public class EnterpriseSearch extends Plugin implements ActionPlugin, SystemInde
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
         Collection<SystemIndexDescriptor> systemIndices = new ArrayList<>(
-            List.of(
+            List.of(SearchApplicationIndexService.getSystemIndexDescriptor(), QueryRulesIndexService.getSystemIndexDescriptor())
-                SearchApplicationIndexService.getSystemIndexDescriptor(),
-                QueryRulesIndexService.getSystemIndexDescriptor(),
-                ConnectorSyncJobIndexService.getSystemIndexDescriptor(),
-                ConnectorIndexService.getSystemIndexDescriptor()
-            )
         );
 
         if (ConnectorSecretsFeature.isEnabled()) {

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java

@@ -10,12 +10,10 @@ package org.elasticsearch.xpack.application.connector;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DelegatingActionListener;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.DocWriteResponse;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -35,7 +33,6 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.query.WildcardQueryBuilder;
-import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -62,7 +59,6 @@ import org.elasticsearch.xpack.application.connector.filtering.FilteringValidati
 import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationState;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService;
-import org.elasticsearch.xpack.core.template.TemplateUtils;
 
 import java.time.Instant;
 import java.util.ArrayList;
@@ -80,7 +76,6 @@ import java.util.stream.Collectors;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering;
 import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder;
-import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTORS_ALLOWED_PRODUCT_ORIGINS;
 import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX;
 import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN;
 
@@ -92,20 +87,7 @@ public class ConnectorIndexService {
     // The client to interact with the system index (internal user).
     private final Client clientWithOrigin;
 
-    // TODO use proper version IDs (see org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java)
+    public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN;
-    // TODO if this version is updated, a test should be added to
-    // javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java
-    private static final int CONNECTORS_INDEX_VERSION = 1;
-    // TODO rename to CONNECTOR_ALIAS_NAME
-    public static final String CONNECTOR_INDEX_NAME = ".elastic-connectors";
-    public static final String CONNECTOR_INDEX_PREFIX = ".elastic-connectors-v";
-    public static final String CONNECTOR_CONCRETE_INDEX_NAME = CONNECTOR_INDEX_PREFIX + CONNECTORS_INDEX_VERSION;
-    // The index pattern needs a stricter regex to prevent conflicts with .elastic-connectors-sync-jobs
-
-    public static final String CONNECTOR_INDEX_NAME_PATTERN = CONNECTOR_INDEX_PREFIX + "*";
-
-    private static final String CONNECTORS_MAPPING_VERSION_VARIABLE = "elastic-connectors.version";
-    private static final String CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE = "elastic-connectors.managed.index.version";
 
     /**
      * @param client A client for executing actions on the connector index
@@ -114,36 +96,6 @@ public class ConnectorIndexService {
         this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN);
     }
 
-    /**
-     * Returns the {@link SystemIndexDescriptor} for the Connector system index.
-     *
-     * @return The {@link SystemIndexDescriptor} for the Connector system index.
-     */
-    public static SystemIndexDescriptor getSystemIndexDescriptor() {
-        PutIndexTemplateRequest request = new PutIndexTemplateRequest();
-        String templateSource = TemplateUtils.loadTemplate(
-            "/elastic-connectors.json",
-            Version.CURRENT.toString(),
-            CONNECTORS_MAPPING_VERSION_VARIABLE,
-            Map.of(CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE, Integer.toString(CONNECTORS_INDEX_VERSION))
-        );
-        request.source(templateSource, XContentType.JSON);
-
-        // The index pattern needs a stricter regex to prevent conflicts with .elastic-connectors-sync-jobs
-        return SystemIndexDescriptor.builder()
-            .setIndexPattern(CONNECTOR_INDEX_NAME_PATTERN)
-            .setPrimaryIndex(CONNECTOR_CONCRETE_INDEX_NAME)
-            .setAliasName(CONNECTOR_INDEX_NAME)
-            .setDescription("Search connectors")
-            .setMappings(request.mappings())
-            .setSettings(request.settings())
-            .setOrigin(CONNECTORS_ORIGIN)
-            .setType(SystemIndexDescriptor.Type.EXTERNAL_MANAGED)
-            .setAllowedElasticProductOrigins(CONNECTORS_ALLOWED_PRODUCT_ORIGINS)
-            .setNetNew()
-            .build();
-    }
-
     /**
      * Creates or updates the {@link Connector} in the underlying index with a specific doc ID
      * if connectorId is provided. Otherwise, the connector doc is indexed with auto-generated doc ID.

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java

@@ -8,23 +8,25 @@
 package org.elasticsearch.xpack.application.connector;
 
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
 import org.elasticsearch.xpack.core.template.IngestPipelineConfig;
 import org.elasticsearch.xpack.core.template.JsonIngestPipelineConfig;
 
+import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.core.ClientHelper.CLOUD_ORIGIN;
-import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN;
-import static org.elasticsearch.xpack.core.ClientHelper.KIBANA_ORIGIN;
 
 public class ConnectorTemplateRegistry extends IndexTemplateRegistry {
 
@@ -32,6 +34,13 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry {
     static final int REGISTRY_VERSION = 3;
 
     // Connector indices constants
+
+    public static final String CONNECTOR_INDEX_NAME_PATTERN = ".elastic-connectors-v1";
+    public static final String CONNECTOR_TEMPLATE_NAME = "elastic-connectors";
+
+    public static final String CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN = ".elastic-connectors-sync-jobs-v1";
+    public static final String CONNECTOR_SYNC_JOBS_TEMPLATE_NAME = "elastic-connectors-sync-jobs";
+
     public static final String ACCESS_CONTROL_INDEX_PREFIX = ".search-acl-filter-";
     public static final String ACCESS_CONTROL_INDEX_NAME_PATTERN = ".search-acl-filter-*";
     public static final String ACCESS_CONTROL_TEMPLATE_NAME = "search-acl-filter";
@@ -49,8 +58,51 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry {
     // Variable used to replace template version in index templates
     public static final String TEMPLATE_VERSION_VARIABLE = "xpack.application.connector.template.version";
 
-    // Sources allowed to access system indices using X-elastic-product-origin header
+    private static final String MAPPINGS_SUFFIX = "-mappings";
-    public static final List<String> CONNECTORS_ALLOWED_PRODUCT_ORIGINS = List.of(KIBANA_ORIGIN, CONNECTORS_ORIGIN, CLOUD_ORIGIN);
+
+    private static final String SETTINGS_SUFFIX = "-settings";
+
+    private static final String JSON_EXTENSION = ".json";
+
+    static final Map<String, ComponentTemplate> COMPONENT_TEMPLATES;
+
+    static {
+        final Map<String, ComponentTemplate> componentTemplates = new HashMap<>();
+        for (IndexTemplateConfig config : List.of(
+            new IndexTemplateConfig(
+                CONNECTOR_TEMPLATE_NAME + MAPPINGS_SUFFIX,
+                ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + MAPPINGS_SUFFIX + JSON_EXTENSION,
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE
+            ),
+            new IndexTemplateConfig(
+                CONNECTOR_TEMPLATE_NAME + SETTINGS_SUFFIX,
+                ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + SETTINGS_SUFFIX + JSON_EXTENSION,
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE
+            ),
+            new IndexTemplateConfig(
+                CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + MAPPINGS_SUFFIX,
+                ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + MAPPINGS_SUFFIX + JSON_EXTENSION,
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE
+            ),
+            new IndexTemplateConfig(
+                CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + SETTINGS_SUFFIX,
+                ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + SETTINGS_SUFFIX + JSON_EXTENSION,
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE
+            )
+        )) {
+
+            try (var parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) {
+                componentTemplates.put(config.getTemplateName(), ComponentTemplate.parse(parser));
+            } catch (IOException e) {
+                throw new AssertionError(e);
+            }
+        }
+        COMPONENT_TEMPLATES = Map.copyOf(componentTemplates);
+    }
 
     @Override
     protected List<IngestPipelineConfig> getIngestPipelines() {
@@ -65,6 +117,20 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry {
     }
 
     static final Map<String, ComposableIndexTemplate> COMPOSABLE_INDEX_TEMPLATES = parseComposableTemplates(
+        new IndexTemplateConfig(
+            CONNECTOR_TEMPLATE_NAME,
+            ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_TEMPLATE_NAME + ".json",
+            REGISTRY_VERSION,
+            TEMPLATE_VERSION_VARIABLE,
+            Map.of("connectors.index_pattern", CONNECTOR_INDEX_NAME_PATTERN)
+        ),
+        new IndexTemplateConfig(
+            CONNECTOR_SYNC_JOBS_TEMPLATE_NAME,
+            ROOT_TEMPLATE_RESOURCE_PATH + CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + ".json",
+            REGISTRY_VERSION,
+            TEMPLATE_VERSION_VARIABLE,
+            Map.of("connectors-sync-jobs.index_pattern", CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN)
+        ),
         new IndexTemplateConfig(
             ACCESS_CONTROL_TEMPLATE_NAME,
             ROOT_TEMPLATE_RESOURCE_PATH + ACCESS_CONTROL_TEMPLATE_NAME + ".json",
@@ -88,6 +154,11 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry {
         return ENT_SEARCH_ORIGIN;
     }
 
+    @Override
+    protected Map<String, ComponentTemplate> getComponentTemplateConfigs() {
+        return COMPONENT_TEMPLATES;
+    }
+
     @Override
     protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
         return COMPOSABLE_INDEX_TEMPLATES;

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java

@@ -11,12 +11,10 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DelegatingActionListener;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.DocWriteResponse;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
@@ -42,7 +40,6 @@ import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
-import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -52,10 +49,10 @@ import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.application.connector.Connector;
 import org.elasticsearch.xpack.application.connector.ConnectorFiltering;
 import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus;
+import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
 import org.elasticsearch.xpack.application.connector.filtering.FilteringRules;
 import org.elasticsearch.xpack.application.connector.syncjob.action.PostConnectorSyncJobAction;
 import org.elasticsearch.xpack.application.connector.syncjob.action.UpdateConnectorSyncJobIngestionStatsAction;
-import org.elasticsearch.xpack.core.template.TemplateUtils;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -72,7 +69,6 @@ import java.util.stream.Stream;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.application.connector.ConnectorIndexService.CONNECTOR_INDEX_NAME;
-import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTORS_ALLOWED_PRODUCT_ORIGINS;
 import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN;
 
 /**
@@ -85,17 +81,7 @@ public class ConnectorSyncJobIndexService {
     // The client to interact with the system index (internal user).
     private final Client clientWithOrigin;
 
-    // TODO use proper version IDs (see org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java)
+    public static final String CONNECTOR_SYNC_JOB_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN;
-    // TODO if this version is updated, a test should be added to
-    // javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java
-    private static final int CONNECTOR_SYNC_JOB_INDEX_VERSION = 1;
-    public static final String CONNECTOR_SYNC_JOB_INDEX_NAME = ".elastic-connectors-sync-jobs";
-    public static final String CONNECTOR_SYNC_JOB_INDEX_PREFIX = ".elastic-connectors-sync-jobs-v";
-    public static final String CONNECTOR_SYNC_JOB_CONCRETE_INDEX_NAME = CONNECTOR_SYNC_JOB_INDEX_PREFIX + CONNECTOR_SYNC_JOB_INDEX_VERSION;
-    public static final String CONNECTOR_SYNC_JOB_INDEX_NAME_PATTERN = CONNECTOR_SYNC_JOB_INDEX_NAME + "*";
-
-    private static final String CONNECTOR_SYNC_JOB_MAPPING_VERSION_VARIABLE = "elastic-connectors-sync-jobs.version";
-    private static final String CONNECTOR_SYNC_JOB_MAPPING_MANAGED_VERSION_VARIABLE = "elastic-connectors-sync-jobs.managed.index.version";
 
     /**
      * @param client A client for executing actions on the connectors sync jobs index.
@@ -104,35 +90,6 @@ public class ConnectorSyncJobIndexService {
         this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN);
     }
 
-    /**
-     * Returns the {@link SystemIndexDescriptor} for the Connector system index.
-     *
-     * @return The {@link SystemIndexDescriptor} for the Connector system index.
-     */
-    public static SystemIndexDescriptor getSystemIndexDescriptor() {
-        PutIndexTemplateRequest request = new PutIndexTemplateRequest();
-        String templateSource = TemplateUtils.loadTemplate(
-            "/elastic-connectors-sync-jobs.json",
-            Version.CURRENT.toString(),
-            CONNECTOR_SYNC_JOB_MAPPING_VERSION_VARIABLE,
-            Map.of(CONNECTOR_SYNC_JOB_MAPPING_MANAGED_VERSION_VARIABLE, Integer.toString(CONNECTOR_SYNC_JOB_INDEX_VERSION))
-        );
-        request.source(templateSource, XContentType.JSON);
-
-        return SystemIndexDescriptor.builder()
-            .setIndexPattern(CONNECTOR_SYNC_JOB_INDEX_NAME_PATTERN)
-            .setPrimaryIndex(CONNECTOR_SYNC_JOB_CONCRETE_INDEX_NAME)
-            .setAliasName(CONNECTOR_SYNC_JOB_INDEX_NAME)
-            .setDescription("Search connectors sync jobs")
-            .setMappings(request.mappings())
-            .setSettings(request.settings())
-            .setOrigin(CONNECTORS_ORIGIN)
-            .setType(SystemIndexDescriptor.Type.EXTERNAL_MANAGED)
-            .setAllowedElasticProductOrigins(CONNECTORS_ALLOWED_PRODUCT_ORIGINS)
-            .setNetNew()
-            .build();
-    }
-
     /**
      * @param request   Request for creating a connector sync job.
      * @param listener  Listener to respond to a successful response or an error.

x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java

@@ -14,9 +14,7 @@ import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.MockScriptPlugin;
@@ -61,6 +59,7 @@ import static org.elasticsearch.xpack.application.connector.ConnectorTemplateReg
 import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomConnectorFeatures;
 import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomCronExpression;
 import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.randomConnectorFeatureEnabled;
+import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.registerSimplifiedConnectorIndexTemplates;
 import static org.hamcrest.CoreMatchers.anyOf;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.not;
@@ -73,6 +72,7 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase {
 
     @Before
     public void setup() {
+        registerSimplifiedConnectorIndexTemplates(indicesAdmin());
         this.connectorIndexService = new ConnectorIndexService(client());
     }
 
@@ -80,7 +80,6 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase {
     protected Collection<Class<? extends Plugin>> getPlugins() {
         List<Class<? extends Plugin>> plugins = new ArrayList<>(super.getPlugins());
         plugins.add(MockPainlessScriptEngine.TestPlugin.class);
-        plugins.add(ConnectorIndexServiceTests.TestPlugin.class);
         return plugins;
     }
 
@@ -1613,24 +1612,4 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase {
         }
     }
 
-    /**
-     * Test plugin to register the {@link ConnectorIndexService} system index descriptor.
-     */
-    public static class TestPlugin extends Plugin implements SystemIndexPlugin {
-        @Override
-        public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-            return List.of(ConnectorIndexService.getSystemIndexDescriptor());
-        }
-
-        @Override
-        public String getFeatureName() {
-            return this.getClass().getSimpleName();
-        }
-
-        @Override
-        public String getFeatureDescription() {
-            return this.getClass().getCanonicalName();
-        }
-    }
-
 }

x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistryTests.java

@@ -55,13 +55,15 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.ACCESS_CONTROL_INDEX_NAME_PATTERN;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.oneOf;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.junit.Assert.assertNotNull;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
@@ -90,6 +92,14 @@ public class ConnectorTemplateRegistryTests extends ESTestCase {
         DiscoveryNode node = DiscoveryNodeUtils.create("node");
         DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
         Map<String, Integer> existingComponentTemplates = Map.of(
+            ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings",
+            ConnectorTemplateRegistry.REGISTRY_VERSION,
+            ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings",
+            ConnectorTemplateRegistry.REGISTRY_VERSION,
+            ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-mappings",
+            ConnectorTemplateRegistry.REGISTRY_VERSION,
+            ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-settings",
+            ConnectorTemplateRegistry.REGISTRY_VERSION,
             ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME,
             ConnectorTemplateRegistry.REGISTRY_VERSION
         );
@@ -115,6 +125,131 @@ public class ConnectorTemplateRegistryTests extends ESTestCase {
         });
     }
 
+    public void testThatNonExistingComponentTemplatesAreAddedImmediately() throws Exception {
+        DiscoveryNode node = DiscoveryNodeUtils.create("node");
+        DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
+
+        ClusterChangedEvent event = createClusterChangedEvent(
+            Collections.emptyMap(),
+            Collections.emptyMap(),
+            Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION),
+            Collections.emptyMap(),
+            nodes
+        );
+
+        AtomicInteger calledTimes = new AtomicInteger(0);
+        client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener));
+        registry.clusterChanged(event);
+        assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size())));
+
+        calledTimes.set(0);
+
+        // attempting to register the event multiple times as a race condition can yield this test flaky, namely:
+        // when calling registry.clusterChanged(newEvent) the templateCreationsInProgress state that the IndexTemplateRegistry maintains
+        // might've not yet been updated to reflect that the first template registration was complete, so a second template registration
+        // will not be issued anymore, leaving calledTimes to 0
+        assertBusy(() -> {
+            // now delete all templates from the cluster state and let's retry
+            ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), nodes);
+            registry.clusterChanged(newEvent);
+            assertThat(calledTimes.get(), greaterThan(4));
+        });
+    }
+
+    public void testThatVersionedOldComponentTemplatesAreUpgraded() throws Exception {
+        DiscoveryNode node = DiscoveryNodeUtils.create("node");
+        DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
+
+        ClusterChangedEvent event = createClusterChangedEvent(
+            Collections.emptyMap(),
+            Collections.singletonMap(
+                ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings",
+                ConnectorTemplateRegistry.REGISTRY_VERSION - 1
+            ),
+            Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION),
+            Collections.emptyMap(),
+            nodes
+        );
+        AtomicInteger calledTimes = new AtomicInteger(0);
+        client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener));
+        registry.clusterChanged(event);
+        assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size())));
+    }
+
+    public void testThatUnversionedOldComponentTemplatesAreUpgraded() throws Exception {
+        DiscoveryNode node = DiscoveryNodeUtils.create("node");
+        DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
+
+        ClusterChangedEvent event = createClusterChangedEvent(
+            Collections.emptyMap(),
+            Collections.singletonMap(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", null),
+            Collections.singletonMap(ConnectorTemplateRegistry.SEARCH_DEFAULT_PIPELINE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION),
+            Collections.emptyMap(),
+            nodes
+        );
+        AtomicInteger calledTimes = new AtomicInteger(0);
+        client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener));
+        registry.clusterChanged(event);
+        assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size())));
+    }
+
+    public void testSameOrHigherVersionComponentTemplateNotUpgraded() {
+        DiscoveryNode node = DiscoveryNodeUtils.create("node");
+        DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
+
+        Map<String, Integer> versions = new HashMap<>();
+        versions.put(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings", ConnectorTemplateRegistry.REGISTRY_VERSION);
+        versions.put(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings", ConnectorTemplateRegistry.REGISTRY_VERSION);
+        versions.put(ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-mappings", ConnectorTemplateRegistry.REGISTRY_VERSION);
+        versions.put(ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-settings", ConnectorTemplateRegistry.REGISTRY_VERSION);
+        versions.put(ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME, ConnectorTemplateRegistry.REGISTRY_VERSION);
+        ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(Collections.emptyMap(), versions, nodes);
+        client.setVerifier((action, request, listener) -> {
+            if (action == PutPipelineTransportAction.TYPE) {
+                // Ignore this, it's verified in another test
+                return AcknowledgedResponse.TRUE;
+            }
+            if (action instanceof PutComponentTemplateAction) {
+                fail("template should not have been re-installed");
+                return null;
+            } else if (action == ILMActions.PUT) {
+                // Ignore this, it's verified in another test
+                return AcknowledgedResponse.TRUE;
+            } else if (action == TransportPutComposableIndexTemplateAction.TYPE) {
+                // Ignore this, it's verified in another test
+                return AcknowledgedResponse.TRUE;
+            } else {
+                fail("client called with unexpected request:" + request.toString());
+                return null;
+            }
+        });
+        registry.clusterChanged(sameVersionEvent);
+
+        versions.clear();
+        versions.put(
+            ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-mappings",
+            ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000)
+        );
+        versions.put(
+            ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME + "-settings",
+            ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000)
+        );
+        versions.put(
+            ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-mappings",
+            ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000)
+        );
+        versions.put(
+            ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME + "-settings",
+            ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000)
+        );
+        versions.put(
+            ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME,
+            ConnectorTemplateRegistry.REGISTRY_VERSION + randomIntBetween(0, 1000)
+        );
+        ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(Collections.emptyMap(), versions, nodes);
+        registry.clusterChanged(higherVersionEvent);
+    }
+
     public void testThatMissingMasterNodeDoesNothing() {
         DiscoveryNode localNode = DiscoveryNodeUtils.create("node");
         DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build();
@@ -125,7 +260,7 @@ public class ConnectorTemplateRegistryTests extends ESTestCase {
         });
 
         ClusterChangedEvent event = createClusterChangedEvent(
-            Collections.singletonMap(ConnectorTemplateRegistry.ACCESS_CONTROL_TEMPLATE_NAME, null),
+            Collections.singletonMap(ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME, null),
             Collections.emptyMap(),
             nodes
         );
@@ -222,7 +357,10 @@ public class ConnectorTemplateRegistryTests extends ESTestCase {
             assertThat(putRequest.indexTemplate().version(), equalTo((long) ConnectorTemplateRegistry.REGISTRY_VERSION));
             final List<String> indexPatterns = putRequest.indexTemplate().indexPatterns();
             assertThat(indexPatterns, hasSize(1));
-            assertThat(indexPatterns, contains(ACCESS_CONTROL_INDEX_NAME_PATTERN));
+            assertThat(
+                indexPatterns,
+                contains(oneOf(ACCESS_CONTROL_INDEX_NAME_PATTERN, CONNECTOR_INDEX_NAME_PATTERN, CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN))
+            );
             assertNotNull(listener);
             return new TestPutIndexTemplateResponse(true);
         } else {

x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.application.connector;
 
+import org.elasticsearch.client.internal.IndicesAdminClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.xcontent.XContentType;
@@ -26,6 +27,7 @@ import org.elasticsearch.xpack.application.connector.filtering.FilteringRules;
 import org.elasticsearch.xpack.application.connector.filtering.FilteringValidation;
 import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationInfo;
 import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationState;
+import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobType;
 import org.elasticsearch.xpack.core.scheduler.Cron;
 
@@ -45,14 +47,55 @@ import static org.elasticsearch.test.ESTestCase.randomBoolean;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.test.ESTestCase.randomInt;
 import static org.elasticsearch.test.ESTestCase.randomList;
+import static org.elasticsearch.test.ESTestCase.randomLong;
 import static org.elasticsearch.test.ESTestCase.randomLongBetween;
-import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN;
-import static org.elasticsearch.test.ESTestCase.randomShort;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_TEMPLATE_NAME;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTOR_TEMPLATE_NAME;
 
 public final class ConnectorTestUtils {
 
     public static final String NULL_STRING = null;
 
+    /**
+     * Registers index templates for instances of {@link Connector} and {@link ConnectorSyncJob} with essential field mappings. This method
+     * only includes mappings for fields relevant to test cases, specifying field types to ensure correct ES query logic behavior.
+     *
+     * @param indicesAdminClient The Elasticsearch indices admin client used for template registration.
+     */
+
+    public static void registerSimplifiedConnectorIndexTemplates(IndicesAdminClient indicesAdminClient) {
+
+        indicesAdminClient.preparePutTemplate(CONNECTOR_TEMPLATE_NAME)
+            .setPatterns(List.of(CONNECTOR_INDEX_NAME_PATTERN))
+            .setVersion(0)
+            .setMapping(
+                "service_type",
+                "type=keyword,store=true",
+                "status",
+                "type=keyword,store=true",
+                "index_name",
+                "type=keyword,store=true",
+                "configuration",
+                "type=object"
+            )
+            .get();
+
+        indicesAdminClient.preparePutTemplate(CONNECTOR_SYNC_JOBS_TEMPLATE_NAME)
+            .setPatterns(List.of(CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN))
+            .setVersion(0)
+            .setMapping(
+                "job_type",
+                "type=keyword,store=true",
+                "connector.id",
+                "type=keyword,store=true",
+                "status",
+                "type=keyword,store=true"
+            )
+            .get();
+    }
+
     public static PutConnectorAction.Request getRandomPutConnectorActionRequest() {
         return new PutConnectorAction.Request(
             randomAlphaOfLengthBetween(5, 15),
@@ -101,9 +144,9 @@ public final class ConnectorTestUtils {
         return new ConnectorSyncInfo.Builder().setLastAccessControlSyncError(randomFrom(new String[] { null, randomAlphaOfLength(10) }))
             .setLastAccessControlSyncScheduledAt(randomFrom(new Instant[] { null, ConnectorTestUtils.randomInstant() }))
             .setLastAccessControlSyncStatus(randomFrom(new ConnectorSyncStatus[] { null, getRandomSyncStatus() }))
-            .setLastDeletedDocumentCount(randomNonNegativeLong())
+            .setLastDeletedDocumentCount(randomLong())
             .setLastIncrementalSyncScheduledAt(randomFrom(new Instant[] { null, ConnectorTestUtils.randomInstant() }))
-            .setLastIndexedDocumentCount(randomNonNegativeLong())
+            .setLastIndexedDocumentCount(randomLong())
             .setLastSyncError(randomFrom(new String[] { null, randomAlphaOfLength(10) }))
             .setLastSyncScheduledAt(randomFrom(new Instant[] { null, ConnectorTestUtils.randomInstant() }))
             .setLastSyncStatus(randomFrom(new ConnectorSyncStatus[] { null, getRandomSyncStatus() }))
@@ -154,7 +197,7 @@ public final class ConnectorTestUtils {
     public static ConnectorFiltering getRandomConnectorFiltering() {
 
         Instant currentTimestamp = Instant.now();
-        int order = randomShort();
+        int order = randomInt();
 
         return new ConnectorFiltering.Builder().setActive(
             new FilteringRules.Builder().setAdvancedSnippet(

x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java

@@ -20,11 +20,8 @@ import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
-import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.reindex.ReindexPlugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -61,6 +58,7 @@ import java.util.stream.Collectors;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.ACCESS_CONTROL_INDEX_PREFIX;
+import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.registerSimplifiedConnectorIndexTemplates;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -88,12 +86,14 @@ public class ConnectorSyncJobIndexServiceTests extends ESSingleNodeTestCase {
         List<Class<? extends Plugin>> plugins = new ArrayList<>(super.getPlugins());
         // Reindex plugin is required for testDeleteAllSyncJobsByConnectorId (supports delete_by_query)
         plugins.add(ReindexPlugin.class);
-        plugins.add(TestPlugin.class);
         return plugins;
     }
 
     @Before
     public void setup() throws Exception {
+
+        registerSimplifiedConnectorIndexTemplates(indicesAdmin());
+
         connectorOneId = createConnector(ConnectorTestUtils.getRandomConnector());
         connectorTwoId = createConnector(ConnectorTestUtils.getRandomConnector());
         connectorThreeId = createConnector(ConnectorTestUtils.getRandomConnectorWithDetachedIndex());
@@ -805,18 +805,18 @@ public class ConnectorSyncJobIndexServiceTests extends ESSingleNodeTestCase {
         Instant requestLastSeen = request.getLastSeen();
         Map<String, Object> metadata = request.getMetadata();
 
-        Long deletedDocumentCountAfterUpdate = ((Number) syncJobSourceAfterUpdate.get(
+        Long deletedDocumentCountAfterUpdate = (Long) syncJobSourceAfterUpdate.get(
             ConnectorSyncJob.DELETED_DOCUMENT_COUNT_FIELD.getPreferredName()
-        )).longValue();
+        );
-        Long indexedDocumentCountAfterUpdate = ((Number) syncJobSourceAfterUpdate.get(
+        Long indexedDocumentCountAfterUpdate = (Long) syncJobSourceAfterUpdate.get(
             ConnectorSyncJob.INDEXED_DOCUMENT_COUNT_FIELD.getPreferredName()
-        )).longValue();
+        );
-        Long indexedDocumentVolumeAfterUpdate = ((Number) syncJobSourceAfterUpdate.get(
+        Long indexedDocumentVolumeAfterUpdate = (Long) syncJobSourceAfterUpdate.get(
             ConnectorSyncJob.INDEXED_DOCUMENT_VOLUME_FIELD.getPreferredName()
-        )).longValue();
+        );
-        Long totalDocumentCountAfterUpdate = ((Number) syncJobSourceAfterUpdate.get(
+        Long totalDocumentCountAfterUpdate = (Long) syncJobSourceAfterUpdate.get(
             ConnectorSyncJob.TOTAL_DOCUMENT_COUNT_FIELD.getPreferredName()
-        )).longValue();
+        );
         Instant lastSeenAfterUpdate = Instant.parse(
             (String) syncJobSourceAfterUpdate.get(ConnectorSyncJob.LAST_SEEN_FIELD.getPreferredName())
         );
@@ -1411,24 +1411,4 @@ public class ConnectorSyncJobIndexServiceTests extends ESSingleNodeTestCase {
         // wait 10 seconds for connector creation
         return index.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).getId();
     }
-
-    /**
-     * Test plugin to register the {@link ConnectorSyncJobIndexService} system index descriptor.
-     */
-    public static class TestPlugin extends Plugin implements SystemIndexPlugin {
-        @Override
-        public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-            return List.of(ConnectorSyncJobIndexService.getSystemIndexDescriptor());
-        }
-
-        @Override
-        public String getFeatureName() {
-            return this.getClass().getSimpleName();
-        }
-
-        @Override
-        public String getFeatureDescription() {
-            return this.getClass().getCanonicalName();
-        }
-    }
 }

x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java

@@ -36,7 +36,7 @@ import static org.elasticsearch.test.ESTestCase.randomInstantBetween;
 import static org.elasticsearch.test.ESTestCase.randomInt;
 import static org.elasticsearch.test.ESTestCase.randomLong;
 import static org.elasticsearch.test.ESTestCase.randomMap;
-import static org.elasticsearch.test.ESTestCase.randomNonNegativeInt;
+import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong;
 
 public class ConnectorSyncJobTestUtils {
 
@@ -51,11 +51,11 @@ public class ConnectorSyncJobTestUtils {
             .setCompletedAt(randomFrom(new Instant[] { null, randomInstantBetween(lowerBoundInstant, upperBoundInstant) }))
             .setConnector(ConnectorTestUtils.getRandomSyncJobConnectorInfo())
             .setCreatedAt(randomInstantBetween(lowerBoundInstant, upperBoundInstant))
-            .setDeletedDocumentCount(randomNonNegativeInt())
+            .setDeletedDocumentCount(randomLong())
             .setError(randomFrom(new String[] { null, randomAlphaOfLength(10) }))
             .setId(randomAlphaOfLength(10))
-            .setIndexedDocumentCount(randomNonNegativeInt())
+            .setIndexedDocumentCount(randomLong())
-            .setIndexedDocumentVolume(randomNonNegativeInt())
+            .setIndexedDocumentVolume(randomLong())
             .setJobType(getRandomConnectorJobType())
             .setLastSeen(randomFrom(new Instant[] { null, randomInstantBetween(lowerBoundInstant, upperBoundInstant) }))
             .setMetadata(
@@ -67,7 +67,7 @@ public class ConnectorSyncJobTestUtils {
             )
             .setStartedAt(randomFrom(new Instant[] { null, randomInstantBetween(lowerBoundInstant, upperBoundInstant) }))
             .setStatus(ConnectorTestUtils.getRandomSyncStatus())
-            .setTotalDocumentCount(randomNonNegativeInt())
+            .setTotalDocumentCount(randomLong())
             .setTriggerMethod(getRandomConnectorSyncJobTriggerMethod())
             .setWorkerHostname(randomAlphaOfLength(10))
             .build();
@@ -156,10 +156,10 @@ public class ConnectorSyncJobTestUtils {
 
         return new UpdateConnectorSyncJobIngestionStatsAction.Request(
             randomAlphaOfLength(10),
-            (long) randomNonNegativeInt(),
+            randomNonNegativeLong(),
-            (long) randomNonNegativeInt(),
+            randomNonNegativeLong(),
-            (long) randomNonNegativeInt(),
+            randomNonNegativeLong(),
-            (long) randomNonNegativeInt(),
+            randomNonNegativeLong(),
             randomInstantBetween(lowerBoundInstant, upperBoundInstant),
             randomMap(2, 3, () -> new Tuple<>(randomAlphaOfLength(4), randomAlphaOfLength(4)))
         );
@@ -173,10 +173,10 @@ public class ConnectorSyncJobTestUtils {
 
         return new UpdateConnectorSyncJobIngestionStatsAction.Request(
             syncJobId,
-            (long) randomNonNegativeInt(),
+            randomNonNegativeLong(),
-            (long) randomNonNegativeInt(),
+            randomNonNegativeLong(),
-            (long) randomNonNegativeInt(),
+            randomNonNegativeLong(),
-            (long) randomNonNegativeInt(),
+            randomNonNegativeLong(),
             randomInstantBetween(lowerBoundInstant, upperBoundInstant),
             randomMap(2, 3, () -> new Tuple<>(randomAlphaOfLength(4), randomAlphaOfLength(4)))
         );