Add xpack core and security to INTEG TEST (#77632)

This change makes it so x-pack-core and x-pack-security are bundled
in the INTEG TEST distribution that we use for testClusters in our
tests. There are two reasons for this:

- In https://github.com/elastic/elasticsearch/pull/77231 where we
are looking into enabling and auto-configuring security by default
for all nodes, we need to call out to ConfigInitialNode to
determine whether we should do the auto-configuration or not.
- Since we are enabling security by default, we should be looking
into enabling security for all for our tests moving forward, or
at least make a conscious decision about which ones run without
security. This change is a step towards that direction.

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -318,6 +318,9 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
         }
         ElasticsearchNode firstNode = null;
         for (ElasticsearchNode node : nodes) {
+            if (node.getTestDistribution().equals(TestDistribution.INTEG_TEST)) {
+                node.defaultConfig.put("xpack.security.enabled", "false");
+            }
             // Can only configure master nodes if we have node names defined
             if (nodeNames != null) {
                 if (node.getVersion().onOrAfter("7.0.0")) {

distribution/archives/build.gradle

@@ -15,7 +15,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
-        with libFiles(isTestDistro)
+        with libFiles
       }
       into('config') {
         dirMode 0750
@@ -65,7 +65,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
 distribution_archives {
   integTestZip {
     content {
-      archiveFiles(transportModulesFiles, 'zip', null, 'x64', true)
+      archiveFiles(integTestModulesFiles, 'zip', null, 'x64', true)
     }
   }
 

distribution/build.gradle

@@ -84,7 +84,7 @@ tasks.withType(NoticeTask).configureEach {
  *****************************************************************************/
 String defaultOutputs = 'build/outputs/default'
 String systemdOutputs = 'build/outputs/systemd'
-String transportOutputs = 'build/outputs/transport-only'
+String integTestOutputs = 'build/outputs/integ-test-only'
 String externalTestOutputs = 'build/outputs/external-test'
 
 def processDefaultOutputsTaskProvider = tasks.register("processDefaultOutputs", Sync) {
@@ -101,8 +101,8 @@ def processExternalTestOutputsTaskProvider = tasks.register("processExternalTest
 
 // Integ tests work over the rest http layer, so we need a transport included with the integ test zip.
 // All transport modules are included so that they may be randomized for testing
-def processTransportOutputsTaskProvider = tasks.register("processTransportOutputs", Sync) {
+def processIntegTestOutputsTaskProvider = tasks.register("processIntegTestOutputs", Sync) {
-  into transportOutputs
+  into integTestOutputs
 }
 
 def defaultModulesFiles = fileTree("${defaultOutputs}/modules") {
@@ -120,10 +120,11 @@ def systemdModuleFiles = fileTree("${systemdOutputs}/modules") {
   builtBy processSystemdOutputsTaskProvider
 }
 
-def buildTransportModulesTaskProvider = tasks.register("buildTransportModules") {
+def buildIntegTestModulesTaskProvider = tasks.register("buildIntegTestModules") {
-  dependsOn processTransportOutputsTaskProvider
+  dependsOn processIntegTestOutputsTaskProvider
-  outputs.dir "${transportOutputs}/modules"
+  outputs.dir "${integTestOutputs}/modules"
 }
+
 def buildExternalTestModulesTaskProvider = tasks.register("buildExternalTestModules") {
   dependsOn "processExternalTestOutputs"
   outputs.dir "${externalTestOutputs}/modules"
@@ -211,7 +212,7 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each {
 
   copyModule(processDefaultOutputsTaskProvider, module)
   if (module.name.startsWith('transport-')) {
-    copyModule(processTransportOutputsTaskProvider, module)
+    copyModule(processIntegTestOutputsTaskProvider, module)
   }
 
   restTestExpansions['expected.modules.count'] += 1
@@ -228,6 +229,9 @@ xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
     }
   }
   copyModule(processDefaultOutputsTaskProvider, xpackModule)
+  if (xpackModule.name.equals('core') || xpackModule.name.equals('security')) {
+    copyModule(processIntegTestOutputsTaskProvider, xpackModule)
+  }
 }
 
 copyModule(processSystemdOutputsTaskProvider, project(':modules:systemd'))
@@ -289,7 +293,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     /*****************************************************************************
      *                   Common files in all distributions                       *
      *****************************************************************************/
-    libFiles = { testDistro ->
+    libFiles =
       copySpec {
         // delay by using closures, since they have not yet been configured, so no jar task exists yet
         from(configurations.libs)
@@ -302,13 +306,11 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         into('tools/keystore-cli') {
           from(configurations.libsKeystoreCli)
         }
-        if (testDistro == false) {
+        into('tools/security-cli') {
-          into('tools/security-cli') {
+          from(configurations.libsSecurityCli)
-            from(configurations.libsSecurityCli)
-          }
         }
       }
-    }
+
 
     modulesFiles = { platform ->
       copySpec {
@@ -345,8 +347,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       }
     }
 
-    transportModulesFiles = copySpec {
+    integTestModulesFiles = copySpec {
-      from buildTransportModulesTaskProvider
+      from buildIntegTestModulesTaskProvider
     }
 
     configFiles = { distributionType, isTestDistro ->
@@ -388,8 +390,11 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         // module provided bin files
         with copySpec {
           eachFile { it.setMode(0755) }
-          if(testDistro == false) {
+          if (testDistro == false) {
             from(defaultBinFiles)
+          } else {
+            from(defaultBinFiles)
+              include 'x-pack-env', 'x-pack-security-env'
           }
           if (distributionType != 'zip') {
             exclude '*.bat'

distribution/packages/build.gradle

@@ -130,7 +130,7 @@ def commonPackageConfig(String type, String architecture) {
         fileMode 0644
       }
       into('lib') {
-        with libFiles(false)
+        with libFiles
       }
       into('modules') {
         with modulesFiles('linux-' + ((architecture == 'x64') ? 'x86_64' : architecture))

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -170,6 +170,10 @@ public abstract class ESRestTestCase extends ESTestCase {
      */
     private static RestClient adminClient;
     private static Boolean hasXPack;
+    private static Boolean hasIlm;
+    private static Boolean hasRollups;
+    private static Boolean hasCcr;
+    private static Boolean hasShutdown;
     private static TreeSet<Version> nodeVersions;
 
     @Before
@@ -178,6 +182,10 @@ public abstract class ESRestTestCase extends ESTestCase {
             assert adminClient == null;
             assert clusterHosts == null;
             assert hasXPack == null;
+            assert hasIlm == null;
+            assert hasRollups == null;
+            assert hasCcr == null;
+            assert hasShutdown == null;
             assert nodeVersions == null;
             String cluster = getTestRestCluster();
             String[] stringUrls = cluster.split(",");
@@ -197,6 +205,10 @@ public abstract class ESRestTestCase extends ESTestCase {
             adminClient = buildClient(restAdminSettings(), clusterHosts.toArray(new HttpHost[clusterHosts.size()]));
 
             hasXPack = false;
+            hasIlm = false;
+            hasRollups = false;
+            hasCcr = false;
+            hasShutdown = false;
             nodeVersions = new TreeSet<>();
             Map<?, ?> response = entityAsMap(adminClient.performRequest(new Request("GET", "_nodes/plugins")));
             Map<?, ?> nodes = (Map<?, ?>) response.get("nodes");
@@ -205,9 +217,22 @@ public abstract class ESRestTestCase extends ESTestCase {
                 nodeVersions.add(Version.fromString(nodeInfo.get("version").toString()));
                 for (Object module: (List<?>) nodeInfo.get("modules")) {
                     Map<?, ?> moduleInfo = (Map<?, ?>) module;
-                    if (moduleInfo.get("name").toString().startsWith("x-pack-")) {
+                    final String moduleName = moduleInfo.get("name").toString();
+                    if (moduleName.startsWith("x-pack")) {
                         hasXPack = true;
                     }
+                    if (moduleName.equals("x-pack-ilm")) {
+                        hasIlm = true;
+                    }
+                    if (moduleName.equals("x-pack-rollup")) {
+                        hasRollups = true;
+                    }
+                    if (moduleName.equals("x-pack-ccr")) {
+                        hasCcr = true;
+                    }
+                    if (moduleName.equals("x-pack-shutdown")) {
+                        hasShutdown = true;
+                    }
                 }
             }
         }
@@ -215,6 +240,10 @@ public abstract class ESRestTestCase extends ESTestCase {
         assert adminClient != null;
         assert clusterHosts != null;
         assert hasXPack != null;
+        assert hasIlm != null;
+        assert hasRollups != null;
+        assert hasCcr != null;
+        assert hasShutdown != null;
         assert nodeVersions != null;
     }
 
@@ -354,6 +383,10 @@ public abstract class ESRestTestCase extends ESTestCase {
             client = null;
             adminClient = null;
             hasXPack = null;
+            hasRollups = null;
+            hasCcr = null;
+            hasShutdown = null;
+            hasIlm = null;
             nodeVersions = null;
         }
     }
@@ -559,7 +592,7 @@ public abstract class ESRestTestCase extends ESTestCase {
         // Cleanup rollup before deleting indices.  A rollup job might have bulks in-flight,
         // so we need to fully shut them down first otherwise a job might stall waiting
         // for a bulk to finish against a non-existing index (and then fail tests)
-        if (hasXPack && false == preserveRollupJobsUponCompletion()) {
+        if (hasRollups && false == preserveRollupJobsUponCompletion()) {
             wipeRollupJobs();
             waitForPendingRollupTasks();
         }
@@ -711,11 +744,11 @@ public abstract class ESRestTestCase extends ESTestCase {
             wipeClusterSettings();
         }
 
-        if (hasXPack && false == preserveILMPoliciesUponCompletion()) {
+        if (hasIlm && false == preserveILMPoliciesUponCompletion()) {
             deleteAllILMPolicies(preserveILMPolicyIds());
         }
 
-        if (hasXPack && false == preserveAutoFollowPatternsUponCompletion()) {
+        if (hasCcr && false == preserveAutoFollowPatternsUponCompletion()) {
             deleteAllAutoFollowPatterns();
         }
 
@@ -729,7 +762,7 @@ public abstract class ESRestTestCase extends ESTestCase {
      */
     @SuppressWarnings("unchecked")
     protected void deleteAllNodeShutdownMetadata() throws IOException {
-        if (hasXPack() == false || minimumNodeVersion().before(Version.V_7_15_0)) {
+        if (hasShutdown == false || minimumNodeVersion().before(Version.V_7_15_0)) {
             // Node shutdown APIs are only present in xpack
             return;
         }