Re-enable plugin and upgrade bats tests (#51565)

While the tests remained in place, the execution of these tests were
lost when creating the DistroTestPlugin. This commit restores executing
them.

Vagrantfile

@@ -491,22 +491,12 @@ JAVA
       rm -rf /tmp/bats
     }
 
-    cat \<\<VARS > /etc/profile.d/elasticsearch_vars.sh
-export ZIP=/elasticsearch/distribution/zip/build/distributions
-export TAR=/elasticsearch/distribution/tar/build/distributions
-export RPM=/elasticsearch/distribution/rpm/build/distributions
-export DEB=/elasticsearch/distribution/deb/build/distributions
-export PACKAGING_TESTS=/project/build/packaging/tests
-VARS
     cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
-Defaults   env_keep += "ZIP"
-Defaults   env_keep += "TAR"
-Defaults   env_keep += "RPM"
-Defaults   env_keep += "DEB"
-Defaults   env_keep += "PACKAGING_ARCHIVES"
-Defaults   env_keep += "PACKAGING_TESTS"
 Defaults   env_keep += "BATS_UTILS"
 Defaults   env_keep += "BATS_TESTS"
+Defaults   env_keep += "BATS_PLUGINS"
+Defaults   env_keep += "BATS_UPGRADE"
+Defaults   env_keep += "PACKAGE_NAME"
 Defaults   env_keep += "JAVA_HOME"
 Defaults   env_keep += "SYSTEM_JAVA_HOME"
 SUDOERS_VARS

buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -118,13 +118,22 @@ public class DistroTestPlugin implements Plugin<Project> {
             lifecyleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask));
         }
 
-        Map<String, TaskProvider<?>> batsTests = new HashMap<>();
+        TaskProvider<BatsTestTask> batsPluginsTest = configureBatsTest(
-        configureBatsTest(project, "plugins", distributionsDir, copyDistributionsTask, copyPluginsTask).configure(
+            project,
-            t -> t.setPluginsDir(pluginsDir)
+            "plugins",
+            distributionsDir,
+            copyDistributionsTask,
+            copyPluginsTask
         );
-        configureBatsTest(project, "upgrade", distributionsDir, copyDistributionsTask, copyUpgradeTask).configure(
+        batsPluginsTest.configure(t -> t.setPluginsDir(pluginsDir));
-            t -> t.setUpgradeDir(upgradeDir)
+        TaskProvider<BatsTestTask> batsUpgradeTest = configureBatsTest(
+            project,
+            "upgrade",
+            distributionsDir,
+            copyDistributionsTask,
+            copyUpgradeTask
         );
+        batsUpgradeTest.configure(t -> t.setUpgradeDir(upgradeDir));
 
         project.subprojects(vmProject -> {
             vmProject.getPluginManager().apply(VagrantBasePlugin.class);
@@ -168,12 +177,15 @@ public class DistroTestPlugin implements Plugin<Project> {
                 }
             }
 
-            batsTests.forEach((desc, task) -> {
+            configureVMWrapperTask(vmProject, "bats plugins", batsPluginsTest.getName(), vmDependencies).configure(t -> {
-                configureVMWrapperTask(vmProject, desc, task.getName(), vmDependencies).configure(t -> {
+                t.setProgressHandler(new BatsProgressLogger(project.getLogger()));
-                    t.setProgressHandler(new BatsProgressLogger(project.getLogger()));
+                t.onlyIf(spec -> isWindows(vmProject) == false); // bats doesn't run on windows
-                    t.onlyIf(spec -> isWindows(vmProject) == false); // bats doesn't run on windows
+                t.dependsOn(copyDistributionsTask, copyPluginsTask);
-                    t.dependsOn(copyDistributionsTask);
+            });
-                });
+            configureVMWrapperTask(vmProject, "bats upgrade", batsUpgradeTest.getName(), vmDependencies).configure(t -> {
+                t.setProgressHandler(new BatsProgressLogger(project.getLogger()));
+                t.onlyIf(spec -> isWindows(vmProject) == false); // bats doesn't run on windows
+                t.dependsOn(copyDistributionsTask, copyUpgradeTask);
             });
         });
     }

qa/os/bats/plugins/module_and_plugin_test_cases.bash

@@ -210,10 +210,10 @@ fi
 }
 
 @test "[$GROUP] install ingest-attachment plugin" {
-    # we specify the version on the poi-4.0.0.jar so that the test does
+    # we specify the version on the poi-4.0.1.jar so that the test does
     # not spuriously pass if the jar is missing but the other poi jars
     # are present
-    install_and_check_plugin ingest attachment bcprov-jdk15on-*.jar tika-core-*.jar pdfbox-*.jar poi-4.0.0.jar poi-ooxml-4.0.0.jar poi-ooxml-schemas-*.jar poi-scratchpad-*.jar
+    install_and_check_plugin ingest attachment bcprov-jdk15on-*.jar tika-core-*.jar pdfbox-*.jar poi-4.0.1.jar poi-ooxml-4.0.1.jar poi-ooxml-schemas-*.jar poi-scratchpad-*.jar
 }
 
 @test "[$GROUP] check ingest-common module" {
@@ -240,7 +240,7 @@ fi
 }
 
 @test "[$GROUP] check lang-painless module" {
-    check_secure_module lang-painless antlr4-runtime-*.jar asm-debug-all-*.jar
+    check_secure_module lang-painless antlr4-runtime-*.jar asm-util-*.jar asm-tree-*.jar asm-commons-*.jar asm-analysis-*.jar
 }
 
 @test "[$GROUP] install murmur3 mapper plugin" {
@@ -408,8 +408,8 @@ fi
     sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install --batch "file://$relativePath" > /tmp/plugin-cli-output
     # exclude progress line
     local loglines=$(cat /tmp/plugin-cli-output | grep -v "^[[:cntrl:]]" | wc -l)
-    [ "$loglines" -eq "2" ] || {
+    [ "$loglines" -eq "3" ] || {
-        echo "Expected 2 lines excluding progress bar but the output had $loglines lines and was:"
+        echo "Expected 3 lines excluding progress bar but the output had $loglines lines and was:"
         cat /tmp/plugin-cli-output
         false
     }

qa/os/bats/upgrade/80_upgrade.bats

@@ -46,6 +46,8 @@ setup() {
     sameVersion="false"
     if [ "$(cat $BATS_UPGRADE/upgrade_from_version)" == "$(cat version)" ]; then
         sameVersion="true"
+    else
+        echo "BWC test version: $(cat $BATS_UPGRADE/upgrade_from_version)"
     fi
     # TODO: this needs to conditionally change based on version > 6.3.0
     if [ -f $BATS_UPGRADE/upgrade_is_oss ]; then

qa/os/bats/utils/packages.bash

@@ -59,7 +59,7 @@ install_package() {
     local version=$(cat version)
     local rpmCommand='-i'
     local dir='./'
-    while getopts ":fuv:" opt; do
+    while getopts ":ufd:v:" opt; do
         case $opt in
             u)
                 rpmCommand='-U'

qa/os/bats/utils/plugins.bash

@@ -185,5 +185,7 @@ install_meta_plugin() {
 compare_plugins_list() {
     cat $1 | sort > /tmp/plugins
     echo "Checking plugins from $2 (<) against expected plugins (>):"
-    diff -w /elasticsearch/qa/vagrant/build/plugins/expected /tmp/plugins
+    # bats tests are run under build/packaging/distributions, and expected file is in build/plugins/expected
+    # this can't be an absolute path since it differs whether running in vagrant or GCP
+    diff -w ../../plugins/expected /tmp/plugins
 }

qa/os/bats/utils/utils.bash

@@ -428,7 +428,7 @@ describe_port() {
 }
 
 debug_collect_logs() {
-    local es_logfile="$ESLOG/elasticsearch_server.json"
+    local es_logfile="/var/log/elasticsearch/elasticsearch.log"
     local system_logfile='/var/log/messages'
 
     if [ -e "$es_logfile" ]; then
@@ -497,6 +497,7 @@ wait_for_elasticsearch_status() {
     fi
     echo $output | grep $desiredStatus || {
         echo "unexpected status:  '$output' wanted '$desiredStatus'"
+        debug_collect_logs
         false
     }
 }

qa/os/build.gradle

@@ -58,7 +58,7 @@ tasks.dependenciesInfo.enabled = false
 tasks.thirdPartyAudit.ignoreMissingClasses()
 
 tasks.register('destructivePackagingTest') {
-  dependsOn 'destructiveDistroTest'
+  dependsOn 'destructiveDistroTest', 'destructiveBatsTest.plugins', 'destructiveBatsTest.upgrade'
 }
 
 processTestResources {
@@ -67,7 +67,7 @@ processTestResources {
 
 subprojects { Project platformProject ->
   tasks.register('packagingTest') {
-    dependsOn 'distroTest'
+    dependsOn 'distroTest', 'batsTest.plugins', 'batsTest.upgrade'
   }
 
   vagrant {