From 16af105262ca642dfcff2775bc1b7e62c28e3dc7 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Thu, 6 Apr 2023 17:45:48 -0700
Subject: [PATCH] Add mechanism to initialize YAML tests against a subset of
 test cases (#95095)

This commit adds the ability to initialize YAML rest test suites against
a subset of available test cases. Previously, the only way to do this is
via the `tests.rest.suite` system property, but that can only be set at
the test _task_ level. Configuring this at the test _class_ level means
that we can support having multiple test suite classes that execute
subsets of tests within a project. That allows for things like
parallelization, or having different test cluster setups for different
YAML tests within the same project.

For example:

```java
    @ParametersFactory
    public static Iterable<Object[]> parameters() throws Exception {
        return ESClientYamlSuiteTestCase.createParameters(new String[] { "analysis-common", "indices.analyze" });
    }
```

The above example would mean that only tests in the `analysis-common`
and `indices.analyze` directories would be included in this suite.

cc @jdconrad

Closes #95089
---
 TESTING.asciidoc                              | 12 ++++----
 .../rest/yaml/ESClientYamlSuiteTestCase.java  | 29 +++++++++++++++++--
 2 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index f574633a90ec..305e0f637183 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -446,13 +446,15 @@ A specific test case can be run with the following command:
   -Dtests.method="test {yaml=cat.segments/10_basic/Help}"
 ---------------------------------------------------------------------------
 
+You can run a group of YAML test by using wildcards:
+
+---------------------------------------------------------------------------
+./gradlew :rest-api-spec:yamlRestTest \
+  --tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT.test {yaml=index/*/*}"
+---------------------------------------------------------------------------
+
 The YAML REST tests support all the options provided by the randomized runner, plus the following:
 
-* `tests.rest.suite`: comma separated paths of the test suites to be run
-(by default loaded from /rest-api-spec/test). It is possible to run only a subset
-of the tests providing a sub-folder or even a single yaml file (the default
-/rest-api-spec/test prefix is optional when files are loaded from classpath)
-e.g. -Dtests.rest.suite=index,get,create/10_with_id
 * `tests.rest.blacklist`: comma separated globs that identify tests that are
 blacklisted and need to be skipped
 e.g. -Dtests.rest.blacklist=index/*/Index document,get/10_basic/*
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
index 001ed5eb7f94..cb60cb2758dd 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
@@ -220,7 +220,32 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
      * Create parameters for this parameterized test.
      */
     public static Iterable<Object[]> createParameters(NamedXContentRegistry executeableSectionRegistry) throws Exception {
-        String[] paths = resolvePathsProperty(REST_TESTS_SUITE, ""); // default to all tests under the test root
+        return createParameters(executeableSectionRegistry, null);
+    }
+
+    /**
+     * Create parameters for this parameterized test.
+     */
+    public static Iterable<Object[]> createParameters(String[] testPaths) throws Exception {
+        return createParameters(ExecutableSection.XCONTENT_REGISTRY, testPaths);
+    }
+
+    /**
+     * Create parameters for this parameterized test.
+     *
+     * @param executeableSectionRegistry registry of executable sections
+     * @param testPaths list of paths to explicitly search for tests. If <code>null</code> then include all tests in root path.
+     * @return list of test candidates.
+     * @throws Exception
+     */
+    public static Iterable<Object[]> createParameters(NamedXContentRegistry executeableSectionRegistry, String[] testPaths)
+        throws Exception {
+        if (testPaths != null && System.getProperty(REST_TESTS_SUITE) != null) {
+            throw new IllegalArgumentException("The '" + REST_TESTS_SUITE + "' system property is not supported with explicit test paths.");
+        }
+
+        // default to all tests under the test root
+        String[] paths = testPaths == null ? resolvePathsProperty(REST_TESTS_SUITE, "") : testPaths;
         Map<String, Set<Path>> yamlSuites = loadSuites(paths);
         List<ClientYamlTestSuite> suites = new ArrayList<>();
         IllegalArgumentException validationException = null;
@@ -283,7 +308,7 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
                     });
                 } else {
                     path = root.resolve(strPath + ".yml");
-                    assert Files.exists(path);
+                    assert Files.exists(path) : "Path " + path + " does not exist in YAML test root";
                     addSuite(root, path, files);
                 }
             }