From 2b175653d9f70e65ea3dc2d2ac1f30c6bd79b2d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Mon, 15 Jan 2024 14:48:36 +0100 Subject: [PATCH] YAML test framework: separate `skip` and `requires` sections (#104140) * Introduce Prerequisites criteria (Predicate + factory) for modular skip decisions - Removed accessors to specific criteria from SkipSection (used only on tests), adjusted test assertions - Moved Features check (YAML test runner features) to SkipSection build time * Separated check for xpack/no_xpack Check for xpack is cluster-configuration (modules installed) dependent, while Features are meant to be "static" test-runner capabilities. We separate them so checks on one (test-runner features) can be run before and separately from the other. * Consolidate skip() methods - Divide require and skip predicates - Divide requires and skip parsing (distinct sections) - Renaming SkipSection to PrerequisiteSection and related methods/fields (e.g. skip -> evaluate) * Refactoring tests - moving and adding VersionRange tests - adding specific version and os skip tests - modified parse/validate/build to make SkipSection more unit-testable * Adding cluster feature-based skip criteria * Updated javadoc + renaming + better skip reason message --- .../rest/yaml/CcsCommonYamlTestSuiteIT.java | 2 +- .../rest/yaml/ESClientYamlSuiteTestCase.java | 20 +- .../yaml/section/ClientYamlTestSection.java | 14 +- .../yaml/section/ClientYamlTestSuite.java | 53 +- .../yaml/section/PrerequisiteSection.java | 413 ++++++++++++ .../{SkipCriteria.java => Prerequisites.java} | 29 +- .../test/rest/yaml/section/SetupSection.java | 16 +- .../test/rest/yaml/section/SkipSection.java | 255 ------- .../rest/yaml/section/TeardownSection.java | 16 +- .../section/ClientYamlTestSectionTests.java | 10 +- .../section/ClientYamlTestSuiteTests.java | 208 ++++-- .../section/PrerequisiteSectionTests.java | 630 ++++++++++++++++++ .../rest/yaml/section/SetupSectionTests.java | 10 +- .../rest/yaml/section/SkipSectionTests.java | 311 --------- .../yaml/section/TeardownSectionTests.java | 6 +- .../qa/single_node/EsqlClientYamlAsyncIT.java | 2 +- .../test/CoreTestTranslater.java | 4 +- 17 files changed, 1280 insertions(+), 719 deletions(-) create mode 100644 test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java rename test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/{SkipCriteria.java => Prerequisites.java} (54%) delete mode 100644 test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java create mode 100644 test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSectionTests.java delete mode 100644 test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index 88740edffc09..e709b838a26f 100644 --- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -259,7 +259,7 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { new ClientYamlTestSection( testSection.getLocation(), testSection.getName(), - testSection.getSkipSection(), + testSection.getPrerequisiteSection(), modifiedExecutableSections ) ); diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 049102f87a54..4be9481df58b 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -446,20 +446,10 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase { } // skip test if the whole suite (yaml file) is disabled - assumeFalse( - testCandidate.getSetupSection().getSkipSection().getSkipMessage(testCandidate.getSuitePath()), - testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext) - ); - // skip test if the whole suite (yaml file) is disabled - assumeFalse( - testCandidate.getTeardownSection().getSkipSection().getSkipMessage(testCandidate.getSuitePath()), - testCandidate.getTeardownSection().getSkipSection().skip(restTestExecutionContext) - ); + testCandidate.getSetupSection().getPrerequisiteSection().evaluate(restTestExecutionContext, testCandidate.getSuitePath()); + testCandidate.getTeardownSection().getPrerequisiteSection().evaluate(restTestExecutionContext, testCandidate.getSuitePath()); // skip test if test section is disabled - assumeFalse( - testCandidate.getTestSection().getSkipSection().getSkipMessage(testCandidate.getTestPath()), - testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext) - ); + testCandidate.getTestSection().getPrerequisiteSection().evaluate(restTestExecutionContext, testCandidate.getTestPath()); // let's check that there is something to run, otherwise there might be a problem with the test section if (testCandidate.getTestSection().getExecutableSections().isEmpty()) { @@ -468,11 +458,11 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase { assumeFalse( "[" + testCandidate.getTestPath() + "] skipped, reason: in fips 140 mode", - inFipsJvm() && testCandidate.getTestSection().getSkipSection().yamlRunnerHasFeature("fips_140") + inFipsJvm() && testCandidate.getTestSection().getPrerequisiteSection().hasYamlRunnerFeature("fips_140") ); final Settings globalTemplateSettings = getGlobalTemplateSettings( - testCandidate.getTestSection().getSkipSection().yamlRunnerHasFeature("default_shards") + testCandidate.getTestSection().getPrerequisiteSection().hasYamlRunnerFeature("default_shards") ); if (globalTemplateSettings.isEmpty() == false && ESRestTestCase.has(ProductFeature.LEGACY_TEMPLATES)) { diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java index 740befe2f3a6..f679a725c4fe 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -28,7 +28,7 @@ public class ClientYamlTestSection implements Comparable List executableSections = new ArrayList<>(); try { parser.nextToken(); - SkipSection skipSection = SkipSection.parseIfNext(parser); + PrerequisiteSection prerequisiteSection = PrerequisiteSection.parseIfNext(parser); while (parser.currentToken() != XContentParser.Token.END_ARRAY) { ParserUtils.advanceToFieldName(parser); executableSections.add(ExecutableSection.parse(parser)); @@ -45,7 +45,7 @@ public class ClientYamlTestSection implements Comparable ); } parser.nextToken(); - return new ClientYamlTestSection(sectionLocation, sectionName, skipSection, executableSections); + return new ClientYamlTestSection(sectionLocation, sectionName, prerequisiteSection, executableSections); } catch (Exception e) { throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + sectionName + "]", e); } @@ -53,18 +53,18 @@ public class ClientYamlTestSection implements Comparable private final XContentLocation location; private final String name; - private final SkipSection skipSection; + private final PrerequisiteSection prerequisiteSection; private final List executableSections; public ClientYamlTestSection( XContentLocation location, String name, - SkipSection skipSection, + PrerequisiteSection prerequisiteSection, List executableSections ) { this.location = location; this.name = name; - this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null"); + this.prerequisiteSection = Objects.requireNonNull(prerequisiteSection, "skip section cannot be null"); this.executableSections = Collections.unmodifiableList(executableSections); } @@ -76,8 +76,8 @@ public class ClientYamlTestSection implements Comparable return name; } - public SkipSection getSkipSection() { - return skipSection; + public PrerequisiteSection getPrerequisiteSection() { + return prerequisiteSection; } public List getExecutableSections() { diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java index 48f24d3a935a..e5f46ff13517 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java @@ -175,9 +175,9 @@ public class ClientYamlTestSuite { .filter(section -> section instanceof DoSection) .map(section -> (DoSection) section) .filter(section -> false == section.getExpectedWarningHeaders().isEmpty()) - .filter(section -> false == hasSkipFeature("warnings", testSection, setupSection, teardownSection)) + .filter(section -> false == hasYamlRunnerFeature("warnings", testSection, setupSection, teardownSection)) .map(section -> String.format(Locale.ROOT, """ - attempted to add a [do] with a [warnings] section without a corresponding ["skip": "features": "warnings"] \ + attempted to add a [do] with a [warnings] section without a corresponding ["requires": "test_runner_features": "warnings"] \ so runners that do not support the [warnings] section can skip the test at line [%d]\ """, section.getLocation().lineNumber())); @@ -187,10 +187,10 @@ public class ClientYamlTestSuite { .filter(section -> section instanceof DoSection) .map(section -> (DoSection) section) .filter(section -> false == section.getExpectedWarningHeadersRegex().isEmpty()) - .filter(section -> false == hasSkipFeature("warnings_regex", testSection, setupSection, teardownSection)) + .filter(section -> false == hasYamlRunnerFeature("warnings_regex", testSection, setupSection, teardownSection)) .map(section -> String.format(Locale.ROOT, """ attempted to add a [do] with a [warnings_regex] section without a corresponding \ - ["skip": "features": "warnings_regex"] so runners that do not support the [warnings_regex] \ + ["requires": "test_runner_features": "warnings_regex"] so runners that do not support the [warnings_regex] \ section can skip the test at line [%d]\ """, section.getLocation().lineNumber())) ); @@ -201,10 +201,10 @@ public class ClientYamlTestSuite { .filter(section -> section instanceof DoSection) .map(section -> (DoSection) section) .filter(section -> false == section.getAllowedWarningHeaders().isEmpty()) - .filter(section -> false == hasSkipFeature("allowed_warnings", testSection, setupSection, teardownSection)) + .filter(section -> false == hasYamlRunnerFeature("allowed_warnings", testSection, setupSection, teardownSection)) .map(section -> String.format(Locale.ROOT, """ attempted to add a [do] with a [allowed_warnings] section without a corresponding \ - ["skip": "features": "allowed_warnings"] so runners that do not support the [allowed_warnings] \ + ["requires": "test_runner_features": "allowed_warnings"] so runners that do not support the [allowed_warnings] \ section can skip the test at line [%d]\ """, section.getLocation().lineNumber())) ); @@ -215,11 +215,11 @@ public class ClientYamlTestSuite { .filter(section -> section instanceof DoSection) .map(section -> (DoSection) section) .filter(section -> false == section.getAllowedWarningHeadersRegex().isEmpty()) - .filter(section -> false == hasSkipFeature("allowed_warnings_regex", testSection, setupSection, teardownSection)) + .filter(section -> false == hasYamlRunnerFeature("allowed_warnings_regex", testSection, setupSection, teardownSection)) .map(section -> String.format(Locale.ROOT, """ attempted to add a [do] with a [allowed_warnings_regex] section without a corresponding \ - ["skip": "features": "allowed_warnings_regex"] so runners that do not support the [allowed_warnings_regex] \ - section can skip the test at line [%d]\ + ["requires": "test_runner_features": "allowed_warnings_regex"] so runners that do not support the \ + [allowed_warnings_regex] section can skip the test at line [%d]\ """, section.getLocation().lineNumber())) ); @@ -229,10 +229,10 @@ public class ClientYamlTestSuite { .filter(section -> section instanceof DoSection) .map(section -> (DoSection) section) .filter(section -> NodeSelector.ANY != section.getApiCallSection().getNodeSelector()) - .filter(section -> false == hasSkipFeature("node_selector", testSection, setupSection, teardownSection)) + .filter(section -> false == hasYamlRunnerFeature("node_selector", testSection, setupSection, teardownSection)) .map(section -> String.format(Locale.ROOT, """ attempted to add a [do] with a [node_selector] section without a corresponding \ - ["skip": "features": "node_selector"] so runners that do not support the [node_selector] section \ + ["requires": "test_runner_features": "node_selector"] so runners that do not support the [node_selector] section \ can skip the test at line [%d]\ """, section.getLocation().lineNumber())) ); @@ -241,9 +241,9 @@ public class ClientYamlTestSuite { errors, sections.stream() .filter(section -> section instanceof ContainsAssertion) - .filter(section -> false == hasSkipFeature("contains", testSection, setupSection, teardownSection)) + .filter(section -> false == hasYamlRunnerFeature("contains", testSection, setupSection, teardownSection)) .map(section -> String.format(Locale.ROOT, """ - attempted to add a [contains] assertion without a corresponding ["skip": "features": "contains"] \ + attempted to add a [contains] assertion without a corresponding ["requires": "test_runner_features": "contains"] \ so runners that do not support the [contains] assertion can skip the test at line [%d]\ """, section.getLocation().lineNumber())) ); @@ -254,10 +254,11 @@ public class ClientYamlTestSuite { .filter(section -> section instanceof DoSection) .map(section -> (DoSection) section) .filter(section -> false == section.getApiCallSection().getHeaders().isEmpty()) - .filter(section -> false == hasSkipFeature("headers", testSection, setupSection, teardownSection)) + .filter(section -> false == hasYamlRunnerFeature("headers", testSection, setupSection, teardownSection)) .map(section -> String.format(Locale.ROOT, """ - attempted to add a [do] with a [headers] section without a corresponding ["skip": "features": "headers"] \ - so runners that do not support the [headers] section can skip the test at line [%d]\ + attempted to add a [do] with a [headers] section without a corresponding \ + ["requires": "test_runner_features": "headers"] so runners that do not support the [headers] section \ + can skip the test at line [%d]\ """, section.getLocation().lineNumber())) ); @@ -265,9 +266,9 @@ public class ClientYamlTestSuite { errors, sections.stream() .filter(section -> section instanceof CloseToAssertion) - .filter(section -> false == hasSkipFeature("close_to", testSection, setupSection, teardownSection)) + .filter(section -> false == hasYamlRunnerFeature("close_to", testSection, setupSection, teardownSection)) .map(section -> String.format(Locale.ROOT, """ - attempted to add a [close_to] assertion without a corresponding ["skip": "features": "close_to"] \ + attempted to add a [close_to] assertion without a corresponding ["requires": "test_runner_features": "close_to"] \ so runners that do not support the [close_to] assertion can skip the test at line [%d]\ """, section.getLocation().lineNumber())) ); @@ -276,9 +277,9 @@ public class ClientYamlTestSuite { errors, sections.stream() .filter(section -> section instanceof IsAfterAssertion) - .filter(section -> false == hasSkipFeature("is_after", testSection, setupSection, teardownSection)) + .filter(section -> false == hasYamlRunnerFeature("is_after", testSection, setupSection, teardownSection)) .map(section -> String.format(Locale.ROOT, """ - attempted to add an [is_after] assertion without a corresponding ["skip": "features": "is_after"] \ + attempted to add an [is_after] assertion without a corresponding ["requires": "test_runner_features": "is_after"] \ so runners that do not support the [is_after] assertion can skip the test at line [%d]\ """, section.getLocation().lineNumber())) ); @@ -286,19 +287,19 @@ public class ClientYamlTestSuite { return errors; } - private static boolean hasSkipFeature( + private static boolean hasYamlRunnerFeature( String feature, ClientYamlTestSection testSection, SetupSection setupSection, TeardownSection teardownSection ) { - return (testSection != null && hasSkipFeature(feature, testSection.getSkipSection())) - || (setupSection != null && hasSkipFeature(feature, setupSection.getSkipSection())) - || (teardownSection != null && hasSkipFeature(feature, teardownSection.getSkipSection())); + return (testSection != null && hasYamlRunnerFeature(feature, testSection.getPrerequisiteSection())) + || (setupSection != null && hasYamlRunnerFeature(feature, setupSection.getPrerequisiteSection())) + || (teardownSection != null && hasYamlRunnerFeature(feature, teardownSection.getPrerequisiteSection())); } - private static boolean hasSkipFeature(String feature, SkipSection skipSection) { - return skipSection != null && skipSection.yamlRunnerHasFeature(feature); + private static boolean hasYamlRunnerFeature(String feature, PrerequisiteSection prerequisiteSection) { + return prerequisiteSection != null && prerequisiteSection.hasYamlRunnerFeature(feature); } public List getTestSections() { diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java new file mode 100644 index 000000000000..0c11c02110ca --- /dev/null +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java @@ -0,0 +1,413 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.test.rest.yaml.section; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.test.rest.yaml.Features; +import org.elasticsearch.xcontent.XContentLocation; +import org.elasticsearch.xcontent.XContentParser; +import org.junit.AssumptionViolatedException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; + +/** + * Represents a section where prerequisites to run a specific test section or suite are specified. It is possible to specify preconditions + * as a set of `skip` criteria (the test or suite will be skipped if the specified conditions are met) or `requires` criteria (the test or + * suite will be run only if the specified conditions are met) + * Criteria are based on: + * - the elasticsearch cluster version the tests are running against (deprecated) + * - the features supported by the elasticsearch cluster version the tests are running against + * - a specific test runner feature - some runners may not implement the whole set of features + * - an operating system (full name, including specific Linux distributions) - some OS might show a certain behavior + */ +public class PrerequisiteSection { + + private static final Logger logger = LogManager.getLogger(PrerequisiteSection.class); + + static class PrerequisiteSectionBuilder { + String skipVersionRange = null; + String skipReason = null; + String requiresReason = null; + List requiredYamlRunnerFeatures = new ArrayList<>(); + List skipOperatingSystems = new ArrayList<>(); + + Set skipClusterFeatures = new HashSet<>(); + Set requiredClusterFeatures = new HashSet<>(); + + enum XPackRequired { + NOT_SPECIFIED, + YES, + NO, + MISMATCHED + } + + XPackRequired xpackRequired = XPackRequired.NOT_SPECIFIED; + + public PrerequisiteSectionBuilder skipIfVersion(String skipVersionRange) { + this.skipVersionRange = skipVersionRange; + return this; + } + + public PrerequisiteSectionBuilder setSkipReason(String skipReason) { + this.skipReason = skipReason; + return this; + } + + public PrerequisiteSectionBuilder setRequiresReason(String requiresReason) { + this.requiresReason = requiresReason; + return this; + } + + public PrerequisiteSectionBuilder requireYamlRunnerFeature(String featureName) { + requiredYamlRunnerFeatures.add(featureName); + return this; + } + + public PrerequisiteSectionBuilder requireXPack() { + if (xpackRequired == XPackRequired.NO) { + xpackRequired = XPackRequired.MISMATCHED; + } else { + xpackRequired = XPackRequired.YES; + } + return this; + } + + public PrerequisiteSectionBuilder skipIfXPack() { + if (xpackRequired == XPackRequired.YES) { + xpackRequired = XPackRequired.MISMATCHED; + } else { + xpackRequired = XPackRequired.NO; + } + return this; + } + + public PrerequisiteSectionBuilder skipIfClusterFeature(String featureName) { + skipClusterFeatures.add(featureName); + return this; + } + + public PrerequisiteSectionBuilder requireClusterFeature(String featureName) { + requiredClusterFeatures.add(featureName); + return this; + } + + public PrerequisiteSectionBuilder skipIfOs(String osName) { + this.skipOperatingSystems.add(osName); + return this; + } + + void validate(XContentLocation contentLocation) { + if ((Strings.hasLength(skipVersionRange) == false) + && requiredYamlRunnerFeatures.isEmpty() + && skipOperatingSystems.isEmpty() + && xpackRequired == XPackRequired.NOT_SPECIFIED + && requiredClusterFeatures.isEmpty() + && skipClusterFeatures.isEmpty()) { + throw new ParsingException( + contentLocation, + "at least one criteria (version, cluster features, runner features, os) is mandatory within a skip section" + ); + } + if (Strings.hasLength(skipVersionRange) && Strings.hasLength(skipReason) == false) { + throw new ParsingException(contentLocation, "reason is mandatory within skip version section"); + } + if (skipOperatingSystems.isEmpty() == false && Strings.hasLength(skipReason) == false) { + throw new ParsingException(contentLocation, "reason is mandatory within skip os section"); + } + if (skipClusterFeatures.isEmpty() == false && Strings.hasLength(skipReason) == false) { + throw new ParsingException(contentLocation, "reason is mandatory within skip cluster_features section"); + } + if (requiredClusterFeatures.isEmpty() == false && Strings.hasLength(requiresReason) == false) { + throw new ParsingException(contentLocation, "reason is mandatory within requires cluster_features section"); + } + // make feature "skip_os" mandatory if os is given, this is a temporary solution until language client tests know about os + if (skipOperatingSystems.isEmpty() == false && requiredYamlRunnerFeatures.contains("skip_os") == false) { + throw new ParsingException(contentLocation, "if os is specified, test runner feature [skip_os] must be set"); + } + if (xpackRequired == XPackRequired.MISMATCHED) { + throw new ParsingException(contentLocation, "either [xpack] or [no_xpack] can be present, not both"); + } + if (Sets.haveNonEmptyIntersection(skipClusterFeatures, requiredClusterFeatures)) { + throw new ParsingException(contentLocation, "a cluster feature can be specified either in [requires] or [skip], not both"); + } + } + + public PrerequisiteSection build() { + final List> skipCriteriaList = new ArrayList<>(); + final List> requiresCriteriaList; + + // Check if the test runner supports all YAML framework features (see {@link Features}). If not, default to always skip this + // section. + if (Features.areAllSupported(requiredYamlRunnerFeatures) == false) { + requiresCriteriaList = List.of(Prerequisites.FALSE); + } else { + requiresCriteriaList = new ArrayList<>(); + if (xpackRequired == XPackRequired.YES) { + requiresCriteriaList.add(Prerequisites.hasXPack()); + } + if (xpackRequired == XPackRequired.NO) { + skipCriteriaList.add(Prerequisites.hasXPack()); + } + if (Strings.hasLength(skipVersionRange)) { + skipCriteriaList.add(Prerequisites.skipOnVersionRange(skipVersionRange)); + } + if (skipOperatingSystems.isEmpty() == false) { + skipCriteriaList.add(Prerequisites.skipOnOsList(skipOperatingSystems)); + } + if (requiredClusterFeatures.isEmpty() == false) { + requiresCriteriaList.add(Prerequisites.requireClusterFeatures(requiredClusterFeatures)); + } + if (skipClusterFeatures.isEmpty() == false) { + skipCriteriaList.add(Prerequisites.skipOnClusterFeatures(skipClusterFeatures)); + } + } + return new PrerequisiteSection(skipCriteriaList, skipReason, requiresCriteriaList, requiresReason, requiredYamlRunnerFeatures); + } + } + + /** + * Parse a {@link PrerequisiteSection} if the next field is {@code skip}, otherwise returns {@link PrerequisiteSection#EMPTY}. + */ + public static PrerequisiteSection parseIfNext(XContentParser parser) throws IOException { + return parseInternal(parser).build(); + } + + private static void maybeAdvanceToNextField(XContentParser parser) throws IOException { + var token = parser.nextToken(); + if (token != null && token != XContentParser.Token.END_ARRAY) { + ParserUtils.advanceToFieldName(parser); + } + } + + static PrerequisiteSectionBuilder parseInternal(XContentParser parser) throws IOException { + PrerequisiteSectionBuilder builder = new PrerequisiteSectionBuilder(); + var hasPrerequisiteSection = false; + var unknownFieldName = false; + ParserUtils.advanceToFieldName(parser); + while (unknownFieldName == false) { + if ("skip".equals(parser.currentName())) { + parseSkipSection(parser, builder); + hasPrerequisiteSection = true; + maybeAdvanceToNextField(parser); + } else if ("requires".equals(parser.currentName())) { + parseRequiresSection(parser, builder); + hasPrerequisiteSection = true; + maybeAdvanceToNextField(parser); + } else { + unknownFieldName = true; + } + } + if (hasPrerequisiteSection) { + builder.validate(parser.getTokenLocation()); + } + return builder; + } + + private static void parseFeatureField(String feature, PrerequisiteSectionBuilder builder) { + // #31403 introduced YAML test "features" to indicate if the cluster being tested has xpack installed (`xpack`) + // or if it does *not* have xpack installed (`no_xpack`). These are not test runner features, so now that we have + // "modular" skip criteria let's separate them. Eventually, these should move to their own skip section. + if (feature.equals("xpack")) { + builder.requireXPack(); + } else if (feature.equals("no_xpack")) { + builder.skipIfXPack(); + } else { + builder.requireYamlRunnerFeature(feature); + } + } + + // package private for tests + static void parseSkipSection(XContentParser parser, PrerequisiteSectionBuilder builder) throws IOException { + if (parser.nextToken() != XContentParser.Token.START_OBJECT) { + throw new IllegalArgumentException( + "Expected [" + + XContentParser.Token.START_OBJECT + + ", found [" + + parser.currentToken() + + "], the skip section is not properly indented" + ); + } + String currentFieldName = null; + XContentParser.Token token; + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if ("version".equals(currentFieldName)) { + builder.skipIfVersion(parser.text()); + } else if ("reason".equals(currentFieldName)) { + builder.setSkipReason(parser.text()); + } else if ("features".equals(currentFieldName)) { + // TODO: legacy - remove + logger.warn( + "[\"skip\": \"features\"] is deprecated and will be removed. Replace it with " + + "[\"requires\": \"test_runner_features\"]" + ); + parseFeatureField(parser.text(), builder); + } else if ("os".equals(currentFieldName)) { + builder.skipIfOs(parser.text()); + } else if ("cluster_features".equals(currentFieldName)) { + builder.skipIfClusterFeature(parser.text()); + } else { + throw new ParsingException( + parser.getTokenLocation(), + "field " + currentFieldName + " not supported within skip section" + ); + } + } else if (token == XContentParser.Token.START_ARRAY) { + // TODO: legacy - remove + logger.warn( + "[\"skip\": \"features\"] is deprecated and will be removed. Replace it with " + + "[\"requires\": \"test_runner_features\"]" + ); + if ("features".equals(currentFieldName)) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + parseFeatureField(parser.text(), builder); + } + } else if ("os".equals(currentFieldName)) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + builder.skipIfOs(parser.text()); + } + } else if ("cluster_features".equals(currentFieldName)) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + builder.skipIfClusterFeature(parser.text()); + } + } + } + } + parser.nextToken(); + } + + static void parseRequiresSection(XContentParser parser, PrerequisiteSectionBuilder builder) throws IOException { + if (parser.nextToken() != XContentParser.Token.START_OBJECT) { + throw new IllegalArgumentException( + "Expected [" + + XContentParser.Token.START_OBJECT + + ", found [" + + parser.currentToken() + + "], the requires section is not properly indented" + ); + } + String currentFieldName = null; + XContentParser.Token token; + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if ("reason".equals(currentFieldName)) { + builder.setRequiresReason(parser.text()); + } else if ("test_runner_features".equals(currentFieldName)) { + parseFeatureField(parser.text(), builder); + } else if ("cluster_features".equals(currentFieldName)) { + builder.requireClusterFeature(parser.text()); + } else { + throw new ParsingException( + parser.getTokenLocation(), + "field " + currentFieldName + " not supported within requires section" + ); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("test_runner_features".equals(currentFieldName)) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + parseFeatureField(parser.text(), builder); + } + } else if ("cluster_features".equals(currentFieldName)) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + builder.requireClusterFeature(parser.text()); + } + } + } + } + parser.nextToken(); + } + + public static final PrerequisiteSection EMPTY = new PrerequisiteSection(); + + private final List> skipCriteriaList; + private final List> requiresCriteriaList; + private final List yamlRunnerFeatures; + final String skipReason; + final String requireReason; + + private PrerequisiteSection() { + this.skipCriteriaList = new ArrayList<>(); + this.requiresCriteriaList = new ArrayList<>(); + this.yamlRunnerFeatures = new ArrayList<>(); + this.skipReason = null; + this.requireReason = null; + } + + PrerequisiteSection( + List> skipCriteriaList, + String skipReason, + List> requiresCriteriaList, + String requireReason, + List yamlRunnerFeatures + ) { + this.skipCriteriaList = skipCriteriaList; + this.requiresCriteriaList = requiresCriteriaList; + this.yamlRunnerFeatures = yamlRunnerFeatures; + this.skipReason = skipReason; + this.requireReason = requireReason; + } + + public boolean hasYamlRunnerFeature(String feature) { + return yamlRunnerFeatures.contains(feature); + } + + boolean skipCriteriaMet(ClientYamlTestExecutionContext context) { + return skipCriteriaList.stream().anyMatch(c -> c.test(context)); + } + + boolean requiresCriteriaMet(ClientYamlTestExecutionContext context) { + return requiresCriteriaList.stream().allMatch(c -> c.test(context)); + } + + public void evaluate(ClientYamlTestExecutionContext context, String testCandidateDescription) { + if (isEmpty()) { + return; + } + + if (requiresCriteriaMet(context) == false) { + throw new AssumptionViolatedException(buildMessage(testCandidateDescription, false)); + } + + if (skipCriteriaMet(context)) { + throw new AssumptionViolatedException(buildMessage(testCandidateDescription, true)); + } + } + + boolean isEmpty() { + return skipCriteriaList.isEmpty() && requiresCriteriaList.isEmpty() && yamlRunnerFeatures.isEmpty(); + } + + String buildMessage(String description, boolean isSkip) { + StringBuilder messageBuilder = new StringBuilder(); + messageBuilder.append("[").append(description).append("] skipped,"); + var reason = isSkip ? skipReason : requireReason; + if (Strings.isNullOrEmpty(reason) == false) { + messageBuilder.append(" reason: [").append(reason).append("]"); + } + if (yamlRunnerFeatures.isEmpty() == false) { + messageBuilder.append(" unsupported features ").append(yamlRunnerFeatures); + } + return messageBuilder.toString(); + } +} diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipCriteria.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/Prerequisites.java similarity index 54% rename from test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipCriteria.java rename to test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/Prerequisites.java index c864c778a8e7..8049c227b199 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipCriteria.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/Prerequisites.java @@ -12,31 +12,36 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; import java.util.List; +import java.util.Set; import java.util.function.Predicate; -public class SkipCriteria { +public class Prerequisites { - public static final Predicate SKIP_ALWAYS = context -> true; + public static final Predicate TRUE = context -> true; + public static final Predicate FALSE = context -> false; - private SkipCriteria() {} + private Prerequisites() {} - static Predicate fromVersionRange(String versionRange) { + static Predicate skipOnVersionRange(String versionRange) { final var versionRangePredicates = VersionRange.parseVersionRanges(versionRange); assert versionRangePredicates.isEmpty() == false; return context -> versionRangePredicates.stream().anyMatch(range -> range.test(context.nodesVersions())); } - static Predicate fromOsList(List operatingSystems) { + static Predicate skipOnOsList(List operatingSystems) { return context -> operatingSystems.stream().anyMatch(osName -> osName.equals(context.os())); } - static Predicate fromClusterModules(boolean xpackRequired) { + static Predicate hasXPack() { // TODO: change ESRestTestCase.hasXPack() to be context-specific - return context -> { - if (xpackRequired) { - return ESRestTestCase.hasXPack() == false; - } - return ESRestTestCase.hasXPack(); - }; + return context -> ESRestTestCase.hasXPack(); + } + + static Predicate requireClusterFeatures(Set clusterFeatures) { + return context -> clusterFeatures.stream().allMatch(context::clusterHasFeature); + } + + static Predicate skipOnClusterFeatures(Set clusterFeatures) { + return context -> clusterFeatures.stream().anyMatch(context::clusterHasFeature); } } diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java index 351cfa8e40eb..ecf37c4b5cf6 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java @@ -36,7 +36,7 @@ public class SetupSection { } public static SetupSection parse(XContentParser parser) throws IOException { - SkipSection skipSection = SkipSection.parseIfNext(parser); + PrerequisiteSection prerequisiteSection = PrerequisiteSection.parseIfNext(parser); List executableSections = new ArrayList<>(); while (parser.currentToken() != XContentParser.Token.END_ARRAY) { ParserUtils.advanceToFieldName(parser); @@ -51,21 +51,21 @@ public class SetupSection { parser.nextToken(); } parser.nextToken(); - return new SetupSection(skipSection, executableSections); + return new SetupSection(prerequisiteSection, executableSections); } - public static final SetupSection EMPTY = new SetupSection(SkipSection.EMPTY, Collections.emptyList()); + public static final SetupSection EMPTY = new SetupSection(PrerequisiteSection.EMPTY, Collections.emptyList()); - private final SkipSection skipSection; + private final PrerequisiteSection prerequisiteSection; private final List executableSections; - public SetupSection(SkipSection skipSection, List executableSections) { - this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null"); + public SetupSection(PrerequisiteSection prerequisiteSection, List executableSections) { + this.prerequisiteSection = Objects.requireNonNull(prerequisiteSection, "skip section cannot be null"); this.executableSections = Collections.unmodifiableList(executableSections); } - public SkipSection getSkipSection() { - return skipSection; + public PrerequisiteSection getPrerequisiteSection() { + return prerequisiteSection; } public List getExecutableSections() { diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java deleted file mode 100644 index 4bd80fa4d9f1..000000000000 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.test.rest.yaml.section; - -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.Strings; -import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; -import org.elasticsearch.test.rest.yaml.Features; -import org.elasticsearch.xcontent.XContentLocation; -import org.elasticsearch.xcontent.XContentParser; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Predicate; - -/** - * Represents a skip section that tells whether a specific test section or suite needs to be skipped - * based on: - * - the elasticsearch version the tests are running against - * - a specific test feature required that might not be implemented yet by the runner - * - an operating system (full name, including specific Linux distributions) that might show a certain behavior - */ -public class SkipSection { - - static class SkipSectionBuilder { - String version = null; - String reason = null; - List testFeatures = new ArrayList<>(); - List operatingSystems = new ArrayList<>(); - - enum XPackRequested { - NOT_SPECIFIED, - YES, - NO, - MISMATCHED - } - - XPackRequested xpackRequested = XPackRequested.NOT_SPECIFIED; - - public SkipSectionBuilder withVersion(String version) { - this.version = version; - return this; - } - - public SkipSectionBuilder withReason(String reason) { - this.reason = reason; - return this; - } - - public SkipSectionBuilder withTestFeature(String featureName) { - this.testFeatures.add(featureName); - return this; - } - - public void withXPack(boolean xpackRequired) { - if (xpackRequired && xpackRequested == XPackRequested.NO || xpackRequired == false && xpackRequested == XPackRequested.YES) { - xpackRequested = XPackRequested.MISMATCHED; - } else { - xpackRequested = xpackRequired ? XPackRequested.YES : XPackRequested.NO; - } - } - - public SkipSectionBuilder withOs(String osName) { - this.operatingSystems.add(osName); - return this; - } - - void validate(XContentLocation contentLocation) { - if ((Strings.hasLength(version) == false) - && testFeatures.isEmpty() - && operatingSystems.isEmpty() - && xpackRequested == XPackRequested.NOT_SPECIFIED) { - throw new ParsingException( - contentLocation, - "at least one criteria (version, test features, os) is mandatory within a skip section" - ); - } - if (Strings.hasLength(version) && Strings.hasLength(reason) == false) { - throw new ParsingException(contentLocation, "reason is mandatory within skip version section"); - } - if (operatingSystems.isEmpty() == false && Strings.hasLength(reason) == false) { - throw new ParsingException(contentLocation, "reason is mandatory within skip version section"); - } - // make feature "skip_os" mandatory if os is given, this is a temporary solution until language client tests know about os - if (operatingSystems.isEmpty() == false && testFeatures.contains("skip_os") == false) { - throw new ParsingException(contentLocation, "if os is specified, feature skip_os must be set"); - } - if (xpackRequested == XPackRequested.MISMATCHED) { - throw new ParsingException(contentLocation, "either `xpack` or `no_xpack` can be present, not both"); - } - } - - public SkipSection build() { - final List> skipCriteriaList; - - // Check if the test runner supports all YAML framework features (see {@link Features}). If not, default to always skip this - // section. - if (Features.areAllSupported(testFeatures) == false) { - skipCriteriaList = List.of(SkipCriteria.SKIP_ALWAYS); - } else { - skipCriteriaList = new ArrayList<>(); - if (xpackRequested == XPackRequested.YES || xpackRequested == XPackRequested.NO) { - skipCriteriaList.add(SkipCriteria.fromClusterModules(xpackRequested == XPackRequested.YES)); - } - if (Strings.hasLength(version)) { - skipCriteriaList.add(SkipCriteria.fromVersionRange(version)); - } - if (operatingSystems.isEmpty() == false) { - skipCriteriaList.add(SkipCriteria.fromOsList(operatingSystems)); - } - } - return new SkipSection(skipCriteriaList, testFeatures, reason); - } - } - - /** - * Parse a {@link SkipSection} if the next field is {@code skip}, otherwise returns {@link SkipSection#EMPTY}. - */ - public static SkipSection parseIfNext(XContentParser parser) throws IOException { - ParserUtils.advanceToFieldName(parser); - - if ("skip".equals(parser.currentName())) { - SkipSection section = parse(parser); - parser.nextToken(); - return section; - } - - return EMPTY; - } - - public static SkipSection parse(XContentParser parser) throws IOException { - return parseInternal(parser).build(); - } - - private static void parseFeature(String feature, SkipSectionBuilder builder) { - // #31403 introduced YAML test "features" to indicate if the cluster being tested has xpack installed (`xpack`) - // or if it does *not* have xpack installed (`no_xpack`). These are not test runner features, so now that we have - // "modular" skip criteria let's separate them. Eventually, these should move to their own skip section. - if (feature.equals("xpack")) { - builder.withXPack(true); - } else if (feature.equals("no_xpack")) { - builder.withXPack(false); - } else { - builder.withTestFeature(feature); - } - } - - // package private for tests - static SkipSectionBuilder parseInternal(XContentParser parser) throws IOException { - if (parser.nextToken() != XContentParser.Token.START_OBJECT) { - throw new IllegalArgumentException( - "Expected [" - + XContentParser.Token.START_OBJECT - + ", found [" - + parser.currentToken() - + "], the skip section is not properly indented" - ); - } - String currentFieldName = null; - XContentParser.Token token; - - var builder = new SkipSectionBuilder(); - - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token.isValue()) { - if ("version".equals(currentFieldName)) { - builder.withVersion(parser.text()); - } else if ("reason".equals(currentFieldName)) { - builder.withReason(parser.text()); - } else if ("features".equals(currentFieldName)) { - parseFeature(parser.text(), builder); - } else if ("os".equals(currentFieldName)) { - builder.withOs(parser.text()); - } else { - throw new ParsingException( - parser.getTokenLocation(), - "field " + currentFieldName + " not supported within skip section" - ); - } - } else if (token == XContentParser.Token.START_ARRAY) { - if ("features".equals(currentFieldName)) { - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - parseFeature(parser.text(), builder); - } - } else if ("os".equals(currentFieldName)) { - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - builder.withOs(parser.text()); - } - } - } - } - - parser.nextToken(); - builder.validate(parser.getTokenLocation()); - return builder; - } - - public static final SkipSection EMPTY = new SkipSection(); - - private final List> skipCriteriaList; - private final List yamlRunnerFeatures; - private final String reason; - - private SkipSection() { - this.skipCriteriaList = new ArrayList<>(); - this.yamlRunnerFeatures = new ArrayList<>(); - this.reason = null; - } - - SkipSection(List> skipCriteriaList, List yamlRunnerFeatures, String reason) { - this.skipCriteriaList = skipCriteriaList; - this.yamlRunnerFeatures = yamlRunnerFeatures; - this.reason = reason; - } - - public boolean yamlRunnerHasFeature(String feature) { - return yamlRunnerFeatures.contains(feature); - } - - public String getReason() { - return reason; - } - - public boolean skip(ClientYamlTestExecutionContext context) { - if (isEmpty()) { - return false; - } - - return skipCriteriaList.stream().anyMatch(c -> c.test(context)); - } - - public boolean isEmpty() { - return EMPTY.equals(this); - } - - public String getSkipMessage(String description) { - StringBuilder messageBuilder = new StringBuilder(); - messageBuilder.append("[").append(description).append("] skipped,"); - if (reason != null) { - messageBuilder.append(" reason: [").append(getReason()).append("]"); - } - if (yamlRunnerFeatures.isEmpty() == false) { - messageBuilder.append(" unsupported features ").append(yamlRunnerFeatures); - } - return messageBuilder.toString(); - } -} diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java index 682137846374..ca76ee92bb3c 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java @@ -35,7 +35,7 @@ public class TeardownSection { } public static TeardownSection parse(XContentParser parser) throws IOException { - SkipSection skipSection = SkipSection.parseIfNext(parser); + PrerequisiteSection prerequisiteSection = PrerequisiteSection.parseIfNext(parser); List executableSections = new ArrayList<>(); while (parser.currentToken() != XContentParser.Token.END_ARRAY) { ParserUtils.advanceToFieldName(parser); @@ -50,21 +50,21 @@ public class TeardownSection { } parser.nextToken(); - return new TeardownSection(skipSection, executableSections); + return new TeardownSection(prerequisiteSection, executableSections); } - public static final TeardownSection EMPTY = new TeardownSection(SkipSection.EMPTY, Collections.emptyList()); + public static final TeardownSection EMPTY = new TeardownSection(PrerequisiteSection.EMPTY, Collections.emptyList()); - private final SkipSection skipSection; + private final PrerequisiteSection prerequisiteSection; private final List doSections; - TeardownSection(SkipSection skipSection, List doSections) { - this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null"); + TeardownSection(PrerequisiteSection prerequisiteSection, List doSections) { + this.prerequisiteSection = Objects.requireNonNull(prerequisiteSection, "skip section cannot be null"); this.doSections = Collections.unmodifiableList(doSections); } - public SkipSection getSkipSection() { - return skipSection; + public PrerequisiteSection getPrerequisiteSection() { + return prerequisiteSection; } public List getDoSections() { diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java index 0ee275fc89c1..2c6e7e30e0d4 100644 --- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java +++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java @@ -69,7 +69,7 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa assertThat(testSection, notNullValue()); assertThat(testSection.getName(), equalTo("First test section")); - assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY)); + assertThat(testSection.getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(testSection.getExecutableSections().size(), equalTo(1)); DoSection doSection = (DoSection) testSection.getExecutableSections().get(0); assertThat(doSection.getCatch(), equalTo("missing")); @@ -96,8 +96,8 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa assertThat(testSection, notNullValue()); assertThat(testSection.getName(), equalTo("First test section")); - assertThat(testSection.getSkipSection(), notNullValue()); - assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259")); + assertThat(testSection.getPrerequisiteSection(), notNullValue()); + assertThat(testSection.getPrerequisiteSection().skipReason, equalTo("Update doesn't return metadata fields, waiting for #3259")); assertThat(testSection.getExecutableSections().size(), equalTo(2)); DoSection doSection = (DoSection) testSection.getExecutableSections().get(0); assertThat(doSection.getCatch(), equalTo("missing")); @@ -130,7 +130,7 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa assertThat(testSection, notNullValue()); assertThat(testSection.getName(), equalTo("Basic")); - assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY)); + assertThat(testSection.getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(testSection.getExecutableSections().size(), equalTo(2)); DoSection doSection = (DoSection) testSection.getExecutableSections().get(0); assertThat(doSection.getCatch(), nullValue()); @@ -181,7 +181,7 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa assertThat(testSection, notNullValue()); assertThat(testSection.getName(), equalTo("Basic")); - assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY)); + assertThat(testSection.getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(testSection.getExecutableSections().size(), equalTo(10)); DoSection doSection = (DoSection) testSection.getExecutableSections().get(0); diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java index c64a30378e9d..1f5bdc71dde3 100644 --- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java +++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java @@ -34,6 +34,51 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentParserTestCase { + + public void testParseTestSetupWithSkip() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + --- + setup: + - skip: + version: "8.7.00 - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + + --- + date: + - skip: + version: " - 8.1.99" + reason: tsdb indexing changed in 8.2.0 + - do: + indices.get_mapping: + index: test_index + + - match: {test_index.test_type.properties.text.type: string} + - match: {test_index.test_type.properties.text.analyzer: whitespace} + """); + + ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), Optional.empty(), parser); + + assertThat(restTestSuite, notNullValue()); + assertThat(restTestSuite.getName(), equalTo(getTestName())); + assertThat(restTestSuite.getFile().isPresent(), equalTo(false)); + assertThat(restTestSuite.getSetupSection(), notNullValue()); + + assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false)); + assertThat(restTestSuite.getSetupSection().getPrerequisiteSection().isEmpty(), equalTo(false)); + assertThat(restTestSuite.getSetupSection().getExecutableSections().isEmpty(), equalTo(true)); + + assertThat(restTestSuite.getTestSections().size(), equalTo(1)); + + assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("date")); + assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(false)); + assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(3)); + assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class)); + DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0); + assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_mapping")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(1)); + assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index")); + } + public void testParseTestSetupTeardownAndSections() throws Exception { final boolean includeSetup = randomBoolean(); final boolean includeTeardown = randomBoolean(); @@ -92,7 +137,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars assertThat(restTestSuite.getSetupSection(), notNullValue()); if (includeSetup) { assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false)); - assertThat(restTestSuite.getSetupSection().getSkipSection().isEmpty(), equalTo(true)); + assertThat(restTestSuite.getSetupSection().getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(restTestSuite.getSetupSection().getExecutableSections().size(), equalTo(1)); final ExecutableSection maybeDoSection = restTestSuite.getSetupSection().getExecutableSections().get(0); assertThat(maybeDoSection, instanceOf(DoSection.class)); @@ -107,7 +152,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars assertThat(restTestSuite.getTeardownSection(), notNullValue()); if (includeTeardown) { assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false)); - assertThat(restTestSuite.getTeardownSection().getSkipSection().isEmpty(), equalTo(true)); + assertThat(restTestSuite.getTeardownSection().getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(restTestSuite.getTeardownSection().getDoSections().size(), equalTo(1)); assertThat( ((DoSection) restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getApi(), @@ -128,7 +173,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars assertThat(restTestSuite.getTestSections().size(), equalTo(2)); assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Get index mapping")); - assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true)); + assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(3)); assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class)); DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0); @@ -145,9 +190,9 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars assertThat(matchAssertion.getExpectedValue().toString(), equalTo("whitespace")); assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Get type mapping - pre 6.0")); - assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(false)); + assertThat(restTestSuite.getTestSections().get(1).getPrerequisiteSection().isEmpty(), equalTo(false)); assertThat( - restTestSuite.getTestSections().get(1).getSkipSection().getReason(), + restTestSuite.getTestSections().get(1).getPrerequisiteSection().skipReason, equalTo("for newer versions the index name is always returned") ); @@ -209,7 +254,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars assertThat(restTestSuite.getTestSections().size(), equalTo(1)); assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Index with ID")); - assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true)); + assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(12)); assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class)); DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0); @@ -322,7 +367,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars assertThat(restTestSuite.getTestSections().size(), equalTo(2)); assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Missing document (partial doc)")); - assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true)); + assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(2)); assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class)); @@ -339,7 +384,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars assertThat(doSection.getApiCallSection().hasBody(), equalTo(true)); assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Missing document (script)")); - assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(true)); + assertThat(restTestSuite.getTestSections().get(1).getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().size(), equalTo(2)); assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(0), instanceOf(DoSection.class)); assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(1), instanceOf(DoSection.class)); @@ -418,9 +463,44 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars assertThat(restTestSuite.getTestSections().size(), equalTo(1)); assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Broken on some os")); - assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(false)); - assertThat(restTestSuite.getTestSections().get(0).getSkipSection().getReason(), equalTo("not supported")); - assertThat(restTestSuite.getTestSections().get(0).getSkipSection().yamlRunnerHasFeature("skip_os"), equalTo(true)); + assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(false)); + assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().skipReason, containsString("not supported")); + assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().hasYamlRunnerFeature("skip_os"), equalTo(true)); + } + + public void testParseSkipAndRequireClusterFeatures() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + "Broken on some os": + + - skip: + cluster_features: [unsupported-feature1, unsupported-feature2] + reason: "unsupported-features are not supported" + - requires: + cluster_features: required-feature1 + reason: "required-feature1 is required" + - do: + indices.get_mapping: + index: test_index + type: test_type + + - match: {test_type.properties.text.type: string} + - match: {test_type.properties.text.analyzer: whitespace} + """); + + ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), Optional.empty(), parser); + + assertThat(restTestSuite, notNullValue()); + assertThat(restTestSuite.getName(), equalTo(getTestName())); + assertThat(restTestSuite.getFile().isPresent(), equalTo(false)); + assertThat(restTestSuite.getTestSections().size(), equalTo(1)); + + assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Broken on some os")); + assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(false)); + assertThat( + restTestSuite.getTestSections().get(0).getPrerequisiteSection().skipReason, + equalTo("unsupported-features are not supported") + ); + assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().requireReason, equalTo("required-feature1 is required")); } public void testParseFileWithSingleTestSection() throws Exception { @@ -453,7 +533,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars assertThat(restTestSuite.getTestSections().size(), equalTo(1)); assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Index with ID")); - assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true)); + assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(2)); assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class)); DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0); @@ -473,7 +553,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars ClientYamlTestSection section = new ClientYamlTestSection( new XContentLocation(0, 0), "test", - SkipSection.EMPTY, + PrerequisiteSection.EMPTY, Collections.singletonList(doSection) ); ClientYamlTestSuite clientYamlTestSuite = new ClientYamlTestSuite( @@ -492,11 +572,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); doSection.setExpectedWarningHeaders(singletonList("foo")); doSection.setApiCallSection(new ApiCallSection("test")); - ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection); + ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection); Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate); assertThat(e.getMessage(), containsString(Strings.format(""" api/name: - attempted to add a [do] with a [warnings] section without a corresponding ["skip": "features": "warnings"] \ + attempted to add a [do] with a [warnings] section without a corresponding ["requires": "test_runner_features": "warnings"] \ so runners that do not support the [warnings] section can skip the test at line [%d]\ """, lineNumber))); } @@ -506,11 +586,12 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); doSection.setExpectedWarningHeadersRegex(singletonList(Pattern.compile("foo"))); doSection.setApiCallSection(new ApiCallSection("test")); - ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection); + ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection); Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate); assertThat(e.getMessage(), containsString(Strings.format(""" api/name: - attempted to add a [do] with a [warnings_regex] section without a corresponding ["skip": "features": "warnings_regex"] \ + attempted to add a [do] with a [warnings_regex] section without a corresponding \ + ["requires": "test_runner_features": "warnings_regex"] \ so runners that do not support the [warnings_regex] section can skip the test at line [%d]\ """, lineNumber))); } @@ -520,11 +601,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); doSection.setAllowedWarningHeaders(singletonList("foo")); doSection.setApiCallSection(new ApiCallSection("test")); - ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection); + ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection); Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate); assertThat(e.getMessage(), containsString(Strings.format(""" api/name: - attempted to add a [do] with a [allowed_warnings] section without a corresponding ["skip": "features": \ + attempted to add a [do] with a [allowed_warnings] section without a corresponding ["requires": "test_runner_features": \ "allowed_warnings"] so runners that do not support the [allowed_warnings] section can skip the test at \ line [%d]\ """, lineNumber))); @@ -535,11 +616,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); doSection.setAllowedWarningHeadersRegex(singletonList(Pattern.compile("foo"))); doSection.setApiCallSection(new ApiCallSection("test")); - ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection); + ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection); Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate); assertThat(e.getMessage(), containsString(Strings.format(""" api/name: - attempted to add a [do] with a [allowed_warnings_regex] section without a corresponding ["skip": "features": \ + attempted to add a [do] with a [allowed_warnings_regex] section without a corresponding ["requires": "test_runner_features": \ "allowed_warnings_regex"] so runners that do not support the [allowed_warnings_regex] section can skip the test \ at line [%d]\ """, lineNumber))); @@ -551,11 +632,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars ApiCallSection apiCallSection = new ApiCallSection("test"); apiCallSection.addHeaders(Collections.singletonMap("header", "value")); doSection.setApiCallSection(apiCallSection); - ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection); + ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection); Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate); assertThat(e.getMessage(), containsString(Strings.format(""" api/name: - attempted to add a [do] with a [headers] section without a corresponding ["skip": "features": "headers"] \ + attempted to add a [do] with a [headers] section without a corresponding ["requires": "test_runner_features": "headers"] \ so runners that do not support the [headers] section can skip the test at line [%d]\ """, lineNumber))); } @@ -566,11 +647,12 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars ApiCallSection apiCall = new ApiCallSection("test"); apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS); doSection.setApiCallSection(apiCall); - ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection); + ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection); Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate); assertThat(e.getMessage(), containsString(Strings.format(""" api/name: - attempted to add a [do] with a [node_selector] section without a corresponding ["skip": "features": "node_selector"] \ + attempted to add a [do] with a [node_selector] section without a corresponding \ + ["requires": "test_runner_features": "node_selector"] \ so runners that do not support the [node_selector] section can skip the test at line [%d]\ """, lineNumber))); } @@ -582,11 +664,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars randomAlphaOfLength(randomIntBetween(3, 30)), randomDouble() ); - ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, containsAssertion); + ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, containsAssertion); Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate); assertThat(e.getMessage(), containsString(Strings.format(""" api/name: - attempted to add a [contains] assertion without a corresponding ["skip": "features": "contains"] \ + attempted to add a [contains] assertion without a corresponding ["requires": "test_runner_features": "contains"] \ so runners that do not support the [contains] assertion can skip the test at line [%d]\ """, lineNumber))); } @@ -604,7 +686,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars new ClientYamlTestSection( new XContentLocation(0, 0), "section1", - SkipSection.EMPTY, + PrerequisiteSection.EMPTY, Collections.singletonList(containsAssertion) ) ); @@ -625,7 +707,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars doSection.setApiCallSection(apiCall); doSections.add(doSection); } - sections.add(new ClientYamlTestSection(new XContentLocation(0, 0), "section2", SkipSection.EMPTY, doSections)); + sections.add(new ClientYamlTestSection(new XContentLocation(0, 0), "section2", PrerequisiteSection.EMPTY, doSections)); ClientYamlTestSuite testSuite = new ClientYamlTestSuite( "api", @@ -638,23 +720,29 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate); assertEquals(Strings.format(""" api/name: - attempted to add a [contains] assertion without a corresponding ["skip": "features": "contains"] so runners that \ - do not support the [contains] assertion can skip the test at line [%d], - attempted to add a [do] with a [warnings] section without a corresponding ["skip": "features": "warnings"] so runners \ - that do not support the [warnings] section can skip the test at line [%d], - attempted to add a [do] with a [node_selector] section without a corresponding ["skip": "features": "node_selector"] so \ - runners that do not support the [node_selector] section can skip the test \ - at line [%d]\ + attempted to add a [contains] assertion without a corresponding \ + ["requires": "test_runner_features": "contains"] \ + so runners that do not support the [contains] assertion can skip the test at line [%d], + attempted to add a [do] with a [warnings] section without a corresponding \ + ["requires": "test_runner_features": "warnings"] \ + so runners that do not support the [warnings] section can skip the test at line [%d], + attempted to add a [do] with a [node_selector] section without a corresponding \ + ["requires": "test_runner_features": "node_selector"] \ + so runners that do not support the [node_selector] section can skip the test at line [%d]\ """, firstLineNumber, secondLineNumber, thirdLineNumber), e.getMessage()); } + private static PrerequisiteSection createPrerequisiteSection(String yamlTestRunnerFeature) { + return new PrerequisiteSection(emptyList(), null, emptyList(), null, singletonList(yamlTestRunnerFeature)); + } + public void testAddingDoWithWarningWithSkip() { int lineNumber = between(1, 10000); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); doSection.setExpectedWarningHeaders(singletonList("foo")); doSection.setApiCallSection(new ApiCallSection("test")); - SkipSection skipSection = new SkipSection(emptyList(), singletonList("warnings"), null); - createTestSuite(skipSection, doSection).validate(); + PrerequisiteSection prerequisiteSection = createPrerequisiteSection("warnings"); + createTestSuite(prerequisiteSection, doSection).validate(); } public void testAddingDoWithWarningRegexWithSkip() { @@ -662,86 +750,86 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); doSection.setExpectedWarningHeadersRegex(singletonList(Pattern.compile("foo"))); doSection.setApiCallSection(new ApiCallSection("test")); - SkipSection skipSection = new SkipSection(emptyList(), singletonList("warnings_regex"), null); - createTestSuite(skipSection, doSection).validate(); + PrerequisiteSection prerequisiteSection = createPrerequisiteSection("warnings_regex"); + createTestSuite(prerequisiteSection, doSection).validate(); } public void testAddingDoWithNodeSelectorWithSkip() { int lineNumber = between(1, 10000); - SkipSection skipSection = new SkipSection(emptyList(), singletonList("node_selector"), null); + PrerequisiteSection prerequisiteSection = createPrerequisiteSection("node_selector"); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); ApiCallSection apiCall = new ApiCallSection("test"); apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS); doSection.setApiCallSection(apiCall); - createTestSuite(skipSection, doSection).validate(); + createTestSuite(prerequisiteSection, doSection).validate(); } public void testAddingDoWithHeadersWithSkip() { int lineNumber = between(1, 10000); - SkipSection skipSection = new SkipSection(emptyList(), singletonList("headers"), null); + PrerequisiteSection prerequisiteSection = createPrerequisiteSection("headers"); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); ApiCallSection apiCallSection = new ApiCallSection("test"); apiCallSection.addHeaders(singletonMap("foo", "bar")); doSection.setApiCallSection(apiCallSection); - createTestSuite(skipSection, doSection).validate(); + createTestSuite(prerequisiteSection, doSection).validate(); } public void testAddingContainsWithSkip() { int lineNumber = between(1, 10000); - SkipSection skipSection = new SkipSection(emptyList(), singletonList("contains"), null); + PrerequisiteSection prerequisiteSection = createPrerequisiteSection("contains"); ContainsAssertion containsAssertion = new ContainsAssertion( new XContentLocation(lineNumber, 0), randomAlphaOfLength(randomIntBetween(3, 30)), randomDouble() ); - createTestSuite(skipSection, containsAssertion).validate(); + createTestSuite(prerequisiteSection, containsAssertion).validate(); } public void testAddingCloseToWithSkip() { int lineNumber = between(1, 10000); - SkipSection skipSection = new SkipSection(emptyList(), singletonList("close_to"), null); + PrerequisiteSection prerequisiteSection = createPrerequisiteSection("close_to"); CloseToAssertion closeToAssertion = new CloseToAssertion( new XContentLocation(lineNumber, 0), randomAlphaOfLength(randomIntBetween(3, 30)), randomDouble(), randomDouble() ); - createTestSuite(skipSection, closeToAssertion).validate(); + createTestSuite(prerequisiteSection, closeToAssertion).validate(); } public void testAddingIsAfterWithSkip() { int lineNumber = between(1, 10000); - SkipSection skipSection = new SkipSection(emptyList(), singletonList("is_after"), null); + PrerequisiteSection prerequisiteSection = createPrerequisiteSection("is_after"); IsAfterAssertion isAfterAssertion = new IsAfterAssertion( new XContentLocation(lineNumber, 0), randomAlphaOfLength(randomIntBetween(3, 30)), randomInstantBetween(Instant.ofEpochSecond(0L), Instant.ofEpochSecond(3000000000L)) ); - createTestSuite(skipSection, isAfterAssertion).validate(); + createTestSuite(prerequisiteSection, isAfterAssertion).validate(); } - private static ClientYamlTestSuite createTestSuite(SkipSection skipSection, ExecutableSection executableSection) { + private static ClientYamlTestSuite createTestSuite(PrerequisiteSection prerequisiteSection, ExecutableSection executableSection) { final SetupSection setupSection; final TeardownSection teardownSection; final ClientYamlTestSection clientYamlTestSection; switch (randomIntBetween(0, 4)) { case 0 -> { - setupSection = new SetupSection(skipSection, Collections.emptyList()); + setupSection = new SetupSection(prerequisiteSection, Collections.emptyList()); teardownSection = TeardownSection.EMPTY; clientYamlTestSection = new ClientYamlTestSection( new XContentLocation(0, 0), "test", - SkipSection.EMPTY, + PrerequisiteSection.EMPTY, Collections.singletonList(executableSection) ); } case 1 -> { setupSection = SetupSection.EMPTY; - teardownSection = new TeardownSection(skipSection, Collections.emptyList()); + teardownSection = new TeardownSection(prerequisiteSection, Collections.emptyList()); clientYamlTestSection = new ClientYamlTestSection( new XContentLocation(0, 0), "test", - SkipSection.EMPTY, + PrerequisiteSection.EMPTY, Collections.singletonList(executableSection) ); } @@ -751,27 +839,27 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars clientYamlTestSection = new ClientYamlTestSection( new XContentLocation(0, 0), "test", - skipSection, + prerequisiteSection, Collections.singletonList(executableSection) ); } case 3 -> { - setupSection = new SetupSection(skipSection, Collections.singletonList(executableSection)); + setupSection = new SetupSection(prerequisiteSection, Collections.singletonList(executableSection)); teardownSection = TeardownSection.EMPTY; clientYamlTestSection = new ClientYamlTestSection( new XContentLocation(0, 0), "test", - SkipSection.EMPTY, + PrerequisiteSection.EMPTY, randomBoolean() ? Collections.emptyList() : Collections.singletonList(executableSection) ); } case 4 -> { setupSection = SetupSection.EMPTY; - teardownSection = new TeardownSection(skipSection, Collections.singletonList(executableSection)); + teardownSection = new TeardownSection(prerequisiteSection, Collections.singletonList(executableSection)); clientYamlTestSection = new ClientYamlTestSection( new XContentLocation(0, 0), "test", - SkipSection.EMPTY, + PrerequisiteSection.EMPTY, randomBoolean() ? Collections.emptyList() : Collections.singletonList(executableSection) ); } diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSectionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSectionTests.java new file mode 100644 index 000000000000..181ec34fefb7 --- /dev/null +++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSectionTests.java @@ -0,0 +1,630 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test.rest.yaml.section; + +import org.elasticsearch.Version; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.core.Strings; +import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.yaml.YamlXContent; +import org.junit.AssumptionViolatedException; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.emptyOrNullString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PrerequisiteSectionTests extends AbstractClientYamlTestFragmentParserTestCase { + + public void testSkipVersionMultiRange() { + PrerequisiteSection section = new PrerequisiteSection( + List.of(Prerequisites.skipOnVersionRange("6.0.0 - 6.1.0, 7.1.0 - 7.5.0")), + "foobar", + emptyList(), + "foobar", + emptyList() + ); + + var outOfRangeMockContext = mock(ClientYamlTestExecutionContext.class); + when(outOfRangeMockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString())) + .thenReturn(Set.of("6.2.0")) + .thenReturn(Set.of("7.0.0")) + .thenReturn(Set.of("7.6.0")); + + assertFalse(section.skipCriteriaMet(outOfRangeMockContext)); + assertFalse(section.skipCriteriaMet(outOfRangeMockContext)); + assertFalse(section.skipCriteriaMet(outOfRangeMockContext)); + assertFalse(section.skipCriteriaMet(outOfRangeMockContext)); + + var inRangeMockContext = mock(ClientYamlTestExecutionContext.class); + when(inRangeMockContext.nodesVersions()).thenReturn(Set.of("6.0.0")) + .thenReturn(Set.of("6.1.0")) + .thenReturn(Set.of("7.1.0")) + .thenReturn(Set.of("7.5.0")); + + assertTrue(section.skipCriteriaMet(inRangeMockContext)); + assertTrue(section.skipCriteriaMet(inRangeMockContext)); + assertTrue(section.skipCriteriaMet(inRangeMockContext)); + assertTrue(section.skipCriteriaMet(inRangeMockContext)); + } + + public void testSkipVersionMultiOpenRange() { + var section = new PrerequisiteSection( + List.of(Prerequisites.skipOnVersionRange("- 7.1.0, 7.2.0 - 7.5.0, 8.0.0 -")), + "foobar", + emptyList(), + "foobar", + emptyList() + ); + + var outOfRangeMockContext = mock(ClientYamlTestExecutionContext.class); + when(outOfRangeMockContext.nodesVersions()).thenReturn(Set.of("7.1.1")).thenReturn(Set.of("7.6.0")); + + assertFalse(section.skipCriteriaMet(outOfRangeMockContext)); + assertFalse(section.skipCriteriaMet(outOfRangeMockContext)); + + var inRangeMockContext = mock(ClientYamlTestExecutionContext.class); + when(inRangeMockContext.nodesVersions()).thenReturn(Set.of("7.0.0")) + .thenReturn(Set.of("7.3.0")) + .thenReturn(Set.of("8.0.0")) + .thenReturn(Set.of(Version.CURRENT.toString())); + + assertTrue(section.skipCriteriaMet(inRangeMockContext)); + assertTrue(section.skipCriteriaMet(inRangeMockContext)); + assertTrue(section.skipCriteriaMet(inRangeMockContext)); + assertTrue(section.skipCriteriaMet(inRangeMockContext)); + } + + public void testSkipVersion() { + PrerequisiteSection section = new PrerequisiteSection( + List.of(Prerequisites.skipOnVersionRange("6.0.0 - 6.1.0")), + "foobar", + emptyList(), + "foobar", + emptyList() + ); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + when(mockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString())) + .thenReturn(Set.of("6.0.0")) + .thenReturn(Set.of("6.0.0", "6.1.0")) + .thenReturn(Set.of("6.0.0", "5.2.0")); + + assertFalse(section.skipCriteriaMet(mockContext)); + assertTrue(section.skipCriteriaMet(mockContext)); + assertTrue(section.skipCriteriaMet(mockContext)); + assertFalse(section.skipCriteriaMet(mockContext)); + } + + public void testSkipVersionWithTestFeatures() { + PrerequisiteSection section = new PrerequisiteSection( + List.of(Prerequisites.skipOnVersionRange("6.0.0 - 6.1.0")), + "foobar", + emptyList(), + "foobar", + singletonList("warnings") + ); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + when(mockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString())).thenReturn(Set.of("6.0.0")); + + assertFalse(section.skipCriteriaMet(mockContext)); + assertTrue(section.skipCriteriaMet(mockContext)); + } + + public void testSkipTestFeatures() { + var section = new PrerequisiteSection.PrerequisiteSectionBuilder().requireYamlRunnerFeature("boom").build(); + assertFalse(section.requiresCriteriaMet(mock(ClientYamlTestExecutionContext.class))); + } + + public void testSkipTestFeaturesOverridesAnySkipCriteria() { + var section = new PrerequisiteSection.PrerequisiteSectionBuilder().skipIfOs("test-os").requireYamlRunnerFeature("boom").build(); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + when(mockContext.os()).thenReturn("test-os"); + + // Skip even if OS is matching + assertFalse(section.skipCriteriaMet(mockContext)); + assertFalse(section.requiresCriteriaMet(mockContext)); + } + + public void testSkipOs() { + PrerequisiteSection section = new PrerequisiteSection.PrerequisiteSectionBuilder().skipIfOs("windows95") + .skipIfOs("debian-5") + .build(); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + + when(mockContext.os()).thenReturn("debian-5"); + assertTrue(section.skipCriteriaMet(mockContext)); + + when(mockContext.os()).thenReturn("windows95"); + assertTrue(section.skipCriteriaMet(mockContext)); + + when(mockContext.os()).thenReturn("ms-dos"); + assertFalse(section.skipCriteriaMet(mockContext)); + + assertTrue(section.requiresCriteriaMet(mockContext)); + } + + public void testSkipOsWithTestFeatures() { + PrerequisiteSection section = new PrerequisiteSection.PrerequisiteSectionBuilder().requireYamlRunnerFeature("warnings") + .skipIfOs("windows95") + .skipIfOs("debian-5") + .build(); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + when(mockContext.os()).thenReturn("debian-5"); + assertTrue(section.skipCriteriaMet(mockContext)); + + when(mockContext.os()).thenReturn("windows95"); + assertTrue(section.skipCriteriaMet(mockContext)); + + when(mockContext.os()).thenReturn("ms-dos"); + assertFalse(section.skipCriteriaMet(mockContext)); + } + + public void testBuildMessage() { + PrerequisiteSection section = new PrerequisiteSection( + List.of(Prerequisites.skipOnVersionRange("6.0.0 - 6.1.0")), + "unsupported", + emptyList(), + "required", + singletonList("warnings") + ); + assertEquals("[FOOBAR] skipped, reason: [unsupported] unsupported features [warnings]", section.buildMessage("FOOBAR", true)); + assertEquals("[FOOBAR] skipped, reason: [required] unsupported features [warnings]", section.buildMessage("FOOBAR", false)); + section = new PrerequisiteSection(emptyList(), "unsupported", emptyList(), "required", emptyList()); + assertEquals("[FOOBAR] skipped, reason: [unsupported]", section.buildMessage("FOOBAR", true)); + assertEquals("[FOOBAR] skipped, reason: [required]", section.buildMessage("FOOBAR", false)); + section = new PrerequisiteSection(emptyList(), null, emptyList(), null, singletonList("warnings")); + assertEquals("[FOOBAR] skipped, unsupported features [warnings]", section.buildMessage("FOOBAR", true)); + assertEquals("[FOOBAR] skipped, unsupported features [warnings]", section.buildMessage("FOOBAR", false)); + } + + public void testParseNoPrerequisites() throws IOException { + parser = createParser(YamlXContent.yamlXContent, """ + do: + something + """); + var skipSectionBuilder = PrerequisiteSection.parseInternal(parser); + assertThat(skipSectionBuilder, notNullValue()); + + var skipSection = skipSectionBuilder.build(); + assertThat(skipSection.isEmpty(), equalTo(true)); + + // Ensure the input (bogus execute section) was not consumed + var next = ParserUtils.parseField(parser); + assertThat(next, notNullValue()); + assertThat(parser.nextToken(), nullValue()); + } + + public void testParseSkipSectionVersionNoFeature() throws Exception { + Version version = VersionUtils.randomVersion(random()); + parser = createParser(YamlXContent.yamlXContent, Strings.format(""" + version: " - %s" + reason: Delete ignores the parent param""", version)); + + var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder(); + PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, not(emptyOrNullString())); + assertThat(skipSectionBuilder.requiredYamlRunnerFeatures.size(), equalTo(0)); + assertThat(skipSectionBuilder.skipReason, equalTo("Delete ignores the parent param")); + } + + public void testParseSkipSectionFeatureNoVersion() throws Exception { + parser = createParser(YamlXContent.yamlXContent, "features: regex"); + + var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder(); + PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, contains("regex")); + assertThat(skipSectionBuilder.skipReason, nullValue()); + assertThat(skipSectionBuilder.xpackRequired, is(PrerequisiteSection.PrerequisiteSectionBuilder.XPackRequired.NOT_SPECIFIED)); + } + + public void testParseXPackFeature() throws IOException { + parser = createParser(YamlXContent.yamlXContent, "features: xpack"); + + var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder(); + PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, empty()); + assertThat(skipSectionBuilder.skipReason, nullValue()); + assertThat(skipSectionBuilder.xpackRequired, is(PrerequisiteSection.PrerequisiteSectionBuilder.XPackRequired.YES)); + } + + public void testParseNoXPackFeature() throws IOException { + parser = createParser(YamlXContent.yamlXContent, "features: no_xpack"); + + var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder(); + PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, empty()); + assertThat(skipSectionBuilder.skipReason, nullValue()); + assertThat(skipSectionBuilder.xpackRequired, is(PrerequisiteSection.PrerequisiteSectionBuilder.XPackRequired.NO)); + } + + public void testParseBothXPackFeatures() throws IOException { + parser = createParser(YamlXContent.yamlXContent, """ + skip: + features: [xpack, no_xpack] + """); + + var e = expectThrows(ParsingException.class, () -> PrerequisiteSection.parseInternal(parser)); + assertThat(e.getMessage(), containsString("either [xpack] or [no_xpack] can be present, not both")); + } + + public void testParseSkipSectionFeaturesNoVersion() throws Exception { + parser = createParser(YamlXContent.yamlXContent, "features: [regex1,regex2,regex3]"); + + var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder(); + PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, contains("regex1", "regex2", "regex3")); + assertThat(skipSectionBuilder.skipReason, nullValue()); + } + + public void testParseSkipSectionBothFeatureAndVersion() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + version: " - 0.90.2" + features: regex + reason: Delete ignores the parent param"""); + + var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder(); + PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder); + assertThat(skipSectionBuilder.skipVersionRange, not(emptyOrNullString())); + assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, contains("regex")); + assertThat(skipSectionBuilder.skipReason, equalTo("Delete ignores the parent param")); + } + + public void testParseSkipSectionNoReason() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + skip: + version: " - 0.90.2" + """); + + Exception e = expectThrows(ParsingException.class, () -> PrerequisiteSection.parseInternal(parser)); + assertThat(e.getMessage(), is("reason is mandatory within skip version section")); + } + + public void testParseSkipSectionNoVersionNorFeature() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + skip: + reason: Delete ignores the parent param + """); + + Exception e = expectThrows(ParsingException.class, () -> PrerequisiteSection.parseInternal(parser)); + assertThat( + e.getMessage(), + is("at least one criteria (version, cluster features, runner features, os) is mandatory within a skip section") + ); + } + + public void testParseSkipSectionOsNoVersion() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + features: ["skip_os", "some_feature"] + os: debian-9 + reason: memory accounting broken, see gh#xyz + """); + + var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder(); + PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, hasSize(2)); + assertThat(skipSectionBuilder.skipOperatingSystems, contains("debian-9")); + assertThat(skipSectionBuilder.skipReason, is("memory accounting broken, see gh#xyz")); + } + + public void testParseSkipSectionOsListNoVersion() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + features: skip_os + os: [debian-9,windows-95,ms-dos] + reason: see gh#xyz + """); + + var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder(); + PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, hasSize(1)); + assertThat(skipSectionBuilder.skipOperatingSystems, containsInAnyOrder("debian-9", "windows-95", "ms-dos")); + assertThat(skipSectionBuilder.skipReason, is("see gh#xyz")); + } + + public void testParseSkipSectionOsListTestFeaturesInRequires() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + - requires: + test_runner_features: skip_os + reason: skip_os is needed for skip based on os + - skip: + os: [debian-9,windows-95,ms-dos] + reason: see gh#xyz + """); + + var skipSectionBuilder = PrerequisiteSection.parseInternal(parser); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, hasSize(1)); + assertThat(skipSectionBuilder.skipOperatingSystems, containsInAnyOrder("debian-9", "windows-95", "ms-dos")); + assertThat(skipSectionBuilder.skipReason, is("see gh#xyz")); + + assertThat(parser.currentToken(), equalTo(XContentParser.Token.END_ARRAY)); + assertThat(parser.nextToken(), nullValue()); + } + + public void testParseSkipSectionOsNoFeatureNoVersion() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + skip: + os: debian-9 + reason: memory accounting broken, see gh#xyz + """); + + Exception e = expectThrows(ParsingException.class, () -> PrerequisiteSection.parseInternal(parser)); + assertThat(e.getMessage(), is("if os is specified, test runner feature [skip_os] must be set")); + } + + public void testParseRequireSectionClusterFeatures() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + cluster_features: needed-feature + reason: test skipped when cluster lacks needed-feature + """); + + var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder(); + PrerequisiteSection.parseRequiresSection(parser, skipSectionBuilder); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.requiredClusterFeatures, contains("needed-feature")); + assertThat(skipSectionBuilder.requiresReason, is("test skipped when cluster lacks needed-feature")); + } + + public void testParseSkipSectionClusterFeatures() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + cluster_features: undesired-feature + reason: test skipped when undesired-feature is present + """); + + var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder(); + PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.skipClusterFeatures, contains("undesired-feature")); + assertThat(skipSectionBuilder.skipReason, is("test skipped when undesired-feature is present")); + } + + public void testParseRequireAndSkipSectionsClusterFeatures() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + - requires: + cluster_features: needed-feature + reason: test needs needed-feature to run + - skip: + cluster_features: undesired-feature + reason: test cannot run when undesired-feature are present + """); + + var skipSectionBuilder = PrerequisiteSection.parseInternal(parser); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.skipClusterFeatures, contains("undesired-feature")); + assertThat(skipSectionBuilder.requiredClusterFeatures, contains("needed-feature")); + assertThat(skipSectionBuilder.skipReason, is("test cannot run when undesired-feature are present")); + assertThat(skipSectionBuilder.requiresReason, is("test needs needed-feature to run")); + + assertThat(parser.currentToken(), equalTo(XContentParser.Token.END_ARRAY)); + assertThat(parser.nextToken(), nullValue()); + } + + public void testParseRequireAndSkipSectionMultipleClusterFeatures() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + - requires: + cluster_features: [needed-feature-1, needed-feature-2] + reason: test needs some to run + - skip: + cluster_features: [undesired-feature-1, undesired-feature-2] + reason: test cannot run when some are present + """); + + var skipSectionBuilder = PrerequisiteSection.parseInternal(parser); + assertThat(skipSectionBuilder, notNullValue()); + assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString()); + assertThat(skipSectionBuilder.skipClusterFeatures, containsInAnyOrder("undesired-feature-1", "undesired-feature-2")); + assertThat(skipSectionBuilder.requiredClusterFeatures, containsInAnyOrder("needed-feature-1", "needed-feature-2")); + assertThat(skipSectionBuilder.skipReason, is("test cannot run when some are present")); + assertThat(skipSectionBuilder.requiresReason, is("test needs some to run")); + + assertThat(parser.currentToken(), equalTo(XContentParser.Token.END_ARRAY)); + assertThat(parser.nextToken(), nullValue()); + } + + public void testParseSameRequireAndSkipClusterFeatures() throws Exception { + parser = createParser(YamlXContent.yamlXContent, """ + - requires: + cluster_features: some-feature + reason: test needs some-feature to run + - skip: + cluster_features: some-feature + reason: test cannot run with some-feature + """); + + var e = expectThrows(ParsingException.class, () -> PrerequisiteSection.parseInternal(parser)); + assertThat(e.getMessage(), is("a cluster feature can be specified either in [requires] or [skip], not both")); + + assertThat(parser.currentToken(), equalTo(XContentParser.Token.END_ARRAY)); + assertThat(parser.nextToken(), nullValue()); + } + + public void testSkipClusterFeaturesAllRequiredMatch() { + PrerequisiteSection section = new PrerequisiteSection( + emptyList(), + "foobar", + List.of(Prerequisites.requireClusterFeatures(Set.of("required-feature-1", "required-feature-2"))), + "foobar", + emptyList() + ); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true); + when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(true); + + assertFalse(section.skipCriteriaMet(mockContext)); + assertTrue(section.requiresCriteriaMet(mockContext)); + } + + public void testSkipClusterFeaturesSomeRequiredMatch() { + PrerequisiteSection section = new PrerequisiteSection( + emptyList(), + "foobar", + List.of(Prerequisites.requireClusterFeatures(Set.of("required-feature-1", "required-feature-2"))), + "foobar", + emptyList() + ); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true); + when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(false); + + assertFalse(section.skipCriteriaMet(mockContext)); + assertFalse(section.requiresCriteriaMet(mockContext)); + } + + public void testSkipClusterFeaturesSomeToSkipMatch() { + PrerequisiteSection section = new PrerequisiteSection( + List.of(Prerequisites.skipOnClusterFeatures(Set.of("undesired-feature-1", "undesired-feature-2"))), + "foobar", + emptyList(), + "foobar", + emptyList() + ); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + when(mockContext.clusterHasFeature("undesired-feature-1")).thenReturn(true); + + assertTrue(section.skipCriteriaMet(mockContext)); + } + + public void testSkipClusterFeaturesNoneToSkipMatch() { + PrerequisiteSection section = new PrerequisiteSection( + List.of(Prerequisites.skipOnClusterFeatures(Set.of("undesired-feature-1", "undesired-feature-2"))), + "foobar", + emptyList(), + "foobar", + emptyList() + ); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + assertFalse(section.skipCriteriaMet(mockContext)); + } + + public void testSkipClusterFeaturesAllRequiredSomeToSkipMatch() { + PrerequisiteSection section = new PrerequisiteSection( + List.of(Prerequisites.skipOnClusterFeatures(Set.of("undesired-feature-1", "undesired-feature-2"))), + "foobar", + List.of(Prerequisites.requireClusterFeatures(Set.of("required-feature-1", "required-feature-2"))), + "foobar", + emptyList() + ); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true); + when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(true); + when(mockContext.clusterHasFeature("undesired-feature-1")).thenReturn(true); + + assertTrue(section.skipCriteriaMet(mockContext)); + assertTrue(section.requiresCriteriaMet(mockContext)); + } + + public void testSkipClusterFeaturesAllRequiredNoneToSkipMatch() { + PrerequisiteSection section = new PrerequisiteSection( + List.of(Prerequisites.skipOnClusterFeatures(Set.of("undesired-feature-1", "undesired-feature-2"))), + "foobar", + List.of(Prerequisites.requireClusterFeatures(Set.of("required-feature-1", "required-feature-2"))), + "foobar", + emptyList() + ); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true); + when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(true); + + assertFalse(section.skipCriteriaMet(mockContext)); + assertTrue(section.requiresCriteriaMet(mockContext)); + } + + public void evaluateEmpty() { + var section = new PrerequisiteSection(List.of(), "unsupported", List.of(), "required", List.of()); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + section.evaluate(mockContext, "TEST"); + } + + public void evaluateRequiresCriteriaTrue() { + var section = new PrerequisiteSection(List.of(), "unsupported", List.of(Prerequisites.TRUE), "required", List.of()); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + section.evaluate(mockContext, "TEST"); + } + + public void evaluateSkipCriteriaFalse() { + var section = new PrerequisiteSection(List.of(Prerequisites.FALSE), "unsupported", List.of(), "required", List.of()); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + section.evaluate(mockContext, "TEST"); + } + + public void evaluateRequiresCriteriaFalse() { + var section = new PrerequisiteSection( + List.of(Prerequisites.FALSE), + "unsupported", + List.of(Prerequisites.FALSE), + "required", + List.of() + ); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + var e = expectThrows(AssumptionViolatedException.class, () -> section.evaluate(mockContext, "TEST")); + assertThat(e.getMessage(), equalTo("[TEST] skipped, reason: [required]")); + } + + public void evaluateSkipCriteriaTrue() { + var section = new PrerequisiteSection( + List.of(Prerequisites.TRUE), + "unsupported", + List.of(Prerequisites.TRUE), + "required", + List.of() + ); + + var mockContext = mock(ClientYamlTestExecutionContext.class); + var e = expectThrows(AssumptionViolatedException.class, () -> section.evaluate(mockContext, "TEST")); + assertThat(e.getMessage(), equalTo("[TEST] skipped, reason: [unsupported]")); + } +} diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java index 53aaf99d7e27..78c31c85178a 100644 --- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java +++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java @@ -37,7 +37,7 @@ public class SetupSectionTests extends AbstractClientYamlTestFragmentParserTestC SetupSection setupSection = SetupSection.parse(parser); assertThat(setupSection, notNullValue()); - assertThat(setupSection.getSkipSection().isEmpty(), equalTo(true)); + assertThat(setupSection.getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(setupSection.getExecutableSections().size(), equalTo(2)); assertThat(setupSection.getExecutableSections().get(0), instanceOf(DoSection.class)); assertThat(((DoSection) setupSection.getExecutableSections().get(0)).getApiCallSection().getApi(), equalTo("index1")); @@ -60,7 +60,7 @@ public class SetupSectionTests extends AbstractClientYamlTestFragmentParserTestC final SetupSection setupSection = SetupSection.parse(parser); assertNotNull(setupSection); - assertTrue(setupSection.getSkipSection().isEmpty()); + assertTrue(setupSection.getPrerequisiteSection().isEmpty()); assertThat(setupSection.getExecutableSections().size(), equalTo(5)); assertThat(setupSection.getExecutableSections().get(0), instanceOf(DoSection.class)); assertThat(((DoSection) setupSection.getExecutableSections().get(0)).getApiCallSection().getApi(), equalTo("cluster.state")); @@ -105,9 +105,9 @@ public class SetupSectionTests extends AbstractClientYamlTestFragmentParserTestC SetupSection setupSection = SetupSection.parse(parser); assertThat(setupSection, notNullValue()); - assertThat(setupSection.getSkipSection().isEmpty(), equalTo(false)); - assertThat(setupSection.getSkipSection(), notNullValue()); - assertThat(setupSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259")); + assertThat(setupSection.getPrerequisiteSection().isEmpty(), equalTo(false)); + assertThat(setupSection.getPrerequisiteSection(), notNullValue()); + assertThat(setupSection.getPrerequisiteSection().skipReason, equalTo("Update doesn't return metadata fields, waiting for #3259")); assertThat(setupSection.getExecutableSections().size(), equalTo(2)); assertThat(setupSection.getExecutableSections().get(0), instanceOf(DoSection.class)); assertThat(((DoSection) setupSection.getExecutableSections().get(0)).getApiCallSection().getApi(), equalTo("index1")); diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java deleted file mode 100644 index bd1f8fa75849..000000000000 --- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.test.rest.yaml.section; - -import org.elasticsearch.Version; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.core.Strings; -import org.elasticsearch.test.VersionUtils; -import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; -import org.elasticsearch.xcontent.yaml.YamlXContent; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Set; - -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.emptyOrNullString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class SkipSectionTests extends AbstractClientYamlTestFragmentParserTestCase { - - public void testSkipVersionMultiRange() { - SkipSection section = new SkipSection( - List.of(SkipCriteria.fromVersionRange("6.0.0 - 6.1.0, 7.1.0 - 7.5.0")), - Collections.emptyList(), - "foobar" - ); - - var outOfRangeMockContext = mock(ClientYamlTestExecutionContext.class); - when(outOfRangeMockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString())) - .thenReturn(Set.of("6.2.0")) - .thenReturn(Set.of("7.0.0")) - .thenReturn(Set.of("7.6.0")); - - assertFalse(section.skip(outOfRangeMockContext)); - assertFalse(section.skip(outOfRangeMockContext)); - assertFalse(section.skip(outOfRangeMockContext)); - assertFalse(section.skip(outOfRangeMockContext)); - - var inRangeMockContext = mock(ClientYamlTestExecutionContext.class); - when(inRangeMockContext.nodesVersions()).thenReturn(Set.of("6.0.0")) - .thenReturn(Set.of("6.1.0")) - .thenReturn(Set.of("7.1.0")) - .thenReturn(Set.of("7.5.0")); - - assertTrue(section.skip(inRangeMockContext)); - assertTrue(section.skip(inRangeMockContext)); - assertTrue(section.skip(inRangeMockContext)); - assertTrue(section.skip(inRangeMockContext)); - } - - public void testSkipVersionMultiOpenRange() { - var section = new SkipSection( - List.of(SkipCriteria.fromVersionRange("- 7.1.0, 7.2.0 - 7.5.0, 8.0.0 -")), - Collections.emptyList(), - "foobar" - ); - - var outOfRangeMockContext = mock(ClientYamlTestExecutionContext.class); - when(outOfRangeMockContext.nodesVersions()).thenReturn(Set.of("7.1.1")).thenReturn(Set.of("7.6.0")); - - assertFalse(section.skip(outOfRangeMockContext)); - assertFalse(section.skip(outOfRangeMockContext)); - - var inRangeMockContext = mock(ClientYamlTestExecutionContext.class); - when(inRangeMockContext.nodesVersions()).thenReturn(Set.of("7.0.0")) - .thenReturn(Set.of("7.3.0")) - .thenReturn(Set.of("8.0.0")) - .thenReturn(Set.of(Version.CURRENT.toString())); - - assertTrue(section.skip(inRangeMockContext)); - assertTrue(section.skip(inRangeMockContext)); - assertTrue(section.skip(inRangeMockContext)); - assertTrue(section.skip(inRangeMockContext)); - } - - public void testSkipVersion() { - SkipSection section = new SkipSection(List.of(SkipCriteria.fromVersionRange("6.0.0 - 6.1.0")), Collections.emptyList(), "foobar"); - - var mockContext = mock(ClientYamlTestExecutionContext.class); - when(mockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString())) - .thenReturn(Set.of("6.0.0")) - .thenReturn(Set.of("6.0.0", "6.1.0")) - .thenReturn(Set.of("6.0.0", "5.2.0")); - - assertFalse(section.skip(mockContext)); - assertTrue(section.skip(mockContext)); - assertTrue(section.skip(mockContext)); - assertFalse(section.skip(mockContext)); - } - - public void testSkipVersionWithTestFeatures() { - SkipSection section = new SkipSection( - List.of(SkipCriteria.fromVersionRange("6.0.0 - 6.1.0")), - Collections.singletonList("warnings"), - "foobar" - ); - - var mockContext = mock(ClientYamlTestExecutionContext.class); - when(mockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString())).thenReturn(Set.of("6.0.0")); - - assertFalse(section.skip(mockContext)); - assertTrue(section.skip(mockContext)); - } - - public void testSkipTestFeatures() { - var section = new SkipSection.SkipSectionBuilder().withTestFeature("boom").build(); - assertTrue(section.skip(mock(ClientYamlTestExecutionContext.class))); - } - - public void testSkipTestFeaturesOverridesAnySkipCriteria() { - var section = new SkipSection.SkipSectionBuilder().withOs("test-os").withTestFeature("boom").build(); - - var mockContext = mock(ClientYamlTestExecutionContext.class); - when(mockContext.os()).thenReturn("test-os"); - - // Skip even if OS is matching - assertTrue(section.skip(mockContext)); - } - - public void testSkipOs() { - SkipSection section = new SkipSection.SkipSectionBuilder().withOs("windows95").withOs("debian-5").build(); - - var mockContext = mock(ClientYamlTestExecutionContext.class); - - when(mockContext.os()).thenReturn("debian-5"); - assertTrue(section.skip(mockContext)); - - when(mockContext.os()).thenReturn("windows95"); - assertTrue(section.skip(mockContext)); - - when(mockContext.os()).thenReturn("ms-dos"); - assertFalse(section.skip(mockContext)); - } - - public void testSkipOsWithTestFeatures() { - SkipSection section = new SkipSection.SkipSectionBuilder().withTestFeature("warnings") - .withOs("windows95") - .withOs("debian-5") - .build(); - - var mockContext = mock(ClientYamlTestExecutionContext.class); - when(mockContext.os()).thenReturn("debian-5"); - assertTrue(section.skip(mockContext)); - - when(mockContext.os()).thenReturn("windows95"); - assertTrue(section.skip(mockContext)); - - when(mockContext.os()).thenReturn("ms-dos"); - assertFalse(section.skip(mockContext)); - } - - public void testMessage() { - SkipSection section = new SkipSection( - List.of(SkipCriteria.fromVersionRange("6.0.0 - 6.1.0")), - Collections.singletonList("warnings"), - "foobar" - ); - assertEquals("[FOOBAR] skipped, reason: [foobar] unsupported features [warnings]", section.getSkipMessage("FOOBAR")); - section = new SkipSection(List.of(), Collections.singletonList("warnings"), "foobar"); - assertEquals("[FOOBAR] skipped, reason: [foobar] unsupported features [warnings]", section.getSkipMessage("FOOBAR")); - section = new SkipSection(List.of(), Collections.singletonList("warnings"), null); - assertEquals("[FOOBAR] skipped, unsupported features [warnings]", section.getSkipMessage("FOOBAR")); - } - - public void testParseSkipSectionVersionNoFeature() throws Exception { - Version version = VersionUtils.randomVersion(random()); - parser = createParser(YamlXContent.yamlXContent, Strings.format(""" - version: " - %s" - reason: Delete ignores the parent param""", version)); - - var skipSectionBuilder = SkipSection.parseInternal(parser); - assertThat(skipSectionBuilder, notNullValue()); - assertThat(skipSectionBuilder.version, not(emptyOrNullString())); - assertThat(skipSectionBuilder.testFeatures.size(), equalTo(0)); - assertThat(skipSectionBuilder.reason, equalTo("Delete ignores the parent param")); - } - - public void testParseSkipSectionFeatureNoVersion() throws Exception { - parser = createParser(YamlXContent.yamlXContent, "features: regex"); - - var skipSectionBuilder = SkipSection.parseInternal(parser); - assertThat(skipSectionBuilder, notNullValue()); - assertThat(skipSectionBuilder.version, emptyOrNullString()); - assertThat(skipSectionBuilder.testFeatures, contains("regex")); - assertThat(skipSectionBuilder.reason, nullValue()); - assertThat(skipSectionBuilder.xpackRequested, is(SkipSection.SkipSectionBuilder.XPackRequested.NOT_SPECIFIED)); - } - - public void testParseXPackFeature() throws IOException { - parser = createParser(YamlXContent.yamlXContent, "features: xpack"); - - var skipSectionBuilder = SkipSection.parseInternal(parser); - assertThat(skipSectionBuilder, notNullValue()); - assertThat(skipSectionBuilder.version, emptyOrNullString()); - assertThat(skipSectionBuilder.testFeatures, empty()); - assertThat(skipSectionBuilder.reason, nullValue()); - assertThat(skipSectionBuilder.xpackRequested, is(SkipSection.SkipSectionBuilder.XPackRequested.YES)); - } - - public void testParseNoXPackFeature() throws IOException { - parser = createParser(YamlXContent.yamlXContent, "features: no_xpack"); - - var skipSectionBuilder = SkipSection.parseInternal(parser); - assertThat(skipSectionBuilder, notNullValue()); - assertThat(skipSectionBuilder.version, emptyOrNullString()); - assertThat(skipSectionBuilder.testFeatures, empty()); - assertThat(skipSectionBuilder.reason, nullValue()); - assertThat(skipSectionBuilder.xpackRequested, is(SkipSection.SkipSectionBuilder.XPackRequested.NO)); - } - - public void testParseBothXPackFeatures() throws IOException { - parser = createParser(YamlXContent.yamlXContent, "features: [xpack, no_xpack]"); - - var e = expectThrows(ParsingException.class, () -> SkipSection.parseInternal(parser)); - assertThat(e.getMessage(), containsString("either `xpack` or `no_xpack` can be present, not both")); - } - - public void testParseSkipSectionFeaturesNoVersion() throws Exception { - parser = createParser(YamlXContent.yamlXContent, "features: [regex1,regex2,regex3]"); - - var skipSectionBuilder = SkipSection.parseInternal(parser); - assertThat(skipSectionBuilder, notNullValue()); - assertThat(skipSectionBuilder.version, emptyOrNullString()); - assertThat(skipSectionBuilder.testFeatures, contains("regex1", "regex2", "regex3")); - assertThat(skipSectionBuilder.reason, nullValue()); - } - - public void testParseSkipSectionBothFeatureAndVersion() throws Exception { - parser = createParser(YamlXContent.yamlXContent, """ - version: " - 0.90.2" - features: regex - reason: Delete ignores the parent param"""); - - var skipSectionBuilder = SkipSection.parseInternal(parser); - assertThat(skipSectionBuilder.version, not(emptyOrNullString())); - assertThat(skipSectionBuilder.testFeatures, contains("regex")); - assertThat(skipSectionBuilder.reason, equalTo("Delete ignores the parent param")); - } - - public void testParseSkipSectionNoReason() throws Exception { - parser = createParser(YamlXContent.yamlXContent, "version: \" - 0.90.2\"\n"); - - Exception e = expectThrows(ParsingException.class, () -> SkipSection.parseInternal(parser)); - assertThat(e.getMessage(), is("reason is mandatory within skip version section")); - } - - public void testParseSkipSectionNoVersionNorFeature() throws Exception { - parser = createParser(YamlXContent.yamlXContent, "reason: Delete ignores the parent param\n"); - - Exception e = expectThrows(ParsingException.class, () -> SkipSection.parseInternal(parser)); - assertThat(e.getMessage(), is("at least one criteria (version, test features, os) is mandatory within a skip section")); - } - - public void testParseSkipSectionOsNoVersion() throws Exception { - parser = createParser(YamlXContent.yamlXContent, """ - features: ["skip_os", "some_feature"] - os: debian-9 - reason: memory accounting broken, see gh#xyz - """); - - var skipSectionBuilder = SkipSection.parseInternal(parser); - assertThat(skipSectionBuilder, notNullValue()); - assertThat(skipSectionBuilder.version, emptyOrNullString()); - assertThat(skipSectionBuilder.testFeatures, hasSize(2)); - assertThat(skipSectionBuilder.operatingSystems, contains("debian-9")); - assertThat(skipSectionBuilder.reason, is("memory accounting broken, see gh#xyz")); - } - - public void testParseSkipSectionOsListNoVersion() throws Exception { - parser = createParser(YamlXContent.yamlXContent, """ - features: skip_os - os: [debian-9,windows-95,ms-dos] - reason: see gh#xyz - """); - - var skipSectionBuilder = SkipSection.parseInternal(parser); - assertThat(skipSectionBuilder, notNullValue()); - assertThat(skipSectionBuilder.version, emptyOrNullString()); - assertThat(skipSectionBuilder.testFeatures, hasSize(1)); - assertThat(skipSectionBuilder.operatingSystems, containsInAnyOrder("debian-9", "windows-95", "ms-dos")); - assertThat(skipSectionBuilder.reason, is("see gh#xyz")); - } - - public void testParseSkipSectionOsNoFeatureNoVersion() throws Exception { - parser = createParser(YamlXContent.yamlXContent, """ - os: debian-9 - reason: memory accounting broken, see gh#xyz - """); - - Exception e = expectThrows(ParsingException.class, () -> SkipSection.parseInternal(parser)); - assertThat(e.getMessage(), is("if os is specified, feature skip_os must be set")); - } -} diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java index 2c6b4f5be12d..9844b90eb214 100644 --- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java +++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java @@ -35,7 +35,7 @@ public class TeardownSectionTests extends AbstractClientYamlTestFragmentParserTe TeardownSection section = TeardownSection.parse(parser); assertThat(section, notNullValue()); - assertThat(section.getSkipSection().isEmpty(), equalTo(true)); + assertThat(section.getPrerequisiteSection().isEmpty(), equalTo(true)); assertThat(section.getDoSections().size(), equalTo(2)); assertThat(((DoSection) section.getDoSections().get(0)).getApiCallSection().getApi(), equalTo("delete")); assertThat(((DoSection) section.getDoSections().get(1)).getApiCallSection().getApi(), equalTo("delete2")); @@ -62,8 +62,8 @@ public class TeardownSectionTests extends AbstractClientYamlTestFragmentParserTe TeardownSection section = TeardownSection.parse(parser); assertThat(section, notNullValue()); - assertThat(section.getSkipSection().isEmpty(), equalTo(false)); - assertThat(section.getSkipSection().getReason(), equalTo("there is a reason")); + assertThat(section.getPrerequisiteSection().isEmpty(), equalTo(false)); + assertThat(section.getPrerequisiteSection().skipReason, equalTo("there is a reason")); assertThat(section.getDoSections().size(), equalTo(2)); assertThat(((DoSection) section.getDoSections().get(0)).getApiCallSection().getApi(), equalTo("delete")); assertThat(((DoSection) section.getDoSections().get(1)).getApiCallSection().getApi(), equalTo("delete2")); diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncIT.java b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncIT.java index c2fa41a5241d..657f396b2857 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncIT.java @@ -52,7 +52,7 @@ public class EsqlClientYamlAsyncIT extends AbstractEsqlClientYamlIT { ClientYamlTestSection modified = new ClientYamlTestSection( candidate.getTestSection().getLocation(), candidate.getTestSection().getName(), - candidate.getTestSection().getSkipSection(), + candidate.getTestSection().getPrerequisiteSection(), candidate.getTestSection().getExecutableSections().stream().map(e -> modifyExecutableSection(e, modify)).toList() ); result.add(new Object[] { new ClientYamlTestCandidate(candidate.getRestTestSuite(), modified) }); diff --git a/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index 110a1fd24d0d..badc04800e40 100644 --- a/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -67,7 +67,7 @@ public abstract class CoreTestTranslater { ClientYamlTestSection modified = new ClientYamlTestSection( candidate.getTestSection().getLocation(), candidate.getTestSection().getName(), - candidate.getTestSection().getSkipSection(), + candidate.getTestSection().getPrerequisiteSection(), candidate.getTestSection().getExecutableSections() ); result.add(new Object[] { new ClientYamlTestCandidate(suite.modified, modified) }); @@ -169,7 +169,7 @@ public abstract class CoreTestTranslater { candidate.getApi(), candidate.getName(), candidate.getRestTestSuite().getFile(), - new SetupSection(candidate.getSetupSection().getSkipSection(), setup), + new SetupSection(candidate.getSetupSection().getPrerequisiteSection(), setup), candidate.getTeardownSection(), List.of() );