github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-28 01:22:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

YAML test framework: separate skip and requires sections (#104140)

Browse source 
* Introduce Prerequisites criteria (Predicate + factory) for modular skip decisions
- Removed accessors to specific criteria from SkipSection (used only on tests), adjusted test assertions
- Moved Features check (YAML test runner features) to SkipSection build time

* Separated check for xpack/no_xpack
Check for xpack is cluster-configuration (modules installed) dependent, while Features are meant to be "static" test-runner capabilities. We separate them so checks on one (test-runner features) can be run before and separately from the other.

* Consolidate skip() methods
- Divide require and skip predicates
- Divide requires and skip parsing (distinct sections)
- Renaming SkipSection to PrerequisiteSection and related methods/fields (e.g. skip -> evaluate)

* Refactoring tests
- moving and adding VersionRange tests
- adding specific version and os skip tests
- modified parse/validate/build to make SkipSection more unit-testable

* Adding cluster feature-based skip criteria
* Updated javadoc + renaming + better skip reason message
This commit is contained in:
Lorenzo Dematté 2024-01-15 14:48:36 +01:00 • committed by GitHub
parent 797fadceb7
commit 2b175653d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 1280 additions and 719 deletions
Download patch file Download diff file Expand all files Collapse all files

2
qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
View file

@ -259,7 +259,7 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
new ClientYamlTestSection(
testSection.getLocation(),
testSection.getName(),
testSection.getSkipSection(),
testSection.getPrerequisiteSection(),
modifiedExecutableSections
)
);

20
test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
View file

@ -446,20 +446,10 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
}
// skip test if the whole suite (yaml file) is disabled
assumeFalse(
testCandidate.getSetupSection().getSkipSection().getSkipMessage(testCandidate.getSuitePath()),
testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext)
);
// skip test if the whole suite (yaml file) is disabled
assumeFalse(
testCandidate.getTeardownSection().getSkipSection().getSkipMessage(testCandidate.getSuitePath()),
testCandidate.getTeardownSection().getSkipSection().skip(restTestExecutionContext)
);
testCandidate.getSetupSection().getPrerequisiteSection().evaluate(restTestExecutionContext, testCandidate.getSuitePath());
testCandidate.getTeardownSection().getPrerequisiteSection().evaluate(restTestExecutionContext, testCandidate.getSuitePath());
// skip test if test section is disabled
assumeFalse(
testCandidate.getTestSection().getSkipSection().getSkipMessage(testCandidate.getTestPath()),
testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext)
);
testCandidate.getTestSection().getPrerequisiteSection().evaluate(restTestExecutionContext, testCandidate.getTestPath());
// let's check that there is something to run, otherwise there might be a problem with the test section
if (testCandidate.getTestSection().getExecutableSections().isEmpty()) {
@ -468,11 +458,11 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
assumeFalse(
"[" + testCandidate.getTestPath() + "] skipped, reason: in fips 140 mode",
inFipsJvm() && testCandidate.getTestSection().getSkipSection().yamlRunnerHasFeature("fips_140")
inFipsJvm() && testCandidate.getTestSection().getPrerequisiteSection().hasYamlRunnerFeature("fips_140")
);
final Settings globalTemplateSettings = getGlobalTemplateSettings(
testCandidate.getTestSection().getSkipSection().yamlRunnerHasFeature("default_shards")
testCandidate.getTestSection().getPrerequisiteSection().hasYamlRunnerFeature("default_shards")
);
if (globalTemplateSettings.isEmpty() == false && ESRestTestCase.has(ProductFeature.LEGACY_TEMPLATES)) {

14
test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
View file

@ -28,7 +28,7 @@ public class ClientYamlTestSection implements Comparable<ClientYamlTestSection>
List<ExecutableSection> executableSections = new ArrayList<>();
try {
parser.nextToken();
SkipSection skipSection = SkipSection.parseIfNext(parser);
PrerequisiteSection prerequisiteSection = PrerequisiteSection.parseIfNext(parser);
while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
ParserUtils.advanceToFieldName(parser);
executableSections.add(ExecutableSection.parse(parser));
@ -45,7 +45,7 @@ public class ClientYamlTestSection implements Comparable<ClientYamlTestSection>
);
}
parser.nextToken();
return new ClientYamlTestSection(sectionLocation, sectionName, skipSection, executableSections);
return new ClientYamlTestSection(sectionLocation, sectionName, prerequisiteSection, executableSections);
} catch (Exception e) {
throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + sectionName + "]", e);
}
@ -53,18 +53,18 @@ public class ClientYamlTestSection implements Comparable<ClientYamlTestSection>
private final XContentLocation location;
private final String name;
private final SkipSection skipSection;
private final PrerequisiteSection prerequisiteSection;
private final List<ExecutableSection> executableSections;
public ClientYamlTestSection(
XContentLocation location,
String name,
SkipSection skipSection,
PrerequisiteSection prerequisiteSection,
List<ExecutableSection> executableSections
) {
this.location = location;
this.name = name;
this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null");
this.prerequisiteSection = Objects.requireNonNull(prerequisiteSection, "skip section cannot be null");
this.executableSections = Collections.unmodifiableList(executableSections);
}
@ -76,8 +76,8 @@ public class ClientYamlTestSection implements Comparable<ClientYamlTestSection>
return name;
}
public SkipSection getSkipSection() {
return skipSection;
public PrerequisiteSection getPrerequisiteSection() {
return prerequisiteSection;
}
public List<ExecutableSection> getExecutableSections() {

53
test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
View file

@ -175,9 +175,9 @@ public class ClientYamlTestSuite {
.filter(section -> section instanceof DoSection)
.map(section -> (DoSection) section)
.filter(section -> false == section.getExpectedWarningHeaders().isEmpty())
.filter(section -> false == hasSkipFeature("warnings", testSection, setupSection, teardownSection))
.filter(section -> false == hasYamlRunnerFeature("warnings", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [warnings] section without a corresponding ["skip": "features": "warnings"] \
attempted to add a [do] with a [warnings] section without a corresponding ["requires": "test_runner_features": "warnings"] \
so runners that do not support the [warnings] section can skip the test at line [%d]\
""", section.getLocation().lineNumber()));
@ -187,10 +187,10 @@ public class ClientYamlTestSuite {
.filter(section -> section instanceof DoSection)
.map(section -> (DoSection) section)
.filter(section -> false == section.getExpectedWarningHeadersRegex().isEmpty())
.filter(section -> false == hasSkipFeature("warnings_regex", testSection, setupSection, teardownSection))
.filter(section -> false == hasYamlRunnerFeature("warnings_regex", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [warnings_regex] section without a corresponding \
["skip": "features": "warnings_regex"] so runners that do not support the [warnings_regex] \
["requires": "test_runner_features": "warnings_regex"] so runners that do not support the [warnings_regex] \
section can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
@ -201,10 +201,10 @@ public class ClientYamlTestSuite {
.filter(section -> section instanceof DoSection)
.map(section -> (DoSection) section)
.filter(section -> false == section.getAllowedWarningHeaders().isEmpty())
.filter(section -> false == hasSkipFeature("allowed_warnings", testSection, setupSection, teardownSection))
.filter(section -> false == hasYamlRunnerFeature("allowed_warnings", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [allowed_warnings] section without a corresponding \
["skip": "features": "allowed_warnings"] so runners that do not support the [allowed_warnings] \
["requires": "test_runner_features": "allowed_warnings"] so runners that do not support the [allowed_warnings] \
section can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
@ -215,11 +215,11 @@ public class ClientYamlTestSuite {
.filter(section -> section instanceof DoSection)
.map(section -> (DoSection) section)
.filter(section -> false == section.getAllowedWarningHeadersRegex().isEmpty())
.filter(section -> false == hasSkipFeature("allowed_warnings_regex", testSection, setupSection, teardownSection))
.filter(section -> false == hasYamlRunnerFeature("allowed_warnings_regex", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [allowed_warnings_regex] section without a corresponding \
["skip": "features": "allowed_warnings_regex"] so runners that do not support the [allowed_warnings_regex] \
section can skip the test at line [%d]\
["requires": "test_runner_features": "allowed_warnings_regex"] so runners that do not support the \
[allowed_warnings_regex] section can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
@ -229,10 +229,10 @@ public class ClientYamlTestSuite {
.filter(section -> section instanceof DoSection)
.map(section -> (DoSection) section)
.filter(section -> NodeSelector.ANY != section.getApiCallSection().getNodeSelector())
.filter(section -> false == hasSkipFeature("node_selector", testSection, setupSection, teardownSection))
.filter(section -> false == hasYamlRunnerFeature("node_selector", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [node_selector] section without a corresponding \
["skip": "features": "node_selector"] so runners that do not support the [node_selector] section \
["requires": "test_runner_features": "node_selector"] so runners that do not support the [node_selector] section \
can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
@ -241,9 +241,9 @@ public class ClientYamlTestSuite {
errors,
sections.stream()
.filter(section -> section instanceof ContainsAssertion)
.filter(section -> false == hasSkipFeature("contains", testSection, setupSection, teardownSection))
.filter(section -> false == hasYamlRunnerFeature("contains", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [contains] assertion without a corresponding ["skip": "features": "contains"] \
attempted to add a [contains] assertion without a corresponding ["requires": "test_runner_features": "contains"] \
so runners that do not support the [contains] assertion can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
@ -254,10 +254,11 @@ public class ClientYamlTestSuite {
.filter(section -> section instanceof DoSection)
.map(section -> (DoSection) section)
.filter(section -> false == section.getApiCallSection().getHeaders().isEmpty())
.filter(section -> false == hasSkipFeature("headers", testSection, setupSection, teardownSection))
.filter(section -> false == hasYamlRunnerFeature("headers", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [headers] section without a corresponding ["skip": "features": "headers"] \
so runners that do not support the [headers] section can skip the test at line [%d]\
attempted to add a [do] with a [headers] section without a corresponding \
["requires": "test_runner_features": "headers"] so runners that do not support the [headers] section \
can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
@ -265,9 +266,9 @@ public class ClientYamlTestSuite {
errors,
sections.stream()
.filter(section -> section instanceof CloseToAssertion)
.filter(section -> false == hasSkipFeature("close_to", testSection, setupSection, teardownSection))
.filter(section -> false == hasYamlRunnerFeature("close_to", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [close_to] assertion without a corresponding ["skip": "features": "close_to"] \
attempted to add a [close_to] assertion without a corresponding ["requires": "test_runner_features": "close_to"] \
so runners that do not support the [close_to] assertion can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
@ -276,9 +277,9 @@ public class ClientYamlTestSuite {
errors,
sections.stream()
.filter(section -> section instanceof IsAfterAssertion)
.filter(section -> false == hasSkipFeature("is_after", testSection, setupSection, teardownSection))
.filter(section -> false == hasYamlRunnerFeature("is_after", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add an [is_after] assertion without a corresponding ["skip": "features": "is_after"] \
attempted to add an [is_after] assertion without a corresponding ["requires": "test_runner_features": "is_after"] \
so runners that do not support the [is_after] assertion can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
@ -286,19 +287,19 @@ public class ClientYamlTestSuite {
return errors;
}
private static boolean hasSkipFeature(
private static boolean hasYamlRunnerFeature(
String feature,
ClientYamlTestSection testSection,
SetupSection setupSection,
TeardownSection teardownSection
) {
return (testSection != null && hasSkipFeature(feature, testSection.getSkipSection()))
|| (setupSection != null && hasSkipFeature(feature, setupSection.getSkipSection()))
|| (teardownSection != null && hasSkipFeature(feature, teardownSection.getSkipSection()));
return (testSection != null && hasYamlRunnerFeature(feature, testSection.getPrerequisiteSection()))
|| (setupSection != null && hasYamlRunnerFeature(feature, setupSection.getPrerequisiteSection()))
|| (teardownSection != null && hasYamlRunnerFeature(feature, teardownSection.getPrerequisiteSection()));
}
private static boolean hasSkipFeature(String feature, SkipSection skipSection) {
return skipSection != null && skipSection.yamlRunnerHasFeature(feature);
private static boolean hasYamlRunnerFeature(String feature, PrerequisiteSection prerequisiteSection) {
return prerequisiteSection != null && prerequisiteSection.hasYamlRunnerFeature(feature);
}
public List<ClientYamlTestSection> getTestSections() {

413
test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java Normal file
View file

@ -0,0 +1,413 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.test.rest.yaml.section;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import org.elasticsearch.test.rest.yaml.Features;
import org.elasticsearch.xcontent.XContentLocation;
import org.elasticsearch.xcontent.XContentParser;
import org.junit.AssumptionViolatedException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
/**
* Represents a section where prerequisites to run a specific test section or suite are specified. It is possible to specify preconditions
* as a set of `skip` criteria (the test or suite will be skipped if the specified conditions are met) or `requires` criteria (the test or
* suite will be run only if the specified conditions are met)
* Criteria are based on:
* - the elasticsearch cluster version the tests are running against (deprecated)
* - the features supported by the elasticsearch cluster version the tests are running against
* - a specific test runner feature - some runners may not implement the whole set of features
* - an operating system (full name, including specific Linux distributions) - some OS might show a certain behavior
*/
public class PrerequisiteSection {
private static final Logger logger = LogManager.getLogger(PrerequisiteSection.class);
static class PrerequisiteSectionBuilder {
String skipVersionRange = null;
String skipReason = null;
String requiresReason = null;
List<String> requiredYamlRunnerFeatures = new ArrayList<>();
List<String> skipOperatingSystems = new ArrayList<>();
Set<String> skipClusterFeatures = new HashSet<>();
Set<String> requiredClusterFeatures = new HashSet<>();
enum XPackRequired {
NOT_SPECIFIED,
YES,
NO,
MISMATCHED
}
XPackRequired xpackRequired = XPackRequired.NOT_SPECIFIED;
public PrerequisiteSectionBuilder skipIfVersion(String skipVersionRange) {
this.skipVersionRange = skipVersionRange;
return this;
}
public PrerequisiteSectionBuilder setSkipReason(String skipReason) {
this.skipReason = skipReason;
return this;
}
public PrerequisiteSectionBuilder setRequiresReason(String requiresReason) {
this.requiresReason = requiresReason;
return this;
}
public PrerequisiteSectionBuilder requireYamlRunnerFeature(String featureName) {
requiredYamlRunnerFeatures.add(featureName);
return this;
}
public PrerequisiteSectionBuilder requireXPack() {
if (xpackRequired == XPackRequired.NO) {
xpackRequired = XPackRequired.MISMATCHED;
} else {
xpackRequired = XPackRequired.YES;
}
return this;
}
public PrerequisiteSectionBuilder skipIfXPack() {
if (xpackRequired == XPackRequired.YES) {
xpackRequired = XPackRequired.MISMATCHED;
} else {
xpackRequired = XPackRequired.NO;
}
return this;
}
public PrerequisiteSectionBuilder skipIfClusterFeature(String featureName) {
skipClusterFeatures.add(featureName);
return this;
}
public PrerequisiteSectionBuilder requireClusterFeature(String featureName) {
requiredClusterFeatures.add(featureName);
return this;
}
public PrerequisiteSectionBuilder skipIfOs(String osName) {
this.skipOperatingSystems.add(osName);
return this;
}
void validate(XContentLocation contentLocation) {
if ((Strings.hasLength(skipVersionRange) == false)
&& requiredYamlRunnerFeatures.isEmpty()
&& skipOperatingSystems.isEmpty()
&& xpackRequired == XPackRequired.NOT_SPECIFIED
&& requiredClusterFeatures.isEmpty()
&& skipClusterFeatures.isEmpty()) {
throw new ParsingException(
contentLocation,
"at least one criteria (version, cluster features, runner features, os) is mandatory within a skip section"
);
}
if (Strings.hasLength(skipVersionRange) && Strings.hasLength(skipReason) == false) {
throw new ParsingException(contentLocation, "reason is mandatory within skip version section");
}
if (skipOperatingSystems.isEmpty() == false && Strings.hasLength(skipReason) == false) {
throw new ParsingException(contentLocation, "reason is mandatory within skip os section");
}
if (skipClusterFeatures.isEmpty() == false && Strings.hasLength(skipReason) == false) {
throw new ParsingException(contentLocation, "reason is mandatory within skip cluster_features section");
}
if (requiredClusterFeatures.isEmpty() == false && Strings.hasLength(requiresReason) == false) {
throw new ParsingException(contentLocation, "reason is mandatory within requires cluster_features section");
}
// make feature "skip_os" mandatory if os is given, this is a temporary solution until language client tests know about os
if (skipOperatingSystems.isEmpty() == false && requiredYamlRunnerFeatures.contains("skip_os") == false) {
throw new ParsingException(contentLocation, "if os is specified, test runner feature [skip_os] must be set");
}
if (xpackRequired == XPackRequired.MISMATCHED) {
throw new ParsingException(contentLocation, "either [xpack] or [no_xpack] can be present, not both");
}
if (Sets.haveNonEmptyIntersection(skipClusterFeatures, requiredClusterFeatures)) {
throw new ParsingException(contentLocation, "a cluster feature can be specified either in [requires] or [skip], not both");
}
}
public PrerequisiteSection build() {
final List<Predicate<ClientYamlTestExecutionContext>> skipCriteriaList = new ArrayList<>();
final List<Predicate<ClientYamlTestExecutionContext>> requiresCriteriaList;
// Check if the test runner supports all YAML framework features (see {@link Features}). If not, default to always skip this
// section.
if (Features.areAllSupported(requiredYamlRunnerFeatures) == false) {
requiresCriteriaList = List.of(Prerequisites.FALSE);
} else {
requiresCriteriaList = new ArrayList<>();
if (xpackRequired == XPackRequired.YES) {
requiresCriteriaList.add(Prerequisites.hasXPack());
}
if (xpackRequired == XPackRequired.NO) {
skipCriteriaList.add(Prerequisites.hasXPack());
}
if (Strings.hasLength(skipVersionRange)) {
skipCriteriaList.add(Prerequisites.skipOnVersionRange(skipVersionRange));
}
if (skipOperatingSystems.isEmpty() == false) {
skipCriteriaList.add(Prerequisites.skipOnOsList(skipOperatingSystems));
}
if (requiredClusterFeatures.isEmpty() == false) {
requiresCriteriaList.add(Prerequisites.requireClusterFeatures(requiredClusterFeatures));
}
if (skipClusterFeatures.isEmpty() == false) {
skipCriteriaList.add(Prerequisites.skipOnClusterFeatures(skipClusterFeatures));
}
}
return new PrerequisiteSection(skipCriteriaList, skipReason, requiresCriteriaList, requiresReason, requiredYamlRunnerFeatures);
}
}
/**
* Parse a {@link PrerequisiteSection} if the next field is {@code skip}, otherwise returns {@link PrerequisiteSection#EMPTY}.
*/
public static PrerequisiteSection parseIfNext(XContentParser parser) throws IOException {
return parseInternal(parser).build();
}
private static void maybeAdvanceToNextField(XContentParser parser) throws IOException {
var token = parser.nextToken();
if (token != null && token != XContentParser.Token.END_ARRAY) {
ParserUtils.advanceToFieldName(parser);
}
}
static PrerequisiteSectionBuilder parseInternal(XContentParser parser) throws IOException {
PrerequisiteSectionBuilder builder = new PrerequisiteSectionBuilder();
var hasPrerequisiteSection = false;
var unknownFieldName = false;
ParserUtils.advanceToFieldName(parser);
while (unknownFieldName == false) {
if ("skip".equals(parser.currentName())) {
parseSkipSection(parser, builder);
hasPrerequisiteSection = true;
maybeAdvanceToNextField(parser);
} else if ("requires".equals(parser.currentName())) {
parseRequiresSection(parser, builder);
hasPrerequisiteSection = true;
maybeAdvanceToNextField(parser);
} else {
unknownFieldName = true;
}
}
if (hasPrerequisiteSection) {
builder.validate(parser.getTokenLocation());
}
return builder;
}
private static void parseFeatureField(String feature, PrerequisiteSectionBuilder builder) {
// #31403 introduced YAML test "features" to indicate if the cluster being tested has xpack installed (`xpack`)
// or if it does *not* have xpack installed (`no_xpack`). These are not test runner features, so now that we have
// "modular" skip criteria let's separate them. Eventually, these should move to their own skip section.
if (feature.equals("xpack")) {
builder.requireXPack();
} else if (feature.equals("no_xpack")) {
builder.skipIfXPack();
} else {
builder.requireYamlRunnerFeature(feature);
}
}
// package private for tests
static void parseSkipSection(XContentParser parser, PrerequisiteSectionBuilder builder) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new IllegalArgumentException(
"Expected ["
+ XContentParser.Token.START_OBJECT
+ ", found ["
+ parser.currentToken()
+ "], the skip section is not properly indented"
);
}
String currentFieldName = null;
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token.isValue()) {
if ("version".equals(currentFieldName)) {
builder.skipIfVersion(parser.text());
} else if ("reason".equals(currentFieldName)) {
builder.setSkipReason(parser.text());
} else if ("features".equals(currentFieldName)) {
// TODO: legacy - remove
logger.warn(
"[\"skip\": \"features\"] is deprecated and will be removed. Replace it with "
+ "[\"requires\": \"test_runner_features\"]"
);
parseFeatureField(parser.text(), builder);
} else if ("os".equals(currentFieldName)) {
builder.skipIfOs(parser.text());
} else if ("cluster_features".equals(currentFieldName)) {
builder.skipIfClusterFeature(parser.text());
} else {
throw new ParsingException(
parser.getTokenLocation(),
"field " + currentFieldName + " not supported within skip section"
);
}
} else if (token == XContentParser.Token.START_ARRAY) {
// TODO: legacy - remove
logger.warn(
"[\"skip\": \"features\"] is deprecated and will be removed. Replace it with "
+ "[\"requires\": \"test_runner_features\"]"
);
if ("features".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
parseFeatureField(parser.text(), builder);
}
} else if ("os".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
builder.skipIfOs(parser.text());
}
} else if ("cluster_features".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
builder.skipIfClusterFeature(parser.text());
}
}
}
}
parser.nextToken();
}
static void parseRequiresSection(XContentParser parser, PrerequisiteSectionBuilder builder) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new IllegalArgumentException(
"Expected ["
+ XContentParser.Token.START_OBJECT
+ ", found ["
+ parser.currentToken()
+ "], the requires section is not properly indented"
);
}
String currentFieldName = null;
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token.isValue()) {
if ("reason".equals(currentFieldName)) {
builder.setRequiresReason(parser.text());
} else if ("test_runner_features".equals(currentFieldName)) {
parseFeatureField(parser.text(), builder);
} else if ("cluster_features".equals(currentFieldName)) {
builder.requireClusterFeature(parser.text());
} else {
throw new ParsingException(
parser.getTokenLocation(),
"field " + currentFieldName + " not supported within requires section"
);
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("test_runner_features".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
parseFeatureField(parser.text(), builder);
}
} else if ("cluster_features".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
builder.requireClusterFeature(parser.text());
}
}
}
}
parser.nextToken();
}
public static final PrerequisiteSection EMPTY = new PrerequisiteSection();
private final List<Predicate<ClientYamlTestExecutionContext>> skipCriteriaList;
private final List<Predicate<ClientYamlTestExecutionContext>> requiresCriteriaList;
private final List<String> yamlRunnerFeatures;
final String skipReason;
final String requireReason;
private PrerequisiteSection() {
this.skipCriteriaList = new ArrayList<>();
this.requiresCriteriaList = new ArrayList<>();
this.yamlRunnerFeatures = new ArrayList<>();
this.skipReason = null;
this.requireReason = null;
}
PrerequisiteSection(
List<Predicate<ClientYamlTestExecutionContext>> skipCriteriaList,
String skipReason,
List<Predicate<ClientYamlTestExecutionContext>> requiresCriteriaList,
String requireReason,
List<String> yamlRunnerFeatures
) {
this.skipCriteriaList = skipCriteriaList;
this.requiresCriteriaList = requiresCriteriaList;
this.yamlRunnerFeatures = yamlRunnerFeatures;
this.skipReason = skipReason;
this.requireReason = requireReason;
}
public boolean hasYamlRunnerFeature(String feature) {
return yamlRunnerFeatures.contains(feature);
}
boolean skipCriteriaMet(ClientYamlTestExecutionContext context) {
return skipCriteriaList.stream().anyMatch(c -> c.test(context));
}
boolean requiresCriteriaMet(ClientYamlTestExecutionContext context) {
return requiresCriteriaList.stream().allMatch(c -> c.test(context));
}
public void evaluate(ClientYamlTestExecutionContext context, String testCandidateDescription) {
if (isEmpty()) {
return;
}
if (requiresCriteriaMet(context) == false) {
throw new AssumptionViolatedException(buildMessage(testCandidateDescription, false));
}
if (skipCriteriaMet(context)) {
throw new AssumptionViolatedException(buildMessage(testCandidateDescription, true));
}
}
boolean isEmpty() {
return skipCriteriaList.isEmpty() && requiresCriteriaList.isEmpty() && yamlRunnerFeatures.isEmpty();
}
String buildMessage(String description, boolean isSkip) {
StringBuilder messageBuilder = new StringBuilder();
messageBuilder.append("[").append(description).append("] skipped,");
var reason = isSkip ? skipReason : requireReason;
if (Strings.isNullOrEmpty(reason) == false) {
messageBuilder.append(" reason: [").append(reason).append("]");
}
if (yamlRunnerFeatures.isEmpty() == false) {
messageBuilder.append(" unsupported features ").append(yamlRunnerFeatures);
}
return messageBuilder.toString();
}
}

29
test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipCriteria.java → test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/Prerequisites.java
View file

@ -12,31 +12,36 @@ import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
public class SkipCriteria {
public class Prerequisites {
public static final Predicate<ClientYamlTestExecutionContext> SKIP_ALWAYS = context -> true;
public static final Predicate<ClientYamlTestExecutionContext> TRUE = context -> true;
public static final Predicate<ClientYamlTestExecutionContext> FALSE = context -> false;
private SkipCriteria() {}
private Prerequisites() {}
static Predicate<ClientYamlTestExecutionContext> fromVersionRange(String versionRange) {
static Predicate<ClientYamlTestExecutionContext> skipOnVersionRange(String versionRange) {
final var versionRangePredicates = VersionRange.parseVersionRanges(versionRange);
assert versionRangePredicates.isEmpty() == false;
return context -> versionRangePredicates.stream().anyMatch(range -> range.test(context.nodesVersions()));
}
static Predicate<ClientYamlTestExecutionContext> fromOsList(List<String> operatingSystems) {
static Predicate<ClientYamlTestExecutionContext> skipOnOsList(List<String> operatingSystems) {
return context -> operatingSystems.stream().anyMatch(osName -> osName.equals(context.os()));
}
static Predicate<ClientYamlTestExecutionContext> fromClusterModules(boolean xpackRequired) {
static Predicate<ClientYamlTestExecutionContext> hasXPack() {
// TODO: change ESRestTestCase.hasXPack() to be context-specific
return context -> {
if (xpackRequired) {
return ESRestTestCase.hasXPack() == false;
}
return ESRestTestCase.hasXPack();
};
return context -> ESRestTestCase.hasXPack();
}
static Predicate<ClientYamlTestExecutionContext> requireClusterFeatures(Set<String> clusterFeatures) {
return context -> clusterFeatures.stream().allMatch(context::clusterHasFeature);
}
static Predicate<ClientYamlTestExecutionContext> skipOnClusterFeatures(Set<String> clusterFeatures) {
return context -> clusterFeatures.stream().anyMatch(context::clusterHasFeature);
}
}

16
test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
View file

@ -36,7 +36,7 @@ public class SetupSection {
}
public static SetupSection parse(XContentParser parser) throws IOException {
SkipSection skipSection = SkipSection.parseIfNext(parser);
PrerequisiteSection prerequisiteSection = PrerequisiteSection.parseIfNext(parser);
List<ExecutableSection> executableSections = new ArrayList<>();
while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
ParserUtils.advanceToFieldName(parser);
@ -51,21 +51,21 @@ public class SetupSection {
parser.nextToken();
}
parser.nextToken();
return new SetupSection(skipSection, executableSections);
return new SetupSection(prerequisiteSection, executableSections);
}
public static final SetupSection EMPTY = new SetupSection(SkipSection.EMPTY, Collections.emptyList());
public static final SetupSection EMPTY = new SetupSection(PrerequisiteSection.EMPTY, Collections.emptyList());
private final SkipSection skipSection;
private final PrerequisiteSection prerequisiteSection;
private final List<ExecutableSection> executableSections;
public SetupSection(SkipSection skipSection, List<ExecutableSection> executableSections) {
this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null");
public SetupSection(PrerequisiteSection prerequisiteSection, List<ExecutableSection> executableSections) {
this.prerequisiteSection = Objects.requireNonNull(prerequisiteSection, "skip section cannot be null");
this.executableSections = Collections.unmodifiableList(executableSections);
}
public SkipSection getSkipSection() {
return skipSection;
public PrerequisiteSection getPrerequisiteSection() {
return prerequisiteSection;
}
public List<ExecutableSection> getExecutableSections() {

255
test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java
View file

@ -1,255 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import org.elasticsearch.test.rest.yaml.Features;
import org.elasticsearch.xcontent.XContentLocation;
import org.elasticsearch.xcontent.XContentParser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Predicate;
/**
* Represents a skip section that tells whether a specific test section or suite needs to be skipped
* based on:
* - the elasticsearch version the tests are running against
* - a specific test feature required that might not be implemented yet by the runner
* - an operating system (full name, including specific Linux distributions) that might show a certain behavior
*/
public class SkipSection {
static class SkipSectionBuilder {
String version = null;
String reason = null;
List<String> testFeatures = new ArrayList<>();
List<String> operatingSystems = new ArrayList<>();
enum XPackRequested {
NOT_SPECIFIED,
YES,
NO,
MISMATCHED
}
XPackRequested xpackRequested = XPackRequested.NOT_SPECIFIED;
public SkipSectionBuilder withVersion(String version) {
this.version = version;
return this;
}
public SkipSectionBuilder withReason(String reason) {
this.reason = reason;
return this;
}
public SkipSectionBuilder withTestFeature(String featureName) {
this.testFeatures.add(featureName);
return this;
}
public void withXPack(boolean xpackRequired) {
if (xpackRequired && xpackRequested == XPackRequested.NO || xpackRequired == false && xpackRequested == XPackRequested.YES) {
xpackRequested = XPackRequested.MISMATCHED;
} else {
xpackRequested = xpackRequired ? XPackRequested.YES : XPackRequested.NO;
}
}
public SkipSectionBuilder withOs(String osName) {
this.operatingSystems.add(osName);
return this;
}
void validate(XContentLocation contentLocation) {
if ((Strings.hasLength(version) == false)
&& testFeatures.isEmpty()
&& operatingSystems.isEmpty()
&& xpackRequested == XPackRequested.NOT_SPECIFIED) {
throw new ParsingException(
contentLocation,
"at least one criteria (version, test features, os) is mandatory within a skip section"
);
}
if (Strings.hasLength(version) && Strings.hasLength(reason) == false) {
throw new ParsingException(contentLocation, "reason is mandatory within skip version section");
}
if (operatingSystems.isEmpty() == false && Strings.hasLength(reason) == false) {
throw new ParsingException(contentLocation, "reason is mandatory within skip version section");
}
// make feature "skip_os" mandatory if os is given, this is a temporary solution until language client tests know about os
if (operatingSystems.isEmpty() == false && testFeatures.contains("skip_os") == false) {
throw new ParsingException(contentLocation, "if os is specified, feature skip_os must be set");
}
if (xpackRequested == XPackRequested.MISMATCHED) {
throw new ParsingException(contentLocation, "either `xpack` or `no_xpack` can be present, not both");
}
}
public SkipSection build() {
final List<Predicate<ClientYamlTestExecutionContext>> skipCriteriaList;
// Check if the test runner supports all YAML framework features (see {@link Features}). If not, default to always skip this
// section.
if (Features.areAllSupported(testFeatures) == false) {
skipCriteriaList = List.of(SkipCriteria.SKIP_ALWAYS);
} else {
skipCriteriaList = new ArrayList<>();
if (xpackRequested == XPackRequested.YES || xpackRequested == XPackRequested.NO) {
skipCriteriaList.add(SkipCriteria.fromClusterModules(xpackRequested == XPackRequested.YES));
}
if (Strings.hasLength(version)) {
skipCriteriaList.add(SkipCriteria.fromVersionRange(version));
}
if (operatingSystems.isEmpty() == false) {
skipCriteriaList.add(SkipCriteria.fromOsList(operatingSystems));
}
}
return new SkipSection(skipCriteriaList, testFeatures, reason);
}
}
/**
* Parse a {@link SkipSection} if the next field is {@code skip}, otherwise returns {@link SkipSection#EMPTY}.
*/
public static SkipSection parseIfNext(XContentParser parser) throws IOException {
ParserUtils.advanceToFieldName(parser);
if ("skip".equals(parser.currentName())) {
SkipSection section = parse(parser);
parser.nextToken();
return section;
}
return EMPTY;
}
public static SkipSection parse(XContentParser parser) throws IOException {
return parseInternal(parser).build();
}
private static void parseFeature(String feature, SkipSectionBuilder builder) {
// #31403 introduced YAML test "features" to indicate if the cluster being tested has xpack installed (`xpack`)
// or if it does *not* have xpack installed (`no_xpack`). These are not test runner features, so now that we have
// "modular" skip criteria let's separate them. Eventually, these should move to their own skip section.
if (feature.equals("xpack")) {
builder.withXPack(true);
} else if (feature.equals("no_xpack")) {
builder.withXPack(false);
} else {
builder.withTestFeature(feature);
}
}
// package private for tests
static SkipSectionBuilder parseInternal(XContentParser parser) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new IllegalArgumentException(
"Expected ["
+ XContentParser.Token.START_OBJECT
+ ", found ["
+ parser.currentToken()
+ "], the skip section is not properly indented"
);
}
String currentFieldName = null;
XContentParser.Token token;
var builder = new SkipSectionBuilder();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token.isValue()) {
if ("version".equals(currentFieldName)) {
builder.withVersion(parser.text());
} else if ("reason".equals(currentFieldName)) {
builder.withReason(parser.text());
} else if ("features".equals(currentFieldName)) {
parseFeature(parser.text(), builder);
} else if ("os".equals(currentFieldName)) {
builder.withOs(parser.text());
} else {
throw new ParsingException(
parser.getTokenLocation(),
"field " + currentFieldName + " not supported within skip section"
);
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("features".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
parseFeature(parser.text(), builder);
}
} else if ("os".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
builder.withOs(parser.text());
}
}
}
}
parser.nextToken();
builder.validate(parser.getTokenLocation());
return builder;
}
public static final SkipSection EMPTY = new SkipSection();
private final List<Predicate<ClientYamlTestExecutionContext>> skipCriteriaList;
private final List<String> yamlRunnerFeatures;
private final String reason;
private SkipSection() {
this.skipCriteriaList = new ArrayList<>();
this.yamlRunnerFeatures = new ArrayList<>();
this.reason = null;
}
SkipSection(List<Predicate<ClientYamlTestExecutionContext>> skipCriteriaList, List<String> yamlRunnerFeatures, String reason) {
this.skipCriteriaList = skipCriteriaList;
this.yamlRunnerFeatures = yamlRunnerFeatures;
this.reason = reason;
}
public boolean yamlRunnerHasFeature(String feature) {
return yamlRunnerFeatures.contains(feature);
}
public String getReason() {
return reason;
}
public boolean skip(ClientYamlTestExecutionContext context) {
if (isEmpty()) {
return false;
}
return skipCriteriaList.stream().anyMatch(c -> c.test(context));
}
public boolean isEmpty() {
return EMPTY.equals(this);
}
public String getSkipMessage(String description) {
StringBuilder messageBuilder = new StringBuilder();
messageBuilder.append("[").append(description).append("] skipped,");
if (reason != null) {
messageBuilder.append(" reason: [").append(getReason()).append("]");
}
if (yamlRunnerFeatures.isEmpty() == false) {
messageBuilder.append(" unsupported features ").append(yamlRunnerFeatures);
}
return messageBuilder.toString();
}
}

16
test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
View file

@ -35,7 +35,7 @@ public class TeardownSection {
}
public static TeardownSection parse(XContentParser parser) throws IOException {
SkipSection skipSection = SkipSection.parseIfNext(parser);
PrerequisiteSection prerequisiteSection = PrerequisiteSection.parseIfNext(parser);
List<ExecutableSection> executableSections = new ArrayList<>();
while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
ParserUtils.advanceToFieldName(parser);
@ -50,21 +50,21 @@ public class TeardownSection {
}
parser.nextToken();
return new TeardownSection(skipSection, executableSections);
return new TeardownSection(prerequisiteSection, executableSections);
}
public static final TeardownSection EMPTY = new TeardownSection(SkipSection.EMPTY, Collections.emptyList());
public static final TeardownSection EMPTY = new TeardownSection(PrerequisiteSection.EMPTY, Collections.emptyList());
private final SkipSection skipSection;
private final PrerequisiteSection prerequisiteSection;
private final List<ExecutableSection> doSections;
TeardownSection(SkipSection skipSection, List<ExecutableSection> doSections) {
this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null");
TeardownSection(PrerequisiteSection prerequisiteSection, List<ExecutableSection> doSections) {
this.prerequisiteSection = Objects.requireNonNull(prerequisiteSection, "skip section cannot be null");
this.doSections = Collections.unmodifiableList(doSections);
}
public SkipSection getSkipSection() {
return skipSection;
public PrerequisiteSection getPrerequisiteSection() {
return prerequisiteSection;
}
public List<ExecutableSection> getDoSections() {

10
test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
View file

@ -69,7 +69,7 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa
assertThat(testSection, notNullValue());
assertThat(testSection.getName(), equalTo("First test section"));
assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
assertThat(testSection.getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(testSection.getExecutableSections().size(), equalTo(1));
DoSection doSection = (DoSection) testSection.getExecutableSections().get(0);
assertThat(doSection.getCatch(), equalTo("missing"));
@ -96,8 +96,8 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa
assertThat(testSection, notNullValue());
assertThat(testSection.getName(), equalTo("First test section"));
assertThat(testSection.getSkipSection(), notNullValue());
assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259"));
assertThat(testSection.getPrerequisiteSection(), notNullValue());
assertThat(testSection.getPrerequisiteSection().skipReason, equalTo("Update doesn't return metadata fields, waiting for #3259"));
assertThat(testSection.getExecutableSections().size(), equalTo(2));
DoSection doSection = (DoSection) testSection.getExecutableSections().get(0);
assertThat(doSection.getCatch(), equalTo("missing"));
@ -130,7 +130,7 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa
assertThat(testSection, notNullValue());
assertThat(testSection.getName(), equalTo("Basic"));
assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
assertThat(testSection.getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(testSection.getExecutableSections().size(), equalTo(2));
DoSection doSection = (DoSection) testSection.getExecutableSections().get(0);
assertThat(doSection.getCatch(), nullValue());
@ -181,7 +181,7 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa
assertThat(testSection, notNullValue());
assertThat(testSection.getName(), equalTo("Basic"));
assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
assertThat(testSection.getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(testSection.getExecutableSections().size(), equalTo(10));
DoSection doSection = (DoSection) testSection.getExecutableSections().get(0);

208
test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
View file

@ -34,6 +34,51 @@ import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentParserTestCase {
public void testParseTestSetupWithSkip() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
---
setup:
- skip:
version: "8.7.00 - 8.9.99"
reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests"
---
date:
- skip:
version: " - 8.1.99"
reason: tsdb indexing changed in 8.2.0
- do:
indices.get_mapping:
index: test_index
- match: {test_index.test_type.properties.text.type: string}
- match: {test_index.test_type.properties.text.analyzer: whitespace}
""");
ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), Optional.empty(), parser);
assertThat(restTestSuite, notNullValue());
assertThat(restTestSuite.getName(), equalTo(getTestName()));
assertThat(restTestSuite.getFile().isPresent(), equalTo(false));
assertThat(restTestSuite.getSetupSection(), notNullValue());
assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false));
assertThat(restTestSuite.getSetupSection().getPrerequisiteSection().isEmpty(), equalTo(false));
assertThat(restTestSuite.getSetupSection().getExecutableSections().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().size(), equalTo(1));
assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("date"));
assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(false));
assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(3));
assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_mapping"));
assertThat(doSection.getApiCallSection().getParams().size(), equalTo(1));
assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
}
public void testParseTestSetupTeardownAndSections() throws Exception {
final boolean includeSetup = randomBoolean();
final boolean includeTeardown = randomBoolean();
@ -92,7 +137,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
assertThat(restTestSuite.getSetupSection(), notNullValue());
if (includeSetup) {
assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false));
assertThat(restTestSuite.getSetupSection().getSkipSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getSetupSection().getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getSetupSection().getExecutableSections().size(), equalTo(1));
final ExecutableSection maybeDoSection = restTestSuite.getSetupSection().getExecutableSections().get(0);
assertThat(maybeDoSection, instanceOf(DoSection.class));
@ -107,7 +152,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
assertThat(restTestSuite.getTeardownSection(), notNullValue());
if (includeTeardown) {
assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false));
assertThat(restTestSuite.getTeardownSection().getSkipSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTeardownSection().getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTeardownSection().getDoSections().size(), equalTo(1));
assertThat(
((DoSection) restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getApi(),
@ -128,7 +173,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
assertThat(restTestSuite.getTestSections().size(), equalTo(2));
assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Get index mapping"));
assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(3));
assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
@ -145,9 +190,9 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
assertThat(matchAssertion.getExpectedValue().toString(), equalTo("whitespace"));
assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Get type mapping - pre 6.0"));
assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(false));
assertThat(restTestSuite.getTestSections().get(1).getPrerequisiteSection().isEmpty(), equalTo(false));
assertThat(
restTestSuite.getTestSections().get(1).getSkipSection().getReason(),
restTestSuite.getTestSections().get(1).getPrerequisiteSection().skipReason,
equalTo("for newer versions the index name is always returned")
);
@ -209,7 +254,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
assertThat(restTestSuite.getTestSections().size(), equalTo(1));
assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Index with ID"));
assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(12));
assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
@ -322,7 +367,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
assertThat(restTestSuite.getTestSections().size(), equalTo(2));
assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Missing document (partial doc)"));
assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(2));
assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
@ -339,7 +384,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Missing document (script)"));
assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(1).getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().size(), equalTo(2));
assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(0), instanceOf(DoSection.class));
assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().get(1), instanceOf(DoSection.class));
@ -418,9 +463,44 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
assertThat(restTestSuite.getTestSections().size(), equalTo(1));
assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Broken on some os"));
assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(false));
assertThat(restTestSuite.getTestSections().get(0).getSkipSection().getReason(), equalTo("not supported"));
assertThat(restTestSuite.getTestSections().get(0).getSkipSection().yamlRunnerHasFeature("skip_os"), equalTo(true));
assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(false));
assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().skipReason, containsString("not supported"));
assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().hasYamlRunnerFeature("skip_os"), equalTo(true));
}
public void testParseSkipAndRequireClusterFeatures() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
"Broken on some os":
- skip:
cluster_features: [unsupported-feature1, unsupported-feature2]
reason: "unsupported-features are not supported"
- requires:
cluster_features: required-feature1
reason: "required-feature1 is required"
- do:
indices.get_mapping:
index: test_index
type: test_type
- match: {test_type.properties.text.type: string}
- match: {test_type.properties.text.analyzer: whitespace}
""");
ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), Optional.empty(), parser);
assertThat(restTestSuite, notNullValue());
assertThat(restTestSuite.getName(), equalTo(getTestName()));
assertThat(restTestSuite.getFile().isPresent(), equalTo(false));
assertThat(restTestSuite.getTestSections().size(), equalTo(1));
assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Broken on some os"));
assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(false));
assertThat(
restTestSuite.getTestSections().get(0).getPrerequisiteSection().skipReason,
equalTo("unsupported-features are not supported")
);
assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().requireReason, equalTo("required-feature1 is required"));
}
public void testParseFileWithSingleTestSection() throws Exception {
@ -453,7 +533,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
assertThat(restTestSuite.getTestSections().size(), equalTo(1));
assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Index with ID"));
assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(0).getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(2));
assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
@ -473,7 +553,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
ClientYamlTestSection section = new ClientYamlTestSection(
new XContentLocation(0, 0),
"test",
SkipSection.EMPTY,
PrerequisiteSection.EMPTY,
Collections.singletonList(doSection)
);
ClientYamlTestSuite clientYamlTestSuite = new ClientYamlTestSuite(
@ -492,11 +572,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
doSection.setExpectedWarningHeaders(singletonList("foo"));
doSection.setApiCallSection(new ApiCallSection("test"));
ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection);
Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
assertThat(e.getMessage(), containsString(Strings.format("""
api/name:
attempted to add a [do] with a [warnings] section without a corresponding ["skip": "features": "warnings"] \
attempted to add a [do] with a [warnings] section without a corresponding ["requires": "test_runner_features": "warnings"] \
so runners that do not support the [warnings] section can skip the test at line [%d]\
""", lineNumber)));
}
@ -506,11 +586,12 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
doSection.setExpectedWarningHeadersRegex(singletonList(Pattern.compile("foo")));
doSection.setApiCallSection(new ApiCallSection("test"));
ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection);
Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
assertThat(e.getMessage(), containsString(Strings.format("""
api/name:
attempted to add a [do] with a [warnings_regex] section without a corresponding ["skip": "features": "warnings_regex"] \
attempted to add a [do] with a [warnings_regex] section without a corresponding \
["requires": "test_runner_features": "warnings_regex"] \
so runners that do not support the [warnings_regex] section can skip the test at line [%d]\
""", lineNumber)));
}
@ -520,11 +601,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
doSection.setAllowedWarningHeaders(singletonList("foo"));
doSection.setApiCallSection(new ApiCallSection("test"));
ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection);
Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
assertThat(e.getMessage(), containsString(Strings.format("""
api/name:
attempted to add a [do] with a [allowed_warnings] section without a corresponding ["skip": "features": \
attempted to add a [do] with a [allowed_warnings] section without a corresponding ["requires": "test_runner_features": \
"allowed_warnings"] so runners that do not support the [allowed_warnings] section can skip the test at \
line [%d]\
""", lineNumber)));
@ -535,11 +616,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
doSection.setAllowedWarningHeadersRegex(singletonList(Pattern.compile("foo")));
doSection.setApiCallSection(new ApiCallSection("test"));
ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection);
Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
assertThat(e.getMessage(), containsString(Strings.format("""
api/name:
attempted to add a [do] with a [allowed_warnings_regex] section without a corresponding ["skip": "features": \
attempted to add a [do] with a [allowed_warnings_regex] section without a corresponding ["requires": "test_runner_features": \
"allowed_warnings_regex"] so runners that do not support the [allowed_warnings_regex] section can skip the test \
at line [%d]\
""", lineNumber)));
@ -551,11 +632,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
ApiCallSection apiCallSection = new ApiCallSection("test");
apiCallSection.addHeaders(Collections.singletonMap("header", "value"));
doSection.setApiCallSection(apiCallSection);
ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection);
Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
assertThat(e.getMessage(), containsString(Strings.format("""
api/name:
attempted to add a [do] with a [headers] section without a corresponding ["skip": "features": "headers"] \
attempted to add a [do] with a [headers] section without a corresponding ["requires": "test_runner_features": "headers"] \
so runners that do not support the [headers] section can skip the test at line [%d]\
""", lineNumber)));
}
@ -566,11 +647,12 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
ApiCallSection apiCall = new ApiCallSection("test");
apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
doSection.setApiCallSection(apiCall);
ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, doSection);
Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
assertThat(e.getMessage(), containsString(Strings.format("""
api/name:
attempted to add a [do] with a [node_selector] section without a corresponding ["skip": "features": "node_selector"] \
attempted to add a [do] with a [node_selector] section without a corresponding \
["requires": "test_runner_features": "node_selector"] \
so runners that do not support the [node_selector] section can skip the test at line [%d]\
""", lineNumber)));
}
@ -582,11 +664,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
randomAlphaOfLength(randomIntBetween(3, 30)),
randomDouble()
);
ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, containsAssertion);
ClientYamlTestSuite testSuite = createTestSuite(PrerequisiteSection.EMPTY, containsAssertion);
Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
assertThat(e.getMessage(), containsString(Strings.format("""
api/name:
attempted to add a [contains] assertion without a corresponding ["skip": "features": "contains"] \
attempted to add a [contains] assertion without a corresponding ["requires": "test_runner_features": "contains"] \
so runners that do not support the [contains] assertion can skip the test at line [%d]\
""", lineNumber)));
}
@ -604,7 +686,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
new ClientYamlTestSection(
new XContentLocation(0, 0),
"section1",
SkipSection.EMPTY,
PrerequisiteSection.EMPTY,
Collections.singletonList(containsAssertion)
)
);
@ -625,7 +707,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
doSection.setApiCallSection(apiCall);
doSections.add(doSection);
}
sections.add(new ClientYamlTestSection(new XContentLocation(0, 0), "section2", SkipSection.EMPTY, doSections));
sections.add(new ClientYamlTestSection(new XContentLocation(0, 0), "section2", PrerequisiteSection.EMPTY, doSections));
ClientYamlTestSuite testSuite = new ClientYamlTestSuite(
"api",
@ -638,23 +720,29 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
assertEquals(Strings.format("""
api/name:
attempted to add a [contains] assertion without a corresponding ["skip": "features": "contains"] so runners that \
do not support the [contains] assertion can skip the test at line [%d],
attempted to add a [do] with a [warnings] section without a corresponding ["skip": "features": "warnings"] so runners \
that do not support the [warnings] section can skip the test at line [%d],
attempted to add a [do] with a [node_selector] section without a corresponding ["skip": "features": "node_selector"] so \
runners that do not support the [node_selector] section can skip the test \
at line [%d]\
attempted to add a [contains] assertion without a corresponding \
["requires": "test_runner_features": "contains"] \
so runners that do not support the [contains] assertion can skip the test at line [%d],
attempted to add a [do] with a [warnings] section without a corresponding \
["requires": "test_runner_features": "warnings"] \
so runners that do not support the [warnings] section can skip the test at line [%d],
attempted to add a [do] with a [node_selector] section without a corresponding \
["requires": "test_runner_features": "node_selector"] \
so runners that do not support the [node_selector] section can skip the test at line [%d]\
""", firstLineNumber, secondLineNumber, thirdLineNumber), e.getMessage());
}
private static PrerequisiteSection createPrerequisiteSection(String yamlTestRunnerFeature) {
return new PrerequisiteSection(emptyList(), null, emptyList(), null, singletonList(yamlTestRunnerFeature));
}
public void testAddingDoWithWarningWithSkip() {
int lineNumber = between(1, 10000);
DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
doSection.setExpectedWarningHeaders(singletonList("foo"));
doSection.setApiCallSection(new ApiCallSection("test"));
SkipSection skipSection = new SkipSection(emptyList(), singletonList("warnings"), null);
createTestSuite(skipSection, doSection).validate();
PrerequisiteSection prerequisiteSection = createPrerequisiteSection("warnings");
createTestSuite(prerequisiteSection, doSection).validate();
}
public void testAddingDoWithWarningRegexWithSkip() {
@ -662,86 +750,86 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
doSection.setExpectedWarningHeadersRegex(singletonList(Pattern.compile("foo")));
doSection.setApiCallSection(new ApiCallSection("test"));
SkipSection skipSection = new SkipSection(emptyList(), singletonList("warnings_regex"), null);
createTestSuite(skipSection, doSection).validate();
PrerequisiteSection prerequisiteSection = createPrerequisiteSection("warnings_regex");
createTestSuite(prerequisiteSection, doSection).validate();
}
public void testAddingDoWithNodeSelectorWithSkip() {
int lineNumber = between(1, 10000);
SkipSection skipSection = new SkipSection(emptyList(), singletonList("node_selector"), null);
PrerequisiteSection prerequisiteSection = createPrerequisiteSection("node_selector");
DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
ApiCallSection apiCall = new ApiCallSection("test");
apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
doSection.setApiCallSection(apiCall);
createTestSuite(skipSection, doSection).validate();
createTestSuite(prerequisiteSection, doSection).validate();
}
public void testAddingDoWithHeadersWithSkip() {
int lineNumber = between(1, 10000);
SkipSection skipSection = new SkipSection(emptyList(), singletonList("headers"), null);
PrerequisiteSection prerequisiteSection = createPrerequisiteSection("headers");
DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
ApiCallSection apiCallSection = new ApiCallSection("test");
apiCallSection.addHeaders(singletonMap("foo", "bar"));
doSection.setApiCallSection(apiCallSection);
createTestSuite(skipSection, doSection).validate();
createTestSuite(prerequisiteSection, doSection).validate();
}
public void testAddingContainsWithSkip() {
int lineNumber = between(1, 10000);
SkipSection skipSection = new SkipSection(emptyList(), singletonList("contains"), null);
PrerequisiteSection prerequisiteSection = createPrerequisiteSection("contains");
ContainsAssertion containsAssertion = new ContainsAssertion(
new XContentLocation(lineNumber, 0),
randomAlphaOfLength(randomIntBetween(3, 30)),
randomDouble()
);
createTestSuite(skipSection, containsAssertion).validate();
createTestSuite(prerequisiteSection, containsAssertion).validate();
}
public void testAddingCloseToWithSkip() {
int lineNumber = between(1, 10000);
SkipSection skipSection = new SkipSection(emptyList(), singletonList("close_to"), null);
PrerequisiteSection prerequisiteSection = createPrerequisiteSection("close_to");
CloseToAssertion closeToAssertion = new CloseToAssertion(
new XContentLocation(lineNumber, 0),
randomAlphaOfLength(randomIntBetween(3, 30)),
randomDouble(),
randomDouble()
);
createTestSuite(skipSection, closeToAssertion).validate();
createTestSuite(prerequisiteSection, closeToAssertion).validate();
}
public void testAddingIsAfterWithSkip() {
int lineNumber = between(1, 10000);
SkipSection skipSection = new SkipSection(emptyList(), singletonList("is_after"), null);
PrerequisiteSection prerequisiteSection = createPrerequisiteSection("is_after");
IsAfterAssertion isAfterAssertion = new IsAfterAssertion(
new XContentLocation(lineNumber, 0),
randomAlphaOfLength(randomIntBetween(3, 30)),
randomInstantBetween(Instant.ofEpochSecond(0L), Instant.ofEpochSecond(3000000000L))
);
createTestSuite(skipSection, isAfterAssertion).validate();
createTestSuite(prerequisiteSection, isAfterAssertion).validate();
}
private static ClientYamlTestSuite createTestSuite(SkipSection skipSection, ExecutableSection executableSection) {
private static ClientYamlTestSuite createTestSuite(PrerequisiteSection prerequisiteSection, ExecutableSection executableSection) {
final SetupSection setupSection;
final TeardownSection teardownSection;
final ClientYamlTestSection clientYamlTestSection;
switch (randomIntBetween(0, 4)) {
case 0 -> {
setupSection = new SetupSection(skipSection, Collections.emptyList());
setupSection = new SetupSection(prerequisiteSection, Collections.emptyList());
teardownSection = TeardownSection.EMPTY;
clientYamlTestSection = new ClientYamlTestSection(
new XContentLocation(0, 0),
"test",
SkipSection.EMPTY,
PrerequisiteSection.EMPTY,
Collections.singletonList(executableSection)
);
}
case 1 -> {
setupSection = SetupSection.EMPTY;
teardownSection = new TeardownSection(skipSection, Collections.emptyList());
teardownSection = new TeardownSection(prerequisiteSection, Collections.emptyList());
clientYamlTestSection = new ClientYamlTestSection(
new XContentLocation(0, 0),
"test",
SkipSection.EMPTY,
PrerequisiteSection.EMPTY,
Collections.singletonList(executableSection)
);
}
@ -751,27 +839,27 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
clientYamlTestSection = new ClientYamlTestSection(
new XContentLocation(0, 0),
"test",
skipSection,
prerequisiteSection,
Collections.singletonList(executableSection)
);
}
case 3 -> {
setupSection = new SetupSection(skipSection, Collections.singletonList(executableSection));
setupSection = new SetupSection(prerequisiteSection, Collections.singletonList(executableSection));
teardownSection = TeardownSection.EMPTY;
clientYamlTestSection = new ClientYamlTestSection(
new XContentLocation(0, 0),
"test",
SkipSection.EMPTY,
PrerequisiteSection.EMPTY,
randomBoolean() ? Collections.emptyList() : Collections.singletonList(executableSection)
);
}
case 4 -> {
setupSection = SetupSection.EMPTY;
teardownSection = new TeardownSection(skipSection, Collections.singletonList(executableSection));
teardownSection = new TeardownSection(prerequisiteSection, Collections.singletonList(executableSection));
clientYamlTestSection = new ClientYamlTestSection(
new XContentLocation(0, 0),
"test",
SkipSection.EMPTY,
PrerequisiteSection.EMPTY,
randomBoolean() ? Collections.emptyList() : Collections.singletonList(executableSection)
);
}

630
test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSectionTests.java Normal file
View file

@ -0,0 +1,630 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.Version;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.core.Strings;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.yaml.YamlXContent;
import org.junit.AssumptionViolatedException;
import java.io.IOException;
import java.util.List;
import java.util.Set;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.emptyOrNullString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class PrerequisiteSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
public void testSkipVersionMultiRange() {
PrerequisiteSection section = new PrerequisiteSection(
List.of(Prerequisites.skipOnVersionRange("6.0.0 - 6.1.0, 7.1.0 - 7.5.0")),
"foobar",
emptyList(),
"foobar",
emptyList()
);
var outOfRangeMockContext = mock(ClientYamlTestExecutionContext.class);
when(outOfRangeMockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString()))
.thenReturn(Set.of("6.2.0"))
.thenReturn(Set.of("7.0.0"))
.thenReturn(Set.of("7.6.0"));
assertFalse(section.skipCriteriaMet(outOfRangeMockContext));
assertFalse(section.skipCriteriaMet(outOfRangeMockContext));
assertFalse(section.skipCriteriaMet(outOfRangeMockContext));
assertFalse(section.skipCriteriaMet(outOfRangeMockContext));
var inRangeMockContext = mock(ClientYamlTestExecutionContext.class);
when(inRangeMockContext.nodesVersions()).thenReturn(Set.of("6.0.0"))
.thenReturn(Set.of("6.1.0"))
.thenReturn(Set.of("7.1.0"))
.thenReturn(Set.of("7.5.0"));
assertTrue(section.skipCriteriaMet(inRangeMockContext));
assertTrue(section.skipCriteriaMet(inRangeMockContext));
assertTrue(section.skipCriteriaMet(inRangeMockContext));
assertTrue(section.skipCriteriaMet(inRangeMockContext));
}
public void testSkipVersionMultiOpenRange() {
var section = new PrerequisiteSection(
List.of(Prerequisites.skipOnVersionRange("- 7.1.0, 7.2.0 - 7.5.0, 8.0.0 -")),
"foobar",
emptyList(),
"foobar",
emptyList()
);
var outOfRangeMockContext = mock(ClientYamlTestExecutionContext.class);
when(outOfRangeMockContext.nodesVersions()).thenReturn(Set.of("7.1.1")).thenReturn(Set.of("7.6.0"));
assertFalse(section.skipCriteriaMet(outOfRangeMockContext));
assertFalse(section.skipCriteriaMet(outOfRangeMockContext));
var inRangeMockContext = mock(ClientYamlTestExecutionContext.class);
when(inRangeMockContext.nodesVersions()).thenReturn(Set.of("7.0.0"))
.thenReturn(Set.of("7.3.0"))
.thenReturn(Set.of("8.0.0"))
.thenReturn(Set.of(Version.CURRENT.toString()));
assertTrue(section.skipCriteriaMet(inRangeMockContext));
assertTrue(section.skipCriteriaMet(inRangeMockContext));
assertTrue(section.skipCriteriaMet(inRangeMockContext));
assertTrue(section.skipCriteriaMet(inRangeMockContext));
}
public void testSkipVersion() {
PrerequisiteSection section = new PrerequisiteSection(
List.of(Prerequisites.skipOnVersionRange("6.0.0 - 6.1.0")),
"foobar",
emptyList(),
"foobar",
emptyList()
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString()))
.thenReturn(Set.of("6.0.0"))
.thenReturn(Set.of("6.0.0", "6.1.0"))
.thenReturn(Set.of("6.0.0", "5.2.0"));
assertFalse(section.skipCriteriaMet(mockContext));
assertTrue(section.skipCriteriaMet(mockContext));
assertTrue(section.skipCriteriaMet(mockContext));
assertFalse(section.skipCriteriaMet(mockContext));
}
public void testSkipVersionWithTestFeatures() {
PrerequisiteSection section = new PrerequisiteSection(
List.of(Prerequisites.skipOnVersionRange("6.0.0 - 6.1.0")),
"foobar",
emptyList(),
"foobar",
singletonList("warnings")
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString())).thenReturn(Set.of("6.0.0"));
assertFalse(section.skipCriteriaMet(mockContext));
assertTrue(section.skipCriteriaMet(mockContext));
}
public void testSkipTestFeatures() {
var section = new PrerequisiteSection.PrerequisiteSectionBuilder().requireYamlRunnerFeature("boom").build();
assertFalse(section.requiresCriteriaMet(mock(ClientYamlTestExecutionContext.class)));
}
public void testSkipTestFeaturesOverridesAnySkipCriteria() {
var section = new PrerequisiteSection.PrerequisiteSectionBuilder().skipIfOs("test-os").requireYamlRunnerFeature("boom").build();
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.os()).thenReturn("test-os");
// Skip even if OS is matching
assertFalse(section.skipCriteriaMet(mockContext));
assertFalse(section.requiresCriteriaMet(mockContext));
}
public void testSkipOs() {
PrerequisiteSection section = new PrerequisiteSection.PrerequisiteSectionBuilder().skipIfOs("windows95")
.skipIfOs("debian-5")
.build();
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.os()).thenReturn("debian-5");
assertTrue(section.skipCriteriaMet(mockContext));
when(mockContext.os()).thenReturn("windows95");
assertTrue(section.skipCriteriaMet(mockContext));
when(mockContext.os()).thenReturn("ms-dos");
assertFalse(section.skipCriteriaMet(mockContext));
assertTrue(section.requiresCriteriaMet(mockContext));
}
public void testSkipOsWithTestFeatures() {
PrerequisiteSection section = new PrerequisiteSection.PrerequisiteSectionBuilder().requireYamlRunnerFeature("warnings")
.skipIfOs("windows95")
.skipIfOs("debian-5")
.build();
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.os()).thenReturn("debian-5");
assertTrue(section.skipCriteriaMet(mockContext));
when(mockContext.os()).thenReturn("windows95");
assertTrue(section.skipCriteriaMet(mockContext));
when(mockContext.os()).thenReturn("ms-dos");
assertFalse(section.skipCriteriaMet(mockContext));
}
public void testBuildMessage() {
PrerequisiteSection section = new PrerequisiteSection(
List.of(Prerequisites.skipOnVersionRange("6.0.0 - 6.1.0")),
"unsupported",
emptyList(),
"required",
singletonList("warnings")
);
assertEquals("[FOOBAR] skipped, reason: [unsupported] unsupported features [warnings]", section.buildMessage("FOOBAR", true));
assertEquals("[FOOBAR] skipped, reason: [required] unsupported features [warnings]", section.buildMessage("FOOBAR", false));
section = new PrerequisiteSection(emptyList(), "unsupported", emptyList(), "required", emptyList());
assertEquals("[FOOBAR] skipped, reason: [unsupported]", section.buildMessage("FOOBAR", true));
assertEquals("[FOOBAR] skipped, reason: [required]", section.buildMessage("FOOBAR", false));
section = new PrerequisiteSection(emptyList(), null, emptyList(), null, singletonList("warnings"));
assertEquals("[FOOBAR] skipped, unsupported features [warnings]", section.buildMessage("FOOBAR", true));
assertEquals("[FOOBAR] skipped, unsupported features [warnings]", section.buildMessage("FOOBAR", false));
}
public void testParseNoPrerequisites() throws IOException {
parser = createParser(YamlXContent.yamlXContent, """
do:
something
""");
var skipSectionBuilder = PrerequisiteSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
var skipSection = skipSectionBuilder.build();
assertThat(skipSection.isEmpty(), equalTo(true));
// Ensure the input (bogus execute section) was not consumed
var next = ParserUtils.parseField(parser);
assertThat(next, notNullValue());
assertThat(parser.nextToken(), nullValue());
}
public void testParseSkipSectionVersionNoFeature() throws Exception {
Version version = VersionUtils.randomVersion(random());
parser = createParser(YamlXContent.yamlXContent, Strings.format("""
version: " - %s"
reason: Delete ignores the parent param""", version));
var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder();
PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, not(emptyOrNullString()));
assertThat(skipSectionBuilder.requiredYamlRunnerFeatures.size(), equalTo(0));
assertThat(skipSectionBuilder.skipReason, equalTo("Delete ignores the parent param"));
}
public void testParseSkipSectionFeatureNoVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, "features: regex");
var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder();
PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, contains("regex"));
assertThat(skipSectionBuilder.skipReason, nullValue());
assertThat(skipSectionBuilder.xpackRequired, is(PrerequisiteSection.PrerequisiteSectionBuilder.XPackRequired.NOT_SPECIFIED));
}
public void testParseXPackFeature() throws IOException {
parser = createParser(YamlXContent.yamlXContent, "features: xpack");
var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder();
PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, empty());
assertThat(skipSectionBuilder.skipReason, nullValue());
assertThat(skipSectionBuilder.xpackRequired, is(PrerequisiteSection.PrerequisiteSectionBuilder.XPackRequired.YES));
}
public void testParseNoXPackFeature() throws IOException {
parser = createParser(YamlXContent.yamlXContent, "features: no_xpack");
var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder();
PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, empty());
assertThat(skipSectionBuilder.skipReason, nullValue());
assertThat(skipSectionBuilder.xpackRequired, is(PrerequisiteSection.PrerequisiteSectionBuilder.XPackRequired.NO));
}
public void testParseBothXPackFeatures() throws IOException {
parser = createParser(YamlXContent.yamlXContent, """
skip:
features: [xpack, no_xpack]
""");
var e = expectThrows(ParsingException.class, () -> PrerequisiteSection.parseInternal(parser));
assertThat(e.getMessage(), containsString("either [xpack] or [no_xpack] can be present, not both"));
}
public void testParseSkipSectionFeaturesNoVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, "features: [regex1,regex2,regex3]");
var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder();
PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, contains("regex1", "regex2", "regex3"));
assertThat(skipSectionBuilder.skipReason, nullValue());
}
public void testParseSkipSectionBothFeatureAndVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
version: " - 0.90.2"
features: regex
reason: Delete ignores the parent param""");
var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder();
PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder);
assertThat(skipSectionBuilder.skipVersionRange, not(emptyOrNullString()));
assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, contains("regex"));
assertThat(skipSectionBuilder.skipReason, equalTo("Delete ignores the parent param"));
}
public void testParseSkipSectionNoReason() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
skip:
version: " - 0.90.2"
""");
Exception e = expectThrows(ParsingException.class, () -> PrerequisiteSection.parseInternal(parser));
assertThat(e.getMessage(), is("reason is mandatory within skip version section"));
}
public void testParseSkipSectionNoVersionNorFeature() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
skip:
reason: Delete ignores the parent param
""");
Exception e = expectThrows(ParsingException.class, () -> PrerequisiteSection.parseInternal(parser));
assertThat(
e.getMessage(),
is("at least one criteria (version, cluster features, runner features, os) is mandatory within a skip section")
);
}
public void testParseSkipSectionOsNoVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
features: ["skip_os", "some_feature"]
os: debian-9
reason: memory accounting broken, see gh#xyz
""");
var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder();
PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, hasSize(2));
assertThat(skipSectionBuilder.skipOperatingSystems, contains("debian-9"));
assertThat(skipSectionBuilder.skipReason, is("memory accounting broken, see gh#xyz"));
}
public void testParseSkipSectionOsListNoVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
features: skip_os
os: [debian-9,windows-95,ms-dos]
reason: see gh#xyz
""");
var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder();
PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, hasSize(1));
assertThat(skipSectionBuilder.skipOperatingSystems, containsInAnyOrder("debian-9", "windows-95", "ms-dos"));
assertThat(skipSectionBuilder.skipReason, is("see gh#xyz"));
}
public void testParseSkipSectionOsListTestFeaturesInRequires() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
- requires:
test_runner_features: skip_os
reason: skip_os is needed for skip based on os
- skip:
os: [debian-9,windows-95,ms-dos]
reason: see gh#xyz
""");
var skipSectionBuilder = PrerequisiteSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.requiredYamlRunnerFeatures, hasSize(1));
assertThat(skipSectionBuilder.skipOperatingSystems, containsInAnyOrder("debian-9", "windows-95", "ms-dos"));
assertThat(skipSectionBuilder.skipReason, is("see gh#xyz"));
assertThat(parser.currentToken(), equalTo(XContentParser.Token.END_ARRAY));
assertThat(parser.nextToken(), nullValue());
}
public void testParseSkipSectionOsNoFeatureNoVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
skip:
os: debian-9
reason: memory accounting broken, see gh#xyz
""");
Exception e = expectThrows(ParsingException.class, () -> PrerequisiteSection.parseInternal(parser));
assertThat(e.getMessage(), is("if os is specified, test runner feature [skip_os] must be set"));
}
public void testParseRequireSectionClusterFeatures() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
cluster_features: needed-feature
reason: test skipped when cluster lacks needed-feature
""");
var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder();
PrerequisiteSection.parseRequiresSection(parser, skipSectionBuilder);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.requiredClusterFeatures, contains("needed-feature"));
assertThat(skipSectionBuilder.requiresReason, is("test skipped when cluster lacks needed-feature"));
}
public void testParseSkipSectionClusterFeatures() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
cluster_features: undesired-feature
reason: test skipped when undesired-feature is present
""");
var skipSectionBuilder = new PrerequisiteSection.PrerequisiteSectionBuilder();
PrerequisiteSection.parseSkipSection(parser, skipSectionBuilder);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.skipClusterFeatures, contains("undesired-feature"));
assertThat(skipSectionBuilder.skipReason, is("test skipped when undesired-feature is present"));
}
public void testParseRequireAndSkipSectionsClusterFeatures() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
- requires:
cluster_features: needed-feature
reason: test needs needed-feature to run
- skip:
cluster_features: undesired-feature
reason: test cannot run when undesired-feature are present
""");
var skipSectionBuilder = PrerequisiteSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.skipClusterFeatures, contains("undesired-feature"));
assertThat(skipSectionBuilder.requiredClusterFeatures, contains("needed-feature"));
assertThat(skipSectionBuilder.skipReason, is("test cannot run when undesired-feature are present"));
assertThat(skipSectionBuilder.requiresReason, is("test needs needed-feature to run"));
assertThat(parser.currentToken(), equalTo(XContentParser.Token.END_ARRAY));
assertThat(parser.nextToken(), nullValue());
}
public void testParseRequireAndSkipSectionMultipleClusterFeatures() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
- requires:
cluster_features: [needed-feature-1, needed-feature-2]
reason: test needs some to run
- skip:
cluster_features: [undesired-feature-1, undesired-feature-2]
reason: test cannot run when some are present
""");
var skipSectionBuilder = PrerequisiteSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.skipVersionRange, emptyOrNullString());
assertThat(skipSectionBuilder.skipClusterFeatures, containsInAnyOrder("undesired-feature-1", "undesired-feature-2"));
assertThat(skipSectionBuilder.requiredClusterFeatures, containsInAnyOrder("needed-feature-1", "needed-feature-2"));
assertThat(skipSectionBuilder.skipReason, is("test cannot run when some are present"));
assertThat(skipSectionBuilder.requiresReason, is("test needs some to run"));
assertThat(parser.currentToken(), equalTo(XContentParser.Token.END_ARRAY));
assertThat(parser.nextToken(), nullValue());
}
public void testParseSameRequireAndSkipClusterFeatures() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
- requires:
cluster_features: some-feature
reason: test needs some-feature to run
- skip:
cluster_features: some-feature
reason: test cannot run with some-feature
""");
var e = expectThrows(ParsingException.class, () -> PrerequisiteSection.parseInternal(parser));
assertThat(e.getMessage(), is("a cluster feature can be specified either in [requires] or [skip], not both"));
assertThat(parser.currentToken(), equalTo(XContentParser.Token.END_ARRAY));
assertThat(parser.nextToken(), nullValue());
}
public void testSkipClusterFeaturesAllRequiredMatch() {
PrerequisiteSection section = new PrerequisiteSection(
emptyList(),
"foobar",
List.of(Prerequisites.requireClusterFeatures(Set.of("required-feature-1", "required-feature-2"))),
"foobar",
emptyList()
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true);
when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(true);
assertFalse(section.skipCriteriaMet(mockContext));
assertTrue(section.requiresCriteriaMet(mockContext));
}
public void testSkipClusterFeaturesSomeRequiredMatch() {
PrerequisiteSection section = new PrerequisiteSection(
emptyList(),
"foobar",
List.of(Prerequisites.requireClusterFeatures(Set.of("required-feature-1", "required-feature-2"))),
"foobar",
emptyList()
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true);
when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(false);
assertFalse(section.skipCriteriaMet(mockContext));
assertFalse(section.requiresCriteriaMet(mockContext));
}
public void testSkipClusterFeaturesSomeToSkipMatch() {
PrerequisiteSection section = new PrerequisiteSection(
List.of(Prerequisites.skipOnClusterFeatures(Set.of("undesired-feature-1", "undesired-feature-2"))),
"foobar",
emptyList(),
"foobar",
emptyList()
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.clusterHasFeature("undesired-feature-1")).thenReturn(true);
assertTrue(section.skipCriteriaMet(mockContext));
}
public void testSkipClusterFeaturesNoneToSkipMatch() {
PrerequisiteSection section = new PrerequisiteSection(
List.of(Prerequisites.skipOnClusterFeatures(Set.of("undesired-feature-1", "undesired-feature-2"))),
"foobar",
emptyList(),
"foobar",
emptyList()
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
assertFalse(section.skipCriteriaMet(mockContext));
}
public void testSkipClusterFeaturesAllRequiredSomeToSkipMatch() {
PrerequisiteSection section = new PrerequisiteSection(
List.of(Prerequisites.skipOnClusterFeatures(Set.of("undesired-feature-1", "undesired-feature-2"))),
"foobar",
List.of(Prerequisites.requireClusterFeatures(Set.of("required-feature-1", "required-feature-2"))),
"foobar",
emptyList()
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true);
when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(true);
when(mockContext.clusterHasFeature("undesired-feature-1")).thenReturn(true);
assertTrue(section.skipCriteriaMet(mockContext));
assertTrue(section.requiresCriteriaMet(mockContext));
}
public void testSkipClusterFeaturesAllRequiredNoneToSkipMatch() {
PrerequisiteSection section = new PrerequisiteSection(
List.of(Prerequisites.skipOnClusterFeatures(Set.of("undesired-feature-1", "undesired-feature-2"))),
"foobar",
List.of(Prerequisites.requireClusterFeatures(Set.of("required-feature-1", "required-feature-2"))),
"foobar",
emptyList()
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true);
when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(true);
assertFalse(section.skipCriteriaMet(mockContext));
assertTrue(section.requiresCriteriaMet(mockContext));
}
public void evaluateEmpty() {
var section = new PrerequisiteSection(List.of(), "unsupported", List.of(), "required", List.of());
var mockContext = mock(ClientYamlTestExecutionContext.class);
section.evaluate(mockContext, "TEST");
}
public void evaluateRequiresCriteriaTrue() {
var section = new PrerequisiteSection(List.of(), "unsupported", List.of(Prerequisites.TRUE), "required", List.of());
var mockContext = mock(ClientYamlTestExecutionContext.class);
section.evaluate(mockContext, "TEST");
}
public void evaluateSkipCriteriaFalse() {
var section = new PrerequisiteSection(List.of(Prerequisites.FALSE), "unsupported", List.of(), "required", List.of());
var mockContext = mock(ClientYamlTestExecutionContext.class);
section.evaluate(mockContext, "TEST");
}
public void evaluateRequiresCriteriaFalse() {
var section = new PrerequisiteSection(
List.of(Prerequisites.FALSE),
"unsupported",
List.of(Prerequisites.FALSE),
"required",
List.of()
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
var e = expectThrows(AssumptionViolatedException.class, () -> section.evaluate(mockContext, "TEST"));
assertThat(e.getMessage(), equalTo("[TEST] skipped, reason: [required]"));
}
public void evaluateSkipCriteriaTrue() {
var section = new PrerequisiteSection(
List.of(Prerequisites.TRUE),
"unsupported",
List.of(Prerequisites.TRUE),
"required",
List.of()
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
var e = expectThrows(AssumptionViolatedException.class, () -> section.evaluate(mockContext, "TEST"));
assertThat(e.getMessage(), equalTo("[TEST] skipped, reason: [unsupported]"));
}
}

10
test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java
View file

@ -37,7 +37,7 @@ public class SetupSectionTests extends AbstractClientYamlTestFragmentParserTestC
SetupSection setupSection = SetupSection.parse(parser);
assertThat(setupSection, notNullValue());
assertThat(setupSection.getSkipSection().isEmpty(), equalTo(true));
assertThat(setupSection.getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(setupSection.getExecutableSections().size(), equalTo(2));
assertThat(setupSection.getExecutableSections().get(0), instanceOf(DoSection.class));
assertThat(((DoSection) setupSection.getExecutableSections().get(0)).getApiCallSection().getApi(), equalTo("index1"));
@ -60,7 +60,7 @@ public class SetupSectionTests extends AbstractClientYamlTestFragmentParserTestC
final SetupSection setupSection = SetupSection.parse(parser);
assertNotNull(setupSection);
assertTrue(setupSection.getSkipSection().isEmpty());
assertTrue(setupSection.getPrerequisiteSection().isEmpty());
assertThat(setupSection.getExecutableSections().size(), equalTo(5));
assertThat(setupSection.getExecutableSections().get(0), instanceOf(DoSection.class));
assertThat(((DoSection) setupSection.getExecutableSections().get(0)).getApiCallSection().getApi(), equalTo("cluster.state"));
@ -105,9 +105,9 @@ public class SetupSectionTests extends AbstractClientYamlTestFragmentParserTestC
SetupSection setupSection = SetupSection.parse(parser);
assertThat(setupSection, notNullValue());
assertThat(setupSection.getSkipSection().isEmpty(), equalTo(false));
assertThat(setupSection.getSkipSection(), notNullValue());
assertThat(setupSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259"));
assertThat(setupSection.getPrerequisiteSection().isEmpty(), equalTo(false));
assertThat(setupSection.getPrerequisiteSection(), notNullValue());
assertThat(setupSection.getPrerequisiteSection().skipReason, equalTo("Update doesn't return metadata fields, waiting for #3259"));
assertThat(setupSection.getExecutableSections().size(), equalTo(2));
assertThat(setupSection.getExecutableSections().get(0), instanceOf(DoSection.class));
assertThat(((DoSection) setupSection.getExecutableSections().get(0)).getApiCallSection().getApi(), equalTo("index1"));

311
test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
View file

@ -1,311 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.Version;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.core.Strings;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import org.elasticsearch.xcontent.yaml.YamlXContent;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.emptyOrNullString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class SkipSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
public void testSkipVersionMultiRange() {
SkipSection section = new SkipSection(
List.of(SkipCriteria.fromVersionRange("6.0.0 - 6.1.0, 7.1.0 - 7.5.0")),
Collections.emptyList(),
"foobar"
);
var outOfRangeMockContext = mock(ClientYamlTestExecutionContext.class);
when(outOfRangeMockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString()))
.thenReturn(Set.of("6.2.0"))
.thenReturn(Set.of("7.0.0"))
.thenReturn(Set.of("7.6.0"));
assertFalse(section.skip(outOfRangeMockContext));
assertFalse(section.skip(outOfRangeMockContext));
assertFalse(section.skip(outOfRangeMockContext));
assertFalse(section.skip(outOfRangeMockContext));
var inRangeMockContext = mock(ClientYamlTestExecutionContext.class);
when(inRangeMockContext.nodesVersions()).thenReturn(Set.of("6.0.0"))
.thenReturn(Set.of("6.1.0"))
.thenReturn(Set.of("7.1.0"))
.thenReturn(Set.of("7.5.0"));
assertTrue(section.skip(inRangeMockContext));
assertTrue(section.skip(inRangeMockContext));
assertTrue(section.skip(inRangeMockContext));
assertTrue(section.skip(inRangeMockContext));
}
public void testSkipVersionMultiOpenRange() {
var section = new SkipSection(
List.of(SkipCriteria.fromVersionRange("- 7.1.0, 7.2.0 - 7.5.0, 8.0.0 -")),
Collections.emptyList(),
"foobar"
);
var outOfRangeMockContext = mock(ClientYamlTestExecutionContext.class);
when(outOfRangeMockContext.nodesVersions()).thenReturn(Set.of("7.1.1")).thenReturn(Set.of("7.6.0"));
assertFalse(section.skip(outOfRangeMockContext));
assertFalse(section.skip(outOfRangeMockContext));
var inRangeMockContext = mock(ClientYamlTestExecutionContext.class);
when(inRangeMockContext.nodesVersions()).thenReturn(Set.of("7.0.0"))
.thenReturn(Set.of("7.3.0"))
.thenReturn(Set.of("8.0.0"))
.thenReturn(Set.of(Version.CURRENT.toString()));
assertTrue(section.skip(inRangeMockContext));
assertTrue(section.skip(inRangeMockContext));
assertTrue(section.skip(inRangeMockContext));
assertTrue(section.skip(inRangeMockContext));
}
public void testSkipVersion() {
SkipSection section = new SkipSection(List.of(SkipCriteria.fromVersionRange("6.0.0 - 6.1.0")), Collections.emptyList(), "foobar");
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString()))
.thenReturn(Set.of("6.0.0"))
.thenReturn(Set.of("6.0.0", "6.1.0"))
.thenReturn(Set.of("6.0.0", "5.2.0"));
assertFalse(section.skip(mockContext));
assertTrue(section.skip(mockContext));
assertTrue(section.skip(mockContext));
assertFalse(section.skip(mockContext));
}
public void testSkipVersionWithTestFeatures() {
SkipSection section = new SkipSection(
List.of(SkipCriteria.fromVersionRange("6.0.0 - 6.1.0")),
Collections.singletonList("warnings"),
"foobar"
);
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.nodesVersions()).thenReturn(Set.of(Version.CURRENT.toString())).thenReturn(Set.of("6.0.0"));
assertFalse(section.skip(mockContext));
assertTrue(section.skip(mockContext));
}
public void testSkipTestFeatures() {
var section = new SkipSection.SkipSectionBuilder().withTestFeature("boom").build();
assertTrue(section.skip(mock(ClientYamlTestExecutionContext.class)));
}
public void testSkipTestFeaturesOverridesAnySkipCriteria() {
var section = new SkipSection.SkipSectionBuilder().withOs("test-os").withTestFeature("boom").build();
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.os()).thenReturn("test-os");
// Skip even if OS is matching
assertTrue(section.skip(mockContext));
}
public void testSkipOs() {
SkipSection section = new SkipSection.SkipSectionBuilder().withOs("windows95").withOs("debian-5").build();
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.os()).thenReturn("debian-5");
assertTrue(section.skip(mockContext));
when(mockContext.os()).thenReturn("windows95");
assertTrue(section.skip(mockContext));
when(mockContext.os()).thenReturn("ms-dos");
assertFalse(section.skip(mockContext));
}
public void testSkipOsWithTestFeatures() {
SkipSection section = new SkipSection.SkipSectionBuilder().withTestFeature("warnings")
.withOs("windows95")
.withOs("debian-5")
.build();
var mockContext = mock(ClientYamlTestExecutionContext.class);
when(mockContext.os()).thenReturn("debian-5");
assertTrue(section.skip(mockContext));
when(mockContext.os()).thenReturn("windows95");
assertTrue(section.skip(mockContext));
when(mockContext.os()).thenReturn("ms-dos");
assertFalse(section.skip(mockContext));
}
public void testMessage() {
SkipSection section = new SkipSection(
List.of(SkipCriteria.fromVersionRange("6.0.0 - 6.1.0")),
Collections.singletonList("warnings"),
"foobar"
);
assertEquals("[FOOBAR] skipped, reason: [foobar] unsupported features [warnings]", section.getSkipMessage("FOOBAR"));
section = new SkipSection(List.of(), Collections.singletonList("warnings"), "foobar");
assertEquals("[FOOBAR] skipped, reason: [foobar] unsupported features [warnings]", section.getSkipMessage("FOOBAR"));
section = new SkipSection(List.of(), Collections.singletonList("warnings"), null);
assertEquals("[FOOBAR] skipped, unsupported features [warnings]", section.getSkipMessage("FOOBAR"));
}
public void testParseSkipSectionVersionNoFeature() throws Exception {
Version version = VersionUtils.randomVersion(random());
parser = createParser(YamlXContent.yamlXContent, Strings.format("""
version: " - %s"
reason: Delete ignores the parent param""", version));
var skipSectionBuilder = SkipSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.version, not(emptyOrNullString()));
assertThat(skipSectionBuilder.testFeatures.size(), equalTo(0));
assertThat(skipSectionBuilder.reason, equalTo("Delete ignores the parent param"));
}
public void testParseSkipSectionFeatureNoVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, "features: regex");
var skipSectionBuilder = SkipSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.version, emptyOrNullString());
assertThat(skipSectionBuilder.testFeatures, contains("regex"));
assertThat(skipSectionBuilder.reason, nullValue());
assertThat(skipSectionBuilder.xpackRequested, is(SkipSection.SkipSectionBuilder.XPackRequested.NOT_SPECIFIED));
}
public void testParseXPackFeature() throws IOException {
parser = createParser(YamlXContent.yamlXContent, "features: xpack");
var skipSectionBuilder = SkipSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.version, emptyOrNullString());
assertThat(skipSectionBuilder.testFeatures, empty());
assertThat(skipSectionBuilder.reason, nullValue());
assertThat(skipSectionBuilder.xpackRequested, is(SkipSection.SkipSectionBuilder.XPackRequested.YES));
}
public void testParseNoXPackFeature() throws IOException {
parser = createParser(YamlXContent.yamlXContent, "features: no_xpack");
var skipSectionBuilder = SkipSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.version, emptyOrNullString());
assertThat(skipSectionBuilder.testFeatures, empty());
assertThat(skipSectionBuilder.reason, nullValue());
assertThat(skipSectionBuilder.xpackRequested, is(SkipSection.SkipSectionBuilder.XPackRequested.NO));
}
public void testParseBothXPackFeatures() throws IOException {
parser = createParser(YamlXContent.yamlXContent, "features: [xpack, no_xpack]");
var e = expectThrows(ParsingException.class, () -> SkipSection.parseInternal(parser));
assertThat(e.getMessage(), containsString("either `xpack` or `no_xpack` can be present, not both"));
}
public void testParseSkipSectionFeaturesNoVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, "features: [regex1,regex2,regex3]");
var skipSectionBuilder = SkipSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.version, emptyOrNullString());
assertThat(skipSectionBuilder.testFeatures, contains("regex1", "regex2", "regex3"));
assertThat(skipSectionBuilder.reason, nullValue());
}
public void testParseSkipSectionBothFeatureAndVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
version: " - 0.90.2"
features: regex
reason: Delete ignores the parent param""");
var skipSectionBuilder = SkipSection.parseInternal(parser);
assertThat(skipSectionBuilder.version, not(emptyOrNullString()));
assertThat(skipSectionBuilder.testFeatures, contains("regex"));
assertThat(skipSectionBuilder.reason, equalTo("Delete ignores the parent param"));
}
public void testParseSkipSectionNoReason() throws Exception {
parser = createParser(YamlXContent.yamlXContent, "version: \" - 0.90.2\"\n");
Exception e = expectThrows(ParsingException.class, () -> SkipSection.parseInternal(parser));
assertThat(e.getMessage(), is("reason is mandatory within skip version section"));
}
public void testParseSkipSectionNoVersionNorFeature() throws Exception {
parser = createParser(YamlXContent.yamlXContent, "reason: Delete ignores the parent param\n");
Exception e = expectThrows(ParsingException.class, () -> SkipSection.parseInternal(parser));
assertThat(e.getMessage(), is("at least one criteria (version, test features, os) is mandatory within a skip section"));
}
public void testParseSkipSectionOsNoVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
features: ["skip_os", "some_feature"]
os: debian-9
reason: memory accounting broken, see gh#xyz
""");
var skipSectionBuilder = SkipSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.version, emptyOrNullString());
assertThat(skipSectionBuilder.testFeatures, hasSize(2));
assertThat(skipSectionBuilder.operatingSystems, contains("debian-9"));
assertThat(skipSectionBuilder.reason, is("memory accounting broken, see gh#xyz"));
}
public void testParseSkipSectionOsListNoVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
features: skip_os
os: [debian-9,windows-95,ms-dos]
reason: see gh#xyz
""");
var skipSectionBuilder = SkipSection.parseInternal(parser);
assertThat(skipSectionBuilder, notNullValue());
assertThat(skipSectionBuilder.version, emptyOrNullString());
assertThat(skipSectionBuilder.testFeatures, hasSize(1));
assertThat(skipSectionBuilder.operatingSystems, containsInAnyOrder("debian-9", "windows-95", "ms-dos"));
assertThat(skipSectionBuilder.reason, is("see gh#xyz"));
}
public void testParseSkipSectionOsNoFeatureNoVersion() throws Exception {
parser = createParser(YamlXContent.yamlXContent, """
os: debian-9
reason: memory accounting broken, see gh#xyz
""");
Exception e = expectThrows(ParsingException.class, () -> SkipSection.parseInternal(parser));
assertThat(e.getMessage(), is("if os is specified, feature skip_os must be set"));
}
}

6
test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
View file

@ -35,7 +35,7 @@ public class TeardownSectionTests extends AbstractClientYamlTestFragmentParserTe
TeardownSection section = TeardownSection.parse(parser);
assertThat(section, notNullValue());
assertThat(section.getSkipSection().isEmpty(), equalTo(true));
assertThat(section.getPrerequisiteSection().isEmpty(), equalTo(true));
assertThat(section.getDoSections().size(), equalTo(2));
assertThat(((DoSection) section.getDoSections().get(0)).getApiCallSection().getApi(), equalTo("delete"));
assertThat(((DoSection) section.getDoSections().get(1)).getApiCallSection().getApi(), equalTo("delete2"));
@ -62,8 +62,8 @@ public class TeardownSectionTests extends AbstractClientYamlTestFragmentParserTe
TeardownSection section = TeardownSection.parse(parser);
assertThat(section, notNullValue());
assertThat(section.getSkipSection().isEmpty(), equalTo(false));
assertThat(section.getSkipSection().getReason(), equalTo("there is a reason"));
assertThat(section.getPrerequisiteSection().isEmpty(), equalTo(false));
assertThat(section.getPrerequisiteSection().skipReason, equalTo("there is a reason"));
assertThat(section.getDoSections().size(), equalTo(2));
assertThat(((DoSection) section.getDoSections().get(0)).getApiCallSection().getApi(), equalTo("delete"));
assertThat(((DoSection) section.getDoSections().get(1)).getApiCallSection().getApi(), equalTo("delete2"));

2
x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncIT.java
View file

@ -52,7 +52,7 @@ public class EsqlClientYamlAsyncIT extends AbstractEsqlClientYamlIT {
ClientYamlTestSection modified = new ClientYamlTestSection(
candidate.getTestSection().getLocation(),
candidate.getTestSection().getName(),
candidate.getTestSection().getSkipSection(),
candidate.getTestSection().getPrerequisiteSection(),
candidate.getTestSection().getExecutableSections().stream().map(e -> modifyExecutableSection(e, modify)).toList()
);
result.add(new Object[] { new ClientYamlTestCandidate(candidate.getRestTestSuite(), modified) });

4
x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java
View file

@ -67,7 +67,7 @@ public abstract class CoreTestTranslater {
ClientYamlTestSection modified = new ClientYamlTestSection(
candidate.getTestSection().getLocation(),
candidate.getTestSection().getName(),
candidate.getTestSection().getSkipSection(),
candidate.getTestSection().getPrerequisiteSection(),
candidate.getTestSection().getExecutableSections()
);
result.add(new Object[] { new ClientYamlTestCandidate(suite.modified, modified) });
@ -169,7 +169,7 @@ public abstract class CoreTestTranslater {
candidate.getApi(),
candidate.getName(),
candidate.getRestTestSuite().getFile(),
new SetupSection(candidate.getSetupSection().getSkipSection(), setup),
new SetupSection(candidate.getSetupSection().getPrerequisiteSection(), setup),
candidate.getTeardownSection(),
List.of()
);