github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-29 01:44:36 -04:00
Code Issues Projects Releases Packages Wiki Activity

Support mdx file format for docs (#107428)

Browse source 
Adding support for MDX files in our :docs project. We parse those *.mdx files
like we do for asciidoc files for code snippets and generate yaml specs from them that
we test as part of our integration tests.

By default:
When searching for doc sources in the docs folder we fail the build if we detect multiple files of
the same name but different extension. E.g. having painless-field-context.mdx
and painless-field-context.asciidoc in the same source folder will fail the build.

Migration Mode:
To allow easier migration from asciidoc to mdx the build supports a kind of migration mode.
When running the build with -Dgradle.docs.migration=true (e.g. ./gradlew buildRestTests -Dgradle.docs.migration=true)

Duplicate doc source files (asciidoc and mdx) are allowed
The Generated yaml rest specs for duplicates will have the extension *.mdx.yml or *asciidoc.yml.
The generated yaml rest specs for duplicates are compared to each other to ensure they produce the same yml output.
This commit is contained in:
Rene Groeschke 2024-04-25 09:38:46 +02:00 • committed by GitHub
parent b412ae630e
commit a0caf336e6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 2421 additions and 1907 deletions
Download patch file Download diff file Expand all files Collapse all files

2
build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPluginFuncTest.groovy
View file

@ -45,7 +45,7 @@ mapper-annotated-text.asciidoc[51:69](console)// TEST[setup:seats]
""") """)
} }
def "can console candidates"() { def "can list console candidates"() {
when: when:
def result = gradleRunner("listConsoleCandidates").build() def result = gradleRunner("listConsoleCandidates").build()
then: then:

296
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/AsciidocSnippetParser.java
View file

@ -8,296 +8,84 @@
package org.elasticsearch.gradle.internal.doc; package org.elasticsearch.gradle.internal.doc;
import org.gradle.api.InvalidUserDataException;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.function.BiConsumer;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class AsciidocSnippetParser implements SnippetParser { public class AsciidocSnippetParser extends SnippetParser {
public static final Pattern SNIPPET_PATTERN = Pattern.compile("-{4,}\\s*"); public static final Pattern SNIPPET_PATTERN = Pattern.compile("-{4,}\\s*");
public static final Pattern TEST_RESPONSE_PATTERN = Pattern.compile("\\/\\/\s*TESTRESPONSE(\\[(.+)\\])?\s*");
public static final Pattern SOURCE_PATTERN = Pattern.compile(
"\\[\"?source\"?(?:\\.[^,]+)?,\\s*\"?([-\\w]+)\"?(,((?!id=).)*(id=\"?([-\\w]+)\"?)?(.*))?].*"
);
private static final String CATCH = "catch:\\s*((?:\\/[^\\/]+\\/)|[^ \\]]+)"; public static final String CONSOLE_REGEX = "\\/\\/\s*CONSOLE\s*";
private static final String SKIP_REGEX = "skip:([^\\]]+)"; public static final String NOTCONSOLE_REGEX = "\\/\\/\s*NOTCONSOLE\s*";
private static final String SETUP = "setup:([^ \\]]+)"; public static final String TESTSETUP_REGEX = "\\/\\/\s*TESTSETUP\s*";
private static final String TEARDOWN = "teardown:([^ \\]]+)"; public static final String TEARDOWN_REGEX = "\\/\\/\s*TEARDOWN\s*";
private static final String WARNING = "warning:(.+)";
private static final String NON_JSON = "(non_json)";
private static final String SCHAR = "(?:\\\\\\/|[^\\/])";
private static final String SUBSTITUTION = "s\\/(" + SCHAR + "+)\\/(" + SCHAR + "*)\\/";
private static final String TEST_SYNTAX = "(?:"
+ CATCH
+ "|"
+ SUBSTITUTION
+ "|"
+ SKIP_REGEX
+ "|(continued)|"
+ SETUP
+ "|"
+ TEARDOWN
+ "|"
+ WARNING
+ "|(skip_shard_failures)) ?";
private final Map<String, String> defaultSubstitutions;
public AsciidocSnippetParser(Map<String, String> defaultSubstitutions) { public AsciidocSnippetParser(Map<String, String> defaultSubstitutions) {
this.defaultSubstitutions = defaultSubstitutions; super(defaultSubstitutions);
} }
@Override @Override
public List<Snippet> parseDoc(File rootDir, File docFile, List<Map.Entry<String, String>> substitutions) { protected Pattern testResponsePattern() {
String lastLanguage = null; return TEST_RESPONSE_PATTERN;
Snippet snippet = null; }
String name = null;
int lastLanguageLine = 0;
StringBuilder contents = null;
List<Snippet> snippets = new ArrayList<>();
try (Stream<String> lines = Files.lines(docFile.toPath(), StandardCharsets.UTF_8)) { protected Pattern testPattern() {
List<String> linesList = lines.collect(Collectors.toList()); return Pattern.compile("\\/\\/\s*TEST(\\[(.+)\\])?\s*");
for (int lineNumber = 0; lineNumber < linesList.size(); lineNumber++) { }
String line = linesList.get(lineNumber);
private int lastLanguageLine = 0;
private String currentName = null;
private String lastLanguage = null;
protected void parseLine(List<Snippet> snippets, int lineNumber, String line) {
if (SNIPPET_PATTERN.matcher(line).matches()) { if (SNIPPET_PATTERN.matcher(line).matches()) {
if (snippet == null) { if (snippetBuilder == null) {
Path path = rootDir.toPath().relativize(docFile.toPath()); snippetBuilder = newSnippetBuilder().withLineNumber(lineNumber + 1)
snippet = new Snippet(path, lineNumber + 1, name); .withName(currentName)
snippets.add(snippet); .withSubstitutions(defaultSubstitutions);
if (lastLanguageLine == lineNumber - 1) { if (lastLanguageLine == lineNumber - 1) {
snippet.language = lastLanguage; snippetBuilder.withLanguage(lastLanguage);
} }
name = null; currentName = null;
} else { } else {
snippet.end = lineNumber + 1; snippetBuilder.withEnd(lineNumber + 1);
} }
continue; return;
} }
Source source = matchSource(line); Source source = matchSource(line);
if (source.matches) { if (source.matches) {
lastLanguage = source.language; lastLanguage = source.language;
lastLanguageLine = lineNumber; lastLanguageLine = lineNumber;
name = source.name; currentName = source.name;
continue; return;
} }
if (consoleHandled(docFile.getName(), lineNumber, line, snippet)) { handleCommons(snippets, line);
continue;
}
if (testHandled(docFile.getName(), lineNumber, line, snippet, substitutions)) {
continue;
}
if (testResponseHandled(docFile.getName(), lineNumber, line, snippet, substitutions)) {
continue;
}
if (line.matches("\\/\\/\s*TESTSETUP\s*")) {
snippet.testSetup = true;
continue;
}
if (line.matches("\\/\\/\s*TEARDOWN\s*")) {
snippet.testTearDown = true;
continue;
}
if (snippet == null) {
// Outside
continue;
}
if (snippet.end == Snippet.NOT_FINISHED) {
// Inside
if (contents == null) {
contents = new StringBuilder();
}
// We don't need the annotations
line = line.replaceAll("<\\d+>", "");
// Nor any trailing spaces
line = line.replaceAll("\s+$", "");
contents.append(line).append("\n");
continue;
}
// Allow line continuations for console snippets within lists
if (snippet != null && line.trim().equals("+")) {
continue;
}
finalizeSnippet(snippet, contents.toString(), defaultSubstitutions, substitutions);
substitutions = new ArrayList<>();
;
snippet = null;
contents = null;
}
if (snippet != null) {
finalizeSnippet(snippet, contents.toString(), defaultSubstitutions, substitutions);
contents = null;
snippet = null;
substitutions = new ArrayList<>();
}
} catch (IOException e) {
e.printStackTrace();
}
return snippets;
} }
static Snippet finalizeSnippet( protected String getTestSetupRegex() {
final Snippet snippet, return TESTSETUP_REGEX;
String contents,
Map<String, String> defaultSubstitutions,
Collection<Map.Entry<String, String>> substitutions
) {
snippet.contents = contents.toString();
snippet.validate();
escapeSubstitutions(snippet, defaultSubstitutions, substitutions);
return snippet;
} }
private static void escapeSubstitutions( protected String getTeardownRegex() {
Snippet snippet, return TEARDOWN_REGEX;
Map<String, String> defaultSubstitutions,
Collection<Map.Entry<String, String>> substitutions
) {
BiConsumer<String, String> doSubstitution = (pattern, subst) -> {
/*
* $body is really common but it looks like a
* backreference so we just escape it here to make the
* tests cleaner.
*/
subst = subst.replace("$body", "\\$body");
subst = subst.replace("$_path", "\\$_path");
subst = subst.replace("\\n", "\n");
snippet.contents = snippet.contents.replaceAll(pattern, subst);
};
defaultSubstitutions.forEach(doSubstitution);
if (substitutions != null) {
substitutions.forEach(e -> doSubstitution.accept(e.getKey(), e.getValue()));
}
} }
private boolean testResponseHandled( protected String getNotconsoleRegex() {
String name, return NOTCONSOLE_REGEX;
int lineNumber,
String line,
Snippet snippet,
final List<Map.Entry<String, String>> substitutions
) {
Matcher matcher = Pattern.compile("\\/\\/\s*TESTRESPONSE(\\[(.+)\\])?\s*").matcher(line);
if (matcher.matches()) {
if (snippet == null) {
throw new InvalidUserDataException(name + ":" + lineNumber + ": TESTRESPONSE not paired with a snippet at ");
}
snippet.testResponse = true;
if (matcher.group(2) != null) {
String loc = name + ":" + lineNumber;
ParsingUtils.parse(
loc,
matcher.group(2),
"(?:" + SUBSTITUTION + "|" + NON_JSON + "|" + SKIP_REGEX + ") ?",
(Matcher m, Boolean last) -> {
if (m.group(1) != null) {
// TESTRESPONSE[s/adsf/jkl/]
substitutions.add(Map.entry(m.group(1), m.group(2)));
} else if (m.group(3) != null) {
// TESTRESPONSE[non_json]
substitutions.add(Map.entry("^", "/"));
substitutions.add(Map.entry("\n$", "\\\\s*/"));
substitutions.add(Map.entry("( +)", "$1\\\\s+"));
substitutions.add(Map.entry("\n", "\\\\s*\n "));
} else if (m.group(4) != null) {
// TESTRESPONSE[skip:reason]
snippet.skip = m.group(4);
}
}
);
}
return true;
}
return false;
} }
private boolean testHandled(String name, int lineNumber, String line, Snippet snippet, List<Map.Entry<String, String>> substitutions) { protected String getConsoleRegex() {
Matcher matcher = Pattern.compile("\\/\\/\s*TEST(\\[(.+)\\])?\s*").matcher(line); return CONSOLE_REGEX;
if (matcher.matches()) {
if (snippet == null) {
throw new InvalidUserDataException(name + ":" + lineNumber + ": TEST not paired with a snippet at ");
}
snippet.test = true;
if (matcher.group(2) != null) {
String loc = name + ":" + lineNumber;
ParsingUtils.parse(loc, matcher.group(2), TEST_SYNTAX, (Matcher m, Boolean last) -> {
if (m.group(1) != null) {
snippet.catchPart = m.group(1);
return;
}
if (m.group(2) != null) {
substitutions.add(Map.entry(m.group(2), m.group(3)));
return;
}
if (m.group(4) != null) {
snippet.skip = m.group(4);
return;
}
if (m.group(5) != null) {
snippet.continued = true;
return;
}
if (m.group(6) != null) {
snippet.setup = m.group(6);
return;
}
if (m.group(7) != null) {
snippet.teardown = m.group(7);
return;
}
if (m.group(8) != null) {
snippet.warnings.add(m.group(8));
return;
}
if (m.group(9) != null) {
snippet.skipShardsFailures = true;
return;
}
throw new InvalidUserDataException("Invalid test marker: " + line);
});
}
return true;
}
return false;
}
private boolean consoleHandled(String fileName, int lineNumber, String line, Snippet snippet) {
if (line.matches("\\/\\/\s*CONSOLE\s*")) {
if (snippet == null) {
throw new InvalidUserDataException(fileName + ":" + lineNumber + ": CONSOLE not paired with a snippet");
}
if (snippet.console != null) {
throw new InvalidUserDataException(fileName + ":" + lineNumber + ": Can't be both CONSOLE and NOTCONSOLE");
}
snippet.console = true;
return true;
} else if (line.matches("\\/\\/\s*NOTCONSOLE\s*")) {
if (snippet == null) {
throw new InvalidUserDataException(fileName + ":" + lineNumber + ": NOTCONSOLE not paired with a snippet");
}
if (snippet.console != null) {
throw new InvalidUserDataException(fileName + ":" + lineNumber + ": Can't be both CONSOLE and NOTCONSOLE");
}
snippet.console = false;
return true;
}
return false;
} }
static Source matchSource(String line) { static Source matchSource(String line) {
Pattern pattern = Pattern.compile("\\[\"?source\"?(?:\\.[^,]+)?,\\s*\"?([-\\w]+)\"?(,((?!id=).)*(id=\"?([-\\w]+)\"?)?(.*))?].*"); Matcher matcher = SOURCE_PATTERN.matcher(line);
Matcher matcher = pattern.matcher(line);
if (matcher.matches()) { if (matcher.matches()) {
return new Source(true, matcher.group(1), matcher.group(5)); return new Source(true, matcher.group(1), matcher.group(5));
} }

23
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocSnippetTask.java
View file

@ -8,19 +8,17 @@
package org.elasticsearch.gradle.internal.doc; package org.elasticsearch.gradle.internal.doc;
import org.apache.commons.collections.map.HashedMap;
import org.gradle.api.Action; import org.gradle.api.Action;
import org.gradle.api.DefaultTask; import org.gradle.api.DefaultTask;
import org.gradle.api.InvalidUserDataException; import org.gradle.api.InvalidUserDataException;
import org.gradle.api.file.ConfigurableFileTree; import org.gradle.api.file.ConfigurableFileTree;
import org.gradle.api.provider.MapProperty;
import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.TaskAction; import org.gradle.api.tasks.TaskAction;
import java.io.File; import java.io.File;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
public abstract class DocSnippetTask extends DefaultTask { public abstract class DocSnippetTask extends DefaultTask {
@ -36,7 +34,6 @@ public abstract class DocSnippetTask extends DefaultTask {
* directory. * directory.
*/ */
private ConfigurableFileTree docs; private ConfigurableFileTree docs;
private Map<String, String> defaultSubstitutions = new HashedMap();
@InputFiles @InputFiles
public ConfigurableFileTree getDocs() { public ConfigurableFileTree getDocs() {
@ -51,36 +48,32 @@ public abstract class DocSnippetTask extends DefaultTask {
* Substitutions done on every snippet's contents. * Substitutions done on every snippet's contents.
*/ */
@Input @Input
public Map<String, String> getDefaultSubstitutions() { abstract MapProperty<String, String> getDefaultSubstitutions();
return defaultSubstitutions;
}
@TaskAction @TaskAction
void executeTask() { void executeTask() {
for (File file : docs) { for (File file : docs) {
List<Snippet> snippets = parseDocFile(docs.getDir(), file, new ArrayList<>()); List<Snippet> snippets = parseDocFile(docs.getDir(), file);
if (perSnippet != null) { if (perSnippet != null) {
snippets.forEach(perSnippet::execute); snippets.forEach(perSnippet::execute);
} }
} }
} }
List<Snippet> parseDocFile(File rootDir, File docFile, List<Map.Entry<String, String>> substitutions) { List<Snippet> parseDocFile(File rootDir, File docFile) {
SnippetParser parser = parserForFileType(docFile); SnippetParser parser = parserForFileType(docFile);
return parser.parseDoc(rootDir, docFile, substitutions); return parser.parseDoc(rootDir, docFile);
} }
private SnippetParser parserForFileType(File docFile) { private SnippetParser parserForFileType(File docFile) {
if (docFile.getName().endsWith(".asciidoc")) { if (docFile.getName().endsWith(".asciidoc")) {
return new AsciidocSnippetParser(defaultSubstitutions); return new AsciidocSnippetParser(getDefaultSubstitutions().get());
} else if (docFile.getName().endsWith(".mdx")) {
return new MdxSnippetParser(getDefaultSubstitutions().get());
} }
throw new InvalidUserDataException("Unsupported file type: " + docFile.getName()); throw new InvalidUserDataException("Unsupported file type: " + docFile.getName());
} }
public void setDefaultSubstitutions(Map<String, String> defaultSubstitutions) {
this.defaultSubstitutions = defaultSubstitutions;
}
public void setPerSnippet(Action<Snippet> perSnippet) { public void setPerSnippet(Action<Snippet> perSnippet) {
this.perSnippet = perSnippet; this.perSnippet = perSnippet;
} }

9
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.java
View file

@ -75,14 +75,14 @@ public class DocsTestPlugin implements Plugin<Project> {
project.getTasks().register("listSnippets", DocSnippetTask.class, task -> { project.getTasks().register("listSnippets", DocSnippetTask.class, task -> {
task.setGroup("Docs"); task.setGroup("Docs");
task.setDescription("List each snippet"); task.setDescription("List each snippet");
task.setDefaultSubstitutions(commonDefaultSubstitutions); task.getDefaultSubstitutions().putAll(commonDefaultSubstitutions);
task.setPerSnippet(snippet -> System.out.println(snippet)); task.setPerSnippet(System.out::println);
}); });
project.getTasks().register("listConsoleCandidates", DocSnippetTask.class, task -> { project.getTasks().register("listConsoleCandidates", DocSnippetTask.class, task -> {
task.setGroup("Docs"); task.setGroup("Docs");
task.setDescription("List snippets that probably should be marked // CONSOLE"); task.setDescription("List snippets that probably should be marked // CONSOLE");
task.setDefaultSubstitutions(commonDefaultSubstitutions); task.getDefaultSubstitutions().putAll(commonDefaultSubstitutions);
task.setPerSnippet(snippet -> { task.setPerSnippet(snippet -> {
if (snippet.isConsoleCandidate()) { if (snippet.isConsoleCandidate()) {
System.out.println(snippet); System.out.println(snippet);
@ -93,8 +93,9 @@ public class DocsTestPlugin implements Plugin<Project> {
Provider<Directory> restRootDir = projectLayout.getBuildDirectory().dir("rest"); Provider<Directory> restRootDir = projectLayout.getBuildDirectory().dir("rest");
TaskProvider<RestTestsFromDocSnippetTask> buildRestTests = project.getTasks() TaskProvider<RestTestsFromDocSnippetTask> buildRestTests = project.getTasks()
.register("buildRestTests", RestTestsFromDocSnippetTask.class, task -> { .register("buildRestTests", RestTestsFromDocSnippetTask.class, task -> {
task.setDefaultSubstitutions(commonDefaultSubstitutions); task.getDefaultSubstitutions().putAll(commonDefaultSubstitutions);
task.getTestRoot().convention(restRootDir); task.getTestRoot().convention(restRootDir);
task.getMigrationMode().set(Boolean.getBoolean("gradle.docs.migration"));
task.doFirst(task1 -> fileOperations.delete(restRootDir.get())); task.doFirst(task1 -> fileOperations.delete(restRootDir.get()));
}); });

80
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/MdxSnippetParser.java Normal file
View file

@ -0,0 +1,80 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.gradle.internal.doc;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class MdxSnippetParser extends SnippetParser {
public static final Pattern SNIPPET_PATTERN = Pattern.compile("```(.*)");
public static final Pattern TEST_RESPONSE_PATTERN = Pattern.compile("\\{\\/\\*\s*TESTRESPONSE(\\[(.*)\\])?\s\\*\\/\\}");
public static final Pattern TEST_PATTERN = Pattern.compile("\\{\\/\\*\s*TEST(\\[(.*)\\])?\s\\*\\/\\}");
public static final String CONSOLE_REGEX = "\\{\\/\\*\s*CONSOLE\s\\*\\/\\}";
public static final String NOTCONSOLE_REGEX = "\\{\\/\\*\s*NOTCONSOLE\s\\*\\/\\}";
public static final String TESTSETUP_REGEX = "\\{\\/\\*\s*TESTSETUP\s\\*\\/\\}";
public static final String TEARDOWN_REGEX = "\\{\\/\\*\s*TEARDOWN\s\\*\\/\\}";
public MdxSnippetParser(Map<String, String> defaultSubstitutions) {
super(defaultSubstitutions);
}
@Override
protected void parseLine(List<Snippet> snippets, int lineNumber, String line) {
Matcher snippetStartMatcher = SNIPPET_PATTERN.matcher(line);
if (snippetStartMatcher.matches()) {
if (snippetBuilder == null) {
if (snippetStartMatcher.groupCount() == 1) {
String language = snippetStartMatcher.group(1);
snippetBuilder = newSnippetBuilder().withLineNumber(lineNumber + 1)
.withName(null)
.withSubstitutions(defaultSubstitutions)
.withLanguage(language);
}
} else {
snippetBuilder.withEnd(lineNumber + 1);
}
return;
}
handleCommons(snippets, line);
}
@Override
protected String getTestSetupRegex() {
return TESTSETUP_REGEX;
}
@Override
protected String getTeardownRegex() {
return TEARDOWN_REGEX;
}
@Override
protected String getNotconsoleRegex() {
return NOTCONSOLE_REGEX;
}
@Override
protected String getConsoleRegex() {
return CONSOLE_REGEX;
}
@Override
protected Pattern testResponsePattern() {
return TEST_RESPONSE_PATTERN;
}
@Override
protected Pattern testPattern() {
return TEST_PATTERN;
}
}

14
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/ParsingUtils.java
View file

@ -16,15 +16,13 @@ import java.util.regex.Pattern;
public class ParsingUtils { public class ParsingUtils {
static void extraContent(String message, String content, int offset, String location, String pattern) { static void extraContent(String message, String content, int offset, String pattern) {
StringBuilder cutOut = new StringBuilder(); StringBuilder cutOut = new StringBuilder();
cutOut.append(content.substring(offset - 6, offset)); cutOut.append(content.substring(offset - 6, offset));
cutOut.append('*'); cutOut.append('*');
cutOut.append(content.substring(offset, Math.min(offset + 5, content.length()))); cutOut.append(content.substring(offset, Math.min(offset + 5, content.length())));
String cutOutNoNl = cutOut.toString().replace("\n", "\\n"); String cutOutNoNl = cutOut.toString().replace("\n", "\\n");
throw new InvalidUserDataException( throw new InvalidUserDataException("Extra content " + message + " ('" + cutOutNoNl + "') matching [" + pattern + "]: " + content);
location + ": Extra content " + message + " ('" + cutOutNoNl + "') matching [" + pattern + "]: " + content
);
} }
/** /**
@ -33,7 +31,7 @@ public class ParsingUtils {
* match then blow up. If the closure takes two parameters then the second * match then blow up. If the closure takes two parameters then the second
* one is "is this the last match?". * one is "is this the last match?".
*/ */
static void parse(String location, String content, String pattern, BiConsumer<Matcher, Boolean> testHandler) { static void parse(String content, String pattern, BiConsumer<Matcher, Boolean> testHandler) {
if (content == null) { if (content == null) {
return; // Silly null, only real stuff gets to match! return; // Silly null, only real stuff gets to match!
} }
@ -41,16 +39,16 @@ public class ParsingUtils {
int offset = 0; int offset = 0;
while (m.find()) { while (m.find()) {
if (m.start() != offset) { if (m.start() != offset) {
extraContent("between [$offset] and [${m.start()}]", content, offset, location, pattern); extraContent("between [$offset] and [${m.start()}]", content, offset, pattern);
} }
offset = m.end(); offset = m.end();
testHandler.accept(m, offset == content.length()); testHandler.accept(m, offset == content.length());
} }
if (offset == 0) { if (offset == 0) {
throw new InvalidUserDataException(location + ": Didn't match " + pattern + ": " + content); throw new InvalidUserDataException("Didn't match " + pattern + ": " + content);
} }
if (offset != content.length()) { if (offset != content.length()) {
extraContent("after [" + offset + "]", content, offset, location, pattern); extraContent("after [" + offset + "]", content, offset, pattern);
} }
} }

231
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTask.java
View file

@ -10,10 +10,12 @@ package org.elasticsearch.gradle.internal.doc;
import groovy.transform.PackageScope; import groovy.transform.PackageScope;
import org.gradle.api.GradleException;
import org.gradle.api.InvalidUserDataException; import org.gradle.api.InvalidUserDataException;
import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.DirectoryProperty;
import org.gradle.api.internal.file.FileOperations; import org.gradle.api.provider.ListProperty;
import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.MapProperty;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Internal;
import org.gradle.api.tasks.OutputDirectory; import org.gradle.api.tasks.OutputDirectory;
@ -25,8 +27,8 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -36,35 +38,32 @@ import javax.inject.Inject;
public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask { public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
private Map<String, String> setups = new HashMap<>(); /**
* For easier migration from asciidoc to mdx we support a migration mode that
private Map<String, String> teardowns = new HashMap(); * allows generation from the same file name but different extensions. The task
* will compare the generated tests from the asciidoc and mdx files and fail if
* they are not equal (ignoring the line numbers).
* */
@Input
public abstract Property<Boolean> getMigrationMode();
/** /**
* Test setups defined in the build instead of the docs so they can be * Test setups defined in the build instead of the docs so they can be
* shared between many doc files. * shared between many doc files.
*/ */
private Map<String, String> setups = new LinkedHashMap<>();
@Input @Input
public Map<String, String> getSetups() { public Map<String, String> getSetups() {
return setups; return setups;
} }
public void setSetups(Map<String, String> setups) {
this.setups = setups;
}
/** /**
* Test teardowns defined in the build instead of the docs so they can be * Test teardowns defined in the build instead of the docs so they can be
* shared between many doc files. * shared between many doc files.
*/ */
@Input @Input
public Map<String, String> getTeardowns() { public abstract MapProperty<String, String> getTeardowns();
return teardowns;
}
public void setTeardowns(Map<String, String> teardowns) {
this.teardowns = teardowns;
}
/** /**
* A list of files that contain snippets that *probably* should be * A list of files that contain snippets that *probably* should be
@ -73,36 +72,8 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
* If there are unconverted snippets not in this list then this task will * If there are unconverted snippets not in this list then this task will
* fail. All files are paths relative to the docs dir. * fail. All files are paths relative to the docs dir.
*/ */
private List<String> expectedUnconvertedCandidates;
@Input @Input
public List<String> getExpectedUnconvertedCandidates() { public abstract ListProperty<String> getExpectedUnconvertedCandidates();
return expectedUnconvertedCandidates;
}
public void setExpectedUnconvertedCandidates(List<String> expectedUnconvertedCandidates) {
this.expectedUnconvertedCandidates = expectedUnconvertedCandidates;
}
/**
* Root directory of the tests being generated. To make rest tests happy
* we generate them in a testRoot which is contained in this directory.
*/
private DirectoryProperty testRoot;
private Set<String> names = new HashSet<>();
@Internal
public Set<String> getNames() {
return names;
}
public void setNames(Set<String> names) {
this.names = names;
}
@Inject
public abstract FileOperations getFileOperations();
/** /**
* Root directory containing all the files generated by this task. It is * Root directory containing all the files generated by this task. It is
@ -110,23 +81,27 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
*/ */
@OutputDirectory @OutputDirectory
File getOutputRoot() { File getOutputRoot() {
return new File(testRoot.get().getAsFile(), "/rest-api-spec/test"); return new File(getTestRoot().get().getAsFile(), "/rest-api-spec/test");
} }
@OutputDirectory /**
DirectoryProperty getTestRoot() { * Root directory of the tests being generated. To make rest tests happy
return testRoot; * we generate them in a testRoot which is contained in this directory.
} */
@Internal
abstract DirectoryProperty getTestRoot();
@Inject @Inject
public RestTestsFromDocSnippetTask(ObjectFactory objectFactory) { public RestTestsFromDocSnippetTask() {
testRoot = objectFactory.directoryProperty();
TestBuilder builder = new TestBuilder(); TestBuilder builder = new TestBuilder();
setPerSnippet(builder::handleSnippet);
setPerSnippet(snippet -> builder.handleSnippet(snippet)); getMigrationMode().convention(false);
doLast(task -> { doLast(task -> {
builder.finishLastTest(); builder.finishLastTest();
builder.checkUnconverted(); builder.checkUnconverted();
if (getMigrationMode().get()) {
assertEqualTestSnippetFromMigratedDocs();
}
}); });
} }
@ -223,38 +198,37 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
*/ */
public void handleSnippet(Snippet snippet) { public void handleSnippet(Snippet snippet) {
if (snippet.isConsoleCandidate()) { if (snippet.isConsoleCandidate()) {
unconvertedCandidates.add(snippet.path.toString().replace('\\', '/')); unconvertedCandidates.add(snippet.path().toString().replace('\\', '/'));
} }
if (BAD_LANGUAGES.contains(snippet.language)) { if (BAD_LANGUAGES.contains(snippet.language())) {
throw new InvalidUserDataException(snippet + ": Use `js` instead of `" + snippet.language + "`."); throw new InvalidUserDataException(snippet + ": Use `js` instead of `" + snippet.language() + "`.");
} }
if (snippet.testSetup) { if (snippet.testSetup()) {
testSetup(snippet); testSetup(snippet);
previousTest = snippet; previousTest = snippet;
return; return;
} }
if (snippet.testTearDown) { if (snippet.testTearDown()) {
testTearDown(snippet); testTearDown(snippet);
previousTest = snippet; previousTest = snippet;
return; return;
} }
if (snippet.testResponse || snippet.language.equals("console-result")) { if (snippet.testResponse() || snippet.language().equals("console-result")) {
if (previousTest == null) { if (previousTest == null) {
throw new InvalidUserDataException(snippet + ": No paired previous test"); throw new InvalidUserDataException(snippet + ": No paired previous test");
} }
if (previousTest.path.equals(snippet.path) == false) { if (previousTest.path().equals(snippet.path()) == false) {
throw new InvalidUserDataException(snippet + ": Result can't be first in file"); throw new InvalidUserDataException(snippet + ": Result can't be first in file");
} }
response(snippet); response(snippet);
return; return;
} }
if (("js".equals(snippet.language)) && snippet.console != null && snippet.console) { if (("js".equals(snippet.language())) && snippet.console() != null && snippet.console()) {
throw new InvalidUserDataException(snippet + ": Use `[source,console]` instead of `// CONSOLE`."); throw new InvalidUserDataException(snippet + ": Use `[source,console]` instead of `// CONSOLE`.");
} }
if (snippet.test || snippet.language.equals("console")) { if (snippet.test() || snippet.language().equals("console")) {
test(snippet); test(snippet);
previousTest = snippet; previousTest = snippet;
return;
} }
// Must be an unmarked snippet.... // Must be an unmarked snippet....
} }
@ -262,27 +236,27 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
private void test(Snippet test) { private void test(Snippet test) {
setupCurrent(test); setupCurrent(test);
if (test.continued) { if (test.continued()) {
/* Catch some difficult to debug errors with // TEST[continued] /* Catch some difficult to debug errors with // TEST[continued]
* and throw a helpful error message. */ * and throw a helpful error message. */
if (previousTest == null || previousTest.path.equals(test.path) == false) { if (previousTest == null || previousTest.path().equals(test.path()) == false) {
throw new InvalidUserDataException("// TEST[continued] " + "cannot be on first snippet in a file: " + test); throw new InvalidUserDataException("// TEST[continued] " + "cannot be on first snippet in a file: " + test);
} }
if (previousTest != null && previousTest.testSetup) { if (previousTest != null && previousTest.testSetup()) {
throw new InvalidUserDataException("// TEST[continued] " + "cannot immediately follow // TESTSETUP: " + test); throw new InvalidUserDataException("// TEST[continued] " + "cannot immediately follow // TESTSETUP: " + test);
} }
if (previousTest != null && previousTest.testTearDown) { if (previousTest != null && previousTest.testSetup()) {
throw new InvalidUserDataException("// TEST[continued] " + "cannot immediately follow // TEARDOWN: " + test); throw new InvalidUserDataException("// TEST[continued] " + "cannot immediately follow // TEARDOWN: " + test);
} }
} else { } else {
current.println("---"); current.println("---");
if (test.name != null && test.name.isBlank() == false) { if (test.name() != null && test.name().isBlank() == false) {
if (names.add(test.name) == false) { if (names.add(test.name()) == false) {
throw new InvalidUserDataException("Duplicated snippet name '" + test.name + "': " + test); throw new InvalidUserDataException("Duplicated snippet name '" + test.name() + "': " + test);
} }
current.println("\"" + test.name + "\":"); current.println("\"" + test.name() + "\":");
} else { } else {
current.println("\"line_" + test.start + "\":"); current.println("\"line_" + test.start() + "\":");
} }
/* The Elasticsearch test runner doesn't support quite a few /* The Elasticsearch test runner doesn't support quite a few
* constructs unless we output this skip. We don't know if * constructs unless we output this skip. We don't know if
@ -296,36 +270,36 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
current.println(" - stash_path_replace"); current.println(" - stash_path_replace");
current.println(" - warnings"); current.println(" - warnings");
} }
if (test.skip != null) { if (test.skip() != null) {
if (test.continued) { if (test.continued()) {
throw new InvalidUserDataException("Continued snippets " + "can't be skipped"); throw new InvalidUserDataException("Continued snippets " + "can't be skipped");
} }
current.println(" - always_skip"); current.println(" - always_skip");
current.println(" reason: " + test.skip); current.println(" reason: " + test.skip());
} }
if (test.setup != null) { if (test.setup() != null) {
setup(test); setup(test);
} }
body(test, false); body(test, false);
if (test.teardown != null) { if (test.teardown() != null) {
teardown(test); teardown(test);
} }
} }
private void response(Snippet response) { private void response(Snippet response) {
if (null == response.skip) { if (null == response.skip()) {
current.println(" - match:"); current.println(" - match:");
current.println(" $body:"); current.println(" $body:");
replaceBlockQuote(response.contents).lines().forEach(line -> current.println(" " + line)); replaceBlockQuote(response.contents()).lines().forEach(line -> current.println(" " + line));
} }
} }
private void teardown(final Snippet snippet) { private void teardown(final Snippet snippet) {
// insert a teardown defined outside of the docs // insert a teardown defined outside of the docs
for (final String name : snippet.teardown.split(",")) { for (final String name : snippet.teardown().split(",")) {
final String teardown = teardowns.get(name); final String teardown = getTeardowns().get().get(name);
if (teardown == null) { if (teardown == null) {
throw new InvalidUserDataException("Couldn't find named teardown $name for " + snippet); throw new InvalidUserDataException("Couldn't find named teardown $name for " + snippet);
} }
@ -335,7 +309,7 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
} }
private void testTearDown(Snippet snippet) { private void testTearDown(Snippet snippet) {
if (previousTest != null && previousTest.testSetup == false && lastDocsPath == snippet.path) { if (previousTest != null && previousTest.testSetup() == false && lastDocsPath.equals(snippet.path())) {
throw new InvalidUserDataException(snippet + " must follow test setup or be first"); throw new InvalidUserDataException(snippet + " must follow test setup or be first");
} }
setupCurrent(snippet); setupCurrent(snippet);
@ -411,7 +385,7 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
} }
private void body(Snippet snippet, boolean inSetup) { private void body(Snippet snippet, boolean inSetup) {
ParsingUtils.parse(snippet.getLocation(), snippet.contents, SYNTAX, (matcher, last) -> { ParsingUtils.parse(snippet.contents(), SYNTAX, (matcher, last) -> {
if (matcher.group("comment") != null) { if (matcher.group("comment") != null) {
// Comment // Comment
return; return;
@ -424,30 +398,43 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
String method = matcher.group("method"); String method = matcher.group("method");
String pathAndQuery = matcher.group("pathAndQuery"); String pathAndQuery = matcher.group("pathAndQuery");
String body = matcher.group("body"); String body = matcher.group("body");
String catchPart = last ? snippet.catchPart : null; String catchPart = last ? snippet.catchPart() : null;
if (pathAndQuery.startsWith("/")) { if (pathAndQuery.startsWith("/")) {
// Leading '/'s break the generated paths // Leading '/'s break the generated paths
pathAndQuery = pathAndQuery.substring(1); pathAndQuery = pathAndQuery.substring(1);
} }
emitDo(method, pathAndQuery, body, catchPart, snippet.warnings, inSetup, snippet.skipShardsFailures); emitDo(method, pathAndQuery, body, catchPart, snippet.warnings(), inSetup, snippet.skipShardsFailures());
}); });
} }
private PrintWriter setupCurrent(Snippet test) { private PrintWriter setupCurrent(Snippet test) {
if (test.path.equals(lastDocsPath)) { if (test.path().equals(lastDocsPath)) {
return current; return current;
} }
names.clear(); names.clear();
finishLastTest(); finishLastTest();
lastDocsPath = test.path; lastDocsPath = test.path();
// Make the destination file: // Make the destination file:
// Shift the path into the destination directory tree // Shift the path into the destination directory tree
Path dest = getOutputRoot().toPath().resolve(test.path); Path dest = getOutputRoot().toPath().resolve(test.path());
// Replace the extension // Replace the extension
String fileName = dest.getName(dest.getNameCount() - 1).toString(); String fileName = dest.getName(dest.getNameCount() - 1).toString();
dest = dest.getParent().resolve(fileName.replace(".asciidoc", ".yml")); if (hasMultipleDocImplementations(test.path())) {
String fileNameWithoutExt = dest.getName(dest.getNameCount() - 1).toString().replace(".asciidoc", "").replace(".mdx", "");
if (getMigrationMode().get() == false) {
throw new InvalidUserDataException(
"Found multiple files with the same name '" + fileNameWithoutExt + "' but different extensions: [asciidoc, mdx]"
);
}
getLogger().warn("Found multiple doc file types for " + test.path() + ". Generating tests for all of them.");
dest = dest.getParent().resolve(fileName + ".yml");
} else {
dest = dest.getParent().resolve(fileName.replace(".asciidoc", ".yml").replace(".mdx", ".yml"));
}
// Now setup the writer // Now setup the writer
try { try {
@ -460,7 +447,7 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
} }
private void testSetup(Snippet snippet) { private void testSetup(Snippet snippet) {
if (lastDocsPath == snippet.path) { if (lastDocsPath == snippet.path()) {
throw new InvalidUserDataException( throw new InvalidUserDataException(
snippet + ": wasn't first. TESTSETUP can only be used in the first snippet of a document." snippet + ": wasn't first. TESTSETUP can only be used in the first snippet of a document."
); );
@ -468,7 +455,7 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
setupCurrent(snippet); setupCurrent(snippet);
current.println("---"); current.println("---");
current.println("setup:"); current.println("setup:");
if (snippet.setup != null) { if (snippet.setup() != null) {
setup(snippet); setup(snippet);
} }
body(snippet, true); body(snippet, true);
@ -476,8 +463,8 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
private void setup(final Snippet snippet) { private void setup(final Snippet snippet) {
// insert a setup defined outside of the docs // insert a setup defined outside of the docs
for (final String name : snippet.setup.split(",")) { for (final String name : snippet.setup().split(",")) {
final String setup = setups.get(name); final String setup = getSetups().get(name);
if (setup == null) { if (setup == null) {
throw new InvalidUserDataException("Couldn't find named setup " + name + " for " + snippet); throw new InvalidUserDataException("Couldn't find named setup " + name + " for " + snippet);
} }
@ -488,7 +475,7 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
public void checkUnconverted() { public void checkUnconverted() {
List<String> listedButNotFound = new ArrayList<>(); List<String> listedButNotFound = new ArrayList<>();
for (String listed : expectedUnconvertedCandidates) { for (String listed : getExpectedUnconvertedCandidates().get()) {
if (false == unconvertedCandidates.remove(listed)) { if (false == unconvertedCandidates.remove(listed)) {
listedButNotFound.add(listed); listedButNotFound.add(listed);
} }
@ -523,4 +510,54 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
} }
} }
private void assertEqualTestSnippetFromMigratedDocs() {
getTestRoot().getAsFileTree().matching(patternSet -> { patternSet.include("**/*asciidoc.yml"); }).forEach(asciidocFile -> {
File mdxFile = new File(asciidocFile.getAbsolutePath().replace(".asciidoc.yml", ".mdx.yml"));
if (mdxFile.exists() == false) {
throw new InvalidUserDataException("Couldn't find the corresponding mdx file for " + asciidocFile.getAbsolutePath());
}
try {
List<String> asciidocLines = Files.readAllLines(asciidocFile.toPath());
List<String> mdxLines = Files.readAllLines(mdxFile.toPath());
if (asciidocLines.size() != mdxLines.size()) {
throw new GradleException(
"Yaml rest specs ("
+ asciidocFile.toPath()
+ " and "
+ mdxFile.getAbsolutePath()
+ ") are not equal, different line count"
);
}
for (int i = 0; i < asciidocLines.size(); i++) {
if (asciidocLines.get(i)
.replaceAll("line_\\d+", "line_0")
.equals(mdxLines.get(i).replaceAll("line_\\d+", "line_0")) == false) {
throw new GradleException(
"Yaml rest specs ("
+ asciidocFile.toPath()
+ " and "
+ mdxFile.getAbsolutePath()
+ ") are not equal, difference on line: "
+ (i + 1)
);
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
private boolean hasMultipleDocImplementations(Path path) {
File dir = getDocs().getDir();
String fileName = path.getName(path.getNameCount() - 1).toString();
if (fileName.endsWith("asciidoc")) {
return new File(dir, path.toString().replace(".asciidoc", ".mdx")).exists();
} else if (fileName.endsWith("mdx")) {
return new File(dir, path.toString().replace(".mdx", ".asciidoc")).exists();
}
return false;
}
} }

125
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Snippet.java
View file

@ -8,113 +8,30 @@
package org.elasticsearch.gradle.internal.doc; package org.elasticsearch.gradle.internal.doc;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import org.gradle.api.InvalidUserDataException;
import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List; import java.util.List;
public class Snippet { public record Snippet(
static final int NOT_FINISHED = -1; Path path,
int start,
/** int end,
* Path to the file containing this snippet. Relative to docs.dir of the String contents,
* SnippetsTask that created it. Boolean console,
*/ boolean test,
Path path; boolean testResponse,
int start; boolean testSetup,
int end = NOT_FINISHED; boolean testTearDown,
public String contents; String skip,
boolean continued,
Boolean console = null; String language,
boolean test = false; String catchPart,
boolean testResponse = false; String setup,
boolean testSetup = false; String teardown,
boolean testTearDown = false; boolean curl,
String skip = null; List<String> warnings,
boolean continued = false; boolean skipShardsFailures,
String language = null; String name
String catchPart = null; ) {
String setup = null;
String teardown = null;
boolean curl;
List<String> warnings = new ArrayList();
boolean skipShardsFailures = false;
String name;
public Snippet(Path path, int start, String name) {
this.path = path;
this.start = start;
this.name = name;
}
public void validate() {
if (language == null) {
throw new InvalidUserDataException(
name
+ ": "
+ "Snippet missing a language. This is required by "
+ "Elasticsearch's doc testing infrastructure so we "
+ "be sure we don't accidentally forget to test a "
+ "snippet."
);
}
assertValidCurlInput();
assertValidJsonInput();
}
String getLocation() {
return path + "[" + start + ":" + end + "]";
}
private void assertValidCurlInput() {
// Try to detect snippets that contain `curl`
if ("sh".equals(language) || "shell".equals(language)) {
curl = contents.contains("curl");
if (console == Boolean.FALSE && curl == false) {
throw new InvalidUserDataException(name + ": " + "No need for NOTCONSOLE if snippet doesn't " + "contain `curl`.");
}
}
}
private void assertValidJsonInput() {
if (testResponse && ("js" == language || "console-result" == language) && null == skip) {
String quoted = contents
// quote values starting with $
.replaceAll("([:,])\\s*(\\$[^ ,\\n}]+)", "$1 \"$2\"")
// quote fields starting with $
.replaceAll("(\\$[^ ,\\n}]+)\\s*:", "\"$1\":");
JsonFactory jf = new JsonFactory();
jf.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true);
JsonParser jsonParser;
try {
jsonParser = jf.createParser(quoted);
while (jsonParser.isClosed() == false) {
jsonParser.nextToken();
}
} catch (JsonParseException e) {
throw new InvalidUserDataException(
"Invalid json in "
+ name
+ ". The error is:\n"
+ e.getMessage()
+ ".\n"
+ "After substitutions and munging, the json looks like:\n"
+ quoted,
e
);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
@Override @Override
public String toString() { public String toString() {

273
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetBuilder.java Normal file
View file

@ -0,0 +1,273 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.gradle.internal.doc;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import org.apache.commons.collections.map.MultiValueMap;
import org.gradle.api.InvalidUserDataException;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
class SnippetBuilder {
static final int NOT_FINISHED = -1;
private Path path;
private int lineNumber;
private String name;
private String language;
private int end = NOT_FINISHED;
private boolean testSetup;
private boolean testTeardown;
// some tests rely on ugly regex substitutions using the same key multiple times
private MultiValueMap substitutions = MultiValueMap.decorate(new LinkedHashMap<String, String>());
private String catchPart;
private boolean test;
private String skip;
private boolean continued;
private String setup;
private String teardown;
private List<String> warnings = new ArrayList<>();
private boolean skipShardsFailures;
private boolean testResponse;
private boolean curl;
private StringBuilder contentBuilder = new StringBuilder();
private Boolean console = null;
public SnippetBuilder withPath(Path path) {
this.path = path;
return this;
}
public SnippetBuilder withLineNumber(int lineNumber) {
this.lineNumber = lineNumber;
return this;
}
public SnippetBuilder withName(String currentName) {
this.name = currentName;
return this;
}
public SnippetBuilder withLanguage(String language) {
this.language = language;
return this;
}
public SnippetBuilder withEnd(int end) {
this.end = end;
return this;
}
public SnippetBuilder withTestSetup(boolean testSetup) {
this.testSetup = testSetup;
return this;
}
public SnippetBuilder withTestTearDown(boolean testTeardown) {
this.testTeardown = testTeardown;
return this;
}
public boolean notFinished() {
return end == NOT_FINISHED;
}
public SnippetBuilder withSubstitutions(Map<String, String> substitutions) {
this.substitutions.putAll(substitutions);
return this;
}
public SnippetBuilder withSubstitution(String key, String value) {
this.substitutions.put(key, value);
return this;
}
public SnippetBuilder withTest(boolean test) {
this.test = test;
return this;
}
public SnippetBuilder withCatchPart(String catchPart) {
this.catchPart = catchPart;
return this;
}
public SnippetBuilder withSkip(String skip) {
this.skip = skip;
return this;
}
public SnippetBuilder withContinued(boolean continued) {
this.continued = continued;
return this;
}
public SnippetBuilder withSetup(String setup) {
this.setup = setup;
return this;
}
public SnippetBuilder withTeardown(String teardown) {
this.teardown = teardown;
return this;
}
public SnippetBuilder withWarning(String warning) {
this.warnings.add(warning);
return this;
}
public SnippetBuilder withSkipShardsFailures(boolean skipShardsFailures) {
this.skipShardsFailures = skipShardsFailures;
return this;
}
public SnippetBuilder withTestResponse(boolean testResponse) {
this.testResponse = testResponse;
return this;
}
public SnippetBuilder withContent(String content) {
return withContent(content, false);
}
public SnippetBuilder withContent(String content, boolean newLine) {
contentBuilder.append(content);
if (newLine) {
contentBuilder.append("\n");
}
return this;
}
private String escapeSubstitutions(String contents) {
Set<Map.Entry<String, List<String>>> set = substitutions.entrySet();
for (Map.Entry<String, List<String>> substitution : set) {
String pattern = substitution.getKey();
for (String subst : substitution.getValue()) {
/*
* $body is really common, but it looks like a
* backreference, so we just escape it here to make the
* tests cleaner.
*/
subst = subst.replace("$body", "\\$body");
subst = subst.replace("$_path", "\\$_path");
subst = subst.replace("\\n", "\n");
contents = contents.replaceAll(pattern, subst);
}
}
return contents;
}
public Snippet build() {
String content = contentBuilder.toString();
validate(content);
String finalContent = escapeSubstitutions(content);
return new Snippet(
path,
lineNumber,
end,
finalContent,
console,
test,
testResponse,
testSetup,
testTeardown,
skip,
continued,
language,
catchPart,
setup,
teardown,
curl,
warnings,
skipShardsFailures,
name
);
}
public void validate(String content) {
if (language == null) {
throw new InvalidUserDataException(
name
+ ": "
+ "Snippet missing a language. This is required by "
+ "Elasticsearch's doc testing infrastructure so we "
+ "be sure we don't accidentally forget to test a "
+ "snippet."
);
}
assertValidCurlInput(content);
assertValidJsonInput(content);
}
private void assertValidCurlInput(String content) {
// Try to detect snippets that contain `curl`
if ("sh".equals(language) || "shell".equals(language)) {
curl = content.contains("curl");
if (console == Boolean.FALSE && curl == false) {
throw new InvalidUserDataException(name + ": " + "No need for NOTCONSOLE if snippet doesn't " + "contain `curl`.");
}
}
}
private void assertValidJsonInput(String content) {
if (testResponse && ("js" == language || "console-result" == language) && null == skip) {
String quoted = content
// quote values starting with $
.replaceAll("([:,])\\s*(\\$[^ ,\\n}]+)", "$1 \"$2\"")
// quote fields starting with $
.replaceAll("(\\$[^ ,\\n}]+)\\s*:", "\"$1\":");
JsonFactory jf = new JsonFactory();
jf.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true);
JsonParser jsonParser;
try {
jsonParser = jf.createParser(quoted);
while (jsonParser.isClosed() == false) {
jsonParser.nextToken();
}
} catch (JsonParseException e) {
throw new InvalidUserDataException(
"Invalid json in "
+ name
+ ". The error is:\n"
+ e.getMessage()
+ ".\n"
+ "After substitutions and munging, the json looks like:\n"
+ quoted,
e
);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
public SnippetBuilder withConsole(Boolean console) {
this.console = console;
return this;
}
public boolean consoleDefined() {
return console != null;
}
}

245
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParser.java
View file

@ -8,10 +8,251 @@
package org.elasticsearch.gradle.internal.doc; package org.elasticsearch.gradle.internal.doc;
import org.gradle.api.InvalidUserDataException;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
abstract class SnippetParser {
protected static final String SCHAR = "(?:\\\\\\/|[^\\/])";
protected static final String NON_JSON = "(non_json)";
protected static final String SKIP_REGEX = "skip:([^\\]]+)";
protected static final String SUBSTITUTION = "s\\/(" + SCHAR + "+)\\/(" + SCHAR + "*)\\/";
private static final String CATCH = "catch:\\s*((?:\\/[^\\/]+\\/)|[^ \\]]+)";
private static final String SETUP = "setup:([^ \\]]+)";
private static final String TEARDOWN = "teardown:([^ \\]]+)";
private static final String WARNING = "warning:(.+)";
private static final String TEST_SYNTAX = "(?:"
+ CATCH
+ "|"
+ SUBSTITUTION
+ "|"
+ SKIP_REGEX
+ "|(continued)|"
+ SETUP
+ "|"
+ TEARDOWN
+ "|"
+ WARNING
+ "|(skip_shard_failures)) ?";
protected final Map<String, String> defaultSubstitutions;
protected SnippetBuilder snippetBuilder = null;
private Path currentPath;
SnippetParser(Map<String, String> defaultSubstitutions) {
this.defaultSubstitutions = defaultSubstitutions;
}
public List<Snippet> parseDoc(File rootDir, File docFile) {
List<Snippet> snippets = new ArrayList<>();
this.currentPath = rootDir.toPath().relativize(docFile.toPath());
try (Stream<String> lines = Files.lines(docFile.toPath(), StandardCharsets.UTF_8)) {
List<String> linesList = lines.toList();
parseLines(docFile, linesList, snippets);
} catch (IOException e) {
throw new SnippetParserException("Failed to parse file " + docFile, e);
} finally {
this.currentPath = null;
this.snippetBuilder = null;
}
return snippets;
}
void parseLines(File file, List<String> linesList, List<Snippet> snippets) {
for (int lineNumber = 0; lineNumber < linesList.size(); lineNumber++) {
String line = linesList.get(lineNumber);
try {
parseLine(snippets, lineNumber, line);
} catch (InvalidUserDataException e) {
throw new SnippetParserException(file, lineNumber, e);
}
}
fileParsingFinished(snippets);
}
protected void handleCommons(List<Snippet> snippets, String line) {
if (consoleHandled(line, snippetBuilder)) {
return;
}
if (testHandled(line, snippetBuilder)) {
return;
}
if (testResponseHandled(line, snippetBuilder)) {
return;
}
if (line.matches(getTestSetupRegex())) {
snippetBuilder.withTestSetup(true);
return;
}
if (line.matches(getTeardownRegex())) {
snippetBuilder.withTestTearDown(true);
return;
}
if (snippetBuilder == null) {
// Outside
return;
}
if (snippetBuilder.notFinished()) {
// Inside
// We don't need the annotations
line = line.replaceAll("<\\d+>", "");
// nor bookmarks
line = line.replaceAll("\\[\\^\\d+\\]", "");
// Nor any trailing spaces
line = line.replaceAll("\s+$", "");
snippetBuilder.withContent(line, true);
return;
}
// Allow line continuations for console snippets within lists
if (snippetBuilder != null && line.trim().equals("+")) {
return;
}
snippets.add(snippetBuilder.build());
snippetBuilder = null;
}
protected SnippetBuilder newSnippetBuilder() {
snippetBuilder = new SnippetBuilder().withPath(currentPath);
return snippetBuilder;
}
void fileParsingFinished(List<Snippet> snippets) {
if (snippetBuilder != null) {
snippets.add(snippetBuilder.build());
snippetBuilder = null;
}
}
protected abstract void parseLine(List<Snippet> snippets, int lineNumber, String line);
boolean testResponseHandled(String line, SnippetBuilder snippetBuilder) {
Matcher matcher = testResponsePattern().matcher(line);
if (matcher.matches()) {
if (snippetBuilder == null) {
throw new InvalidUserDataException("TESTRESPONSE not paired with a snippet at ");
}
snippetBuilder.withTestResponse(true);
if (matcher.group(2) != null) {
ParsingUtils.parse(
matcher.group(2),
"(?:" + SUBSTITUTION + "|" + NON_JSON + "|" + SKIP_REGEX + ") ?",
(Matcher m, Boolean last) -> {
if (m.group(1) != null) {
// TESTRESPONSE[s/adsf/jkl/]
snippetBuilder.withSubstitution(m.group(1), m.group(2));
} else if (m.group(3) != null) {
// TESTRESPONSE[non_json]
snippetBuilder.withSubstitution("^", "/");
snippetBuilder.withSubstitution("\n$", "\\\\s*/");
snippetBuilder.withSubstitution("( +)", "$1\\\\s+");
snippetBuilder.withSubstitution("\n", "\\\\s*\n ");
} else if (m.group(4) != null) {
// TESTRESPONSE[skip:reason]
snippetBuilder.withSkip(m.group(4));
}
}
);
}
return true;
}
return false;
}
protected boolean testHandled(String line, SnippetBuilder snippetBuilder) {
Matcher matcher = testPattern().matcher(line);
if (matcher.matches()) {
if (snippetBuilder == null) {
throw new InvalidUserDataException("TEST not paired with a snippet at ");
}
snippetBuilder.withTest(true);
if (matcher.group(2) != null) {
ParsingUtils.parse(matcher.group(2), TEST_SYNTAX, (Matcher m, Boolean last) -> {
if (m.group(1) != null) {
snippetBuilder.withCatchPart(m.group(1));
return;
}
if (m.group(2) != null) {
snippetBuilder.withSubstitution(m.group(2), m.group(3));
return;
}
if (m.group(4) != null) {
snippetBuilder.withSkip(m.group(4));
return;
}
if (m.group(5) != null) {
snippetBuilder.withContinued(true);
return;
}
if (m.group(6) != null) {
snippetBuilder.withSetup(m.group(6));
return;
}
if (m.group(7) != null) {
snippetBuilder.withTeardown(m.group(7));
return;
}
if (m.group(8) != null) {
snippetBuilder.withWarning(m.group(8));
return;
}
if (m.group(9) != null) {
snippetBuilder.withSkipShardsFailures(true);
return;
}
throw new InvalidUserDataException("Invalid test marker: " + line);
});
}
return true;
}
return false;
}
protected boolean consoleHandled(String line, SnippetBuilder snippet) {
if (line.matches(getConsoleRegex())) {
if (snippetBuilder == null) {
throw new InvalidUserDataException("CONSOLE not paired with a snippet");
}
if (snippetBuilder.consoleDefined()) {
throw new InvalidUserDataException("Can't be both CONSOLE and NOTCONSOLE");
}
snippetBuilder.withConsole(Boolean.TRUE);
return true;
} else if (line.matches(getNotconsoleRegex())) {
if (snippet == null) {
throw new InvalidUserDataException("NOTCONSOLE not paired with a snippet");
}
if (snippetBuilder.consoleDefined()) {
throw new InvalidUserDataException("Can't be both CONSOLE and NOTCONSOLE");
}
snippet.withConsole(Boolean.FALSE);
return true;
}
return false;
}
protected abstract String getTestSetupRegex();
protected abstract String getTeardownRegex();
protected abstract String getConsoleRegex();
protected abstract String getNotconsoleRegex();
protected abstract Pattern testPattern();
protected abstract Pattern testResponsePattern();
public interface SnippetParser {
List<Snippet> parseDoc(File rootDir, File docFile, List<Map.Entry<String, String>> substitutions);
} }

38
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParserException.java Normal file
View file

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.gradle.internal.doc;
import org.gradle.api.InvalidUserDataException;
import java.io.File;
public class SnippetParserException extends RuntimeException {
private final File file;
private final int lineNumber;
public SnippetParserException(String message, Throwable cause) {
super(message, cause);
this.file = null;
this.lineNumber = -1;
}
public SnippetParserException(File file, int lineNumber, InvalidUserDataException e) {
super("Error parsing snippet in " + file.getName() + " at line " + lineNumber, e);
this.file = file;
this.lineNumber = lineNumber;
}
public File getFile() {
return file;
}
public int getLineNumber() {
return lineNumber;
}
}

191
build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AbstractSnippetParserSpec.groovy Normal file
View file

@ -0,0 +1,191 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.gradle.internal.doc
import spock.lang.Specification
import org.gradle.api.InvalidUserDataException
import java.nio.file.Path
abstract class AbstractSnippetParserSpec extends Specification {
abstract SnippetParser parser()
abstract String docSnippetWithTestResponses()
abstract String docSnippetWithTest()
abstract String docSnippetWithRepetitiveSubstiutions()
abstract String docSnippetWithConsole()
abstract String docSnippetWithNotConsole()
abstract String docSnippetWithMixedConsoleNotConsole()
def "can parse snippet with console"() {
when:
def snippets = parse(docSnippetWithConsole())
then:
snippets*.console() == [true]
}
def "can parse snippet with notconsole"() {
when:
def snippets = parse(docSnippetWithNotConsole())
then:
snippets*.console() == [false]
}
def "fails on mixing console and notconsole"() {
when:
def snippets = parse(docSnippetWithMixedConsoleNotConsole())
then:
def e = thrown(SnippetParserException)
e.message.matches("Error parsing snippet in acme.xyz at line \\d")
e.file.name == "acme.xyz"
e.lineNumber > 0
}
def "can parse snippet with test"() {
when:
def snippets = parse(docSnippetWithTest())
then:
snippets*.test() == [true]
snippets*.testResponse() == [false]
snippets*.language() == ["console"]
snippets*.catchPart() == ["/painless_explain_error/"]
snippets*.teardown() == ["some_teardown"]
snippets*.setup() == ["seats"]
snippets*.warnings() == [["some_warning"]]
snippets*.contents() == ["""PUT /hockey/_doc/1?refresh
{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
POST /hockey/_explain/1?error_trace=false
{
"query": {
"script": {
"script": "Debug.explain(doc.goals)"
}
}
}
"""]
}
def "can parse snippet with test responses"() {
when:
def snippets = parse(docSnippetWithTestResponses())
then:
snippets*.testResponse() == [true]
snippets*.test() == [false]
snippets*.language() == ["console-result"]
snippets*.skip() == ["some_skip_message"]
snippets*.contents() == ["""{
"docs" : [
{
"processor_results" : [
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field2" : "_value2",
"foo" : "bar"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : \$body.docs.0.processor_results.0.doc._ingest.timestamp
}
}
},
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field3" : "_value3",
"field2" : "_value2",
"foo" : "bar"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : \$body.docs.0.processor_results.0.doc._ingest.timestamp
}
}
}
]
},
{
"processor_results" : [
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field2" : "_value2",
"foo" : "rab"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : \$body.docs.1.processor_results.0.doc._ingest.timestamp
}
}
},
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field3" : "_value3",
"field2" : "_value2",
"foo" : "rab"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : \$body.docs.1.processor_results.0.doc._ingest.timestamp
}
}
}
]
}
]
}
"""]
}
def "can parse snippet with repetitive regex substitutions"() {
when:
def snippets = parse(docSnippetWithRepetitiveSubstiutions())
then:
snippets*.test() == [true]
snippets*.testResponse() == [false]
snippets*.language() == ["console"]
snippets*.contents() == ["""PUT /_snapshot/repo1
{"type": "fs", "settings": {"location": "repo/1"}}
PUT /_snapshot/repo1/snap2?wait_for_completion=true
PUT /_snapshot/repo1/snap1?wait_for_completion=true
GET /_cat/snapshots/repo1?v=true&s=id
"""]
}
List<Snippet> parse(String docSnippet) {
List<Snippet> snippets = new ArrayList<>()
def lines = docSnippet.lines().toList()
parser().parseLines(new File("acme.xyz"), lines, snippets)
return snippets
}
}

277
build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AsciidocParserSpec.groovy
View file

@ -6,17 +6,11 @@
* Side Public License, v 1. * Side Public License, v 1.
*/ */
package org.elasticsearch.gradle.internal.doc; package org.elasticsearch.gradle.internal.doc
import spock.lang.Specification import static org.elasticsearch.gradle.internal.doc.AsciidocSnippetParser.matchSource
import spock.lang.Unroll
import org.gradle.api.InvalidUserDataException class AsciidocParserSpec extends AbstractSnippetParserSpec {
import static org.elasticsearch.gradle.internal.doc.AsciidocSnippetParser.finalizeSnippet;
import static org.elasticsearch.gradle.internal.doc.AsciidocSnippetParser.matchSource;
class AsciidocParserSpec extends Specification {
def testMatchSource() { def testMatchSource() {
expect: expect:
@ -78,107 +72,174 @@ class AsciidocParserSpec extends Specification {
} }
} }
@Unroll @Override
def "checks for valid json for #languageParam"() { SnippetParser parser() {
given: return new AsciidocSnippetParser([:]);
def snippet = snippet() {
language = languageParam
testResponse = true
} }
def json = """{
"name": "John Doe", @Override
"age": 30, String docSnippetWithTest() {
"isMarried": true, return """[source,console]
"address": { ---------------------------------------------------------
"street": "123 Main Street", PUT /hockey/_doc/1?refresh
"city": "Springfield", {"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
"state": "IL",
"zip": "62701" POST /hockey/_explain/1
{
"query": {
"script": {
"script": "Debug.explain(doc.goals)"
}
}
}
---------------------------------------------------------
// TEST[s/_explain\\/1/_explain\\/1?error_trace=false/ catch:/painless_explain_error/]
// TEST[teardown:some_teardown]
// TEST[setup:seats]
// TEST[warning:some_warning]
// TEST[skip_shard_failures]
"""
}
@Override
String docSnippetWithRepetitiveSubstiutions() {
return """
[source,console]
--------------------------------------------------
GET /_cat/snapshots/repo1?v=true&s=id
--------------------------------------------------
// TEST[s/^/PUT \\/_snapshot\\/repo1\\/snap1?wait_for_completion=true\\n/]
// TEST[s/^/PUT \\/_snapshot\\/repo1\\/snap2?wait_for_completion=true\\n/]
// TEST[s/^/PUT \\/_snapshot\\/repo1\\n{"type": "fs", "settings": {"location": "repo\\/1"}}\\n/]
"""
}
@Override
String docSnippetWithConsole() {
return """
[source,console]
----
// CONSOLE
----
"""
}
@Override
String docSnippetWithNotConsole() {
return """
[source,console]
----
// NOTCONSOLE
----
"""
}
@Override
String docSnippetWithMixedConsoleNotConsole() {
return """
[source,console]
----
// NOTCONSOLE
// CONSOLE
----
"""
}
@Override
String docSnippetWithTestResponses() {
return """
[source,console-result]
----
{
"docs" : [
{
"processor_results" : [
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field2" : "_value2",
"foo" : "bar"
}, },
"hobbies": ["Reading", "Cooking", "Traveling"] "_ingest" : {
}""" "pipeline" : "_simulate_pipeline",
when: "timestamp" : "2020-07-30T01:21:24.251836Z"
def result = finalizeSnippet(snippet, json, [:], [:].entrySet()) }
then: }
result != null },
{
when: "processor_type" : "set",
finalizeSnippet(snippet, "some no valid json", [:], [:].entrySet()) "status" : "success",
then: "doc" : {
def e = thrown(InvalidUserDataException) "_index" : "index",
e.message.contains("Invalid json in") "_id" : "id",
"_version": "-3",
when: "_source" : {
snippet.skip = "true" "field3" : "_value3",
result = finalizeSnippet(snippet, "some no valid json", [:], [:].entrySet()) "field2" : "_value2",
then: "foo" : "bar"
result != null },
"_ingest" : {
where: "pipeline" : "_simulate_pipeline",
languageParam << ["js", "console-result"] "timestamp" : "2020-07-30T01:21:24.251836Z"
}
}
}
]
},
{
"processor_results" : [
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field2" : "_value2",
"foo" : "rab"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : "2020-07-30T01:21:24.251863Z"
}
}
},
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field3" : "_value3",
"field2" : "_value2",
"foo" : "rab"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : "2020-07-30T01:21:24.251863Z"
}
}
}
]
}
]
}
----
// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/\$body.docs.0.processor_results.0.doc._ingest.timestamp/]
// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/\$body.docs.0.processor_results.1.doc._ingest.timestamp/]
// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/\$body.docs.1.processor_results.0.doc._ingest.timestamp/]
// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/\$body.docs.1.processor_results.1.doc._ingest.timestamp/]
// TESTRESPONSE[skip:some_skip_message]
"""
} }
def "test finalized snippet handles substitutions"() {
given:
def snippet = snippet() {
language = "console"
}
when:
finalizeSnippet(snippet, "snippet-content substDefault subst", [substDefault: "\$body"], [subst: 'substValue'].entrySet())
then:
snippet.contents == "snippet-content \$body substValue"
}
def snippetMustHaveLanguage() {
given:
def snippet = snippet()
when:
finalizeSnippet(snippet, "snippet-content", [:], [])
then:
def e = thrown(InvalidUserDataException)
e.message.contains("Snippet missing a language.")
}
def testEmit() {
given:
def snippet = snippet() {
language = "console"
}
when:
finalizeSnippet(snippet, "snippet-content", [:], [])
then:
snippet.contents == "snippet-content"
}
def testSnippetsWithCurl() {
given:
def snippet = snippet() {
language = "sh"
name = "snippet-name-1"
}
when:
finalizeSnippet(snippet, "curl substDefault subst", [:], [:].entrySet())
then:
snippet.curl == true
}
def "test snippets with no curl no console"() {
given:
def snippet = snippet() {
console = false
language = "shell"
}
when:
finalizeSnippet(snippet, "hello substDefault subst", [:], [:].entrySet())
then:
def e = thrown(InvalidUserDataException)
e.message.contains("No need for NOTCONSOLE if snippet doesn't contain `curl`")
}
Snippet snippet(Closure<DocSnippetTask> configClosure = {}) {
def snippet = new Snippet(new File("SomePath").toPath(), 0, "snippet-name-1")
configClosure.delegate = snippet
configClosure()
return snippet
}
} }

571
build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocSnippetTaskSpec.groovy
View file

@ -10,8 +10,8 @@ package org.elasticsearch.gradle.internal.doc
import spock.lang.Specification import spock.lang.Specification
import spock.lang.TempDir import spock.lang.TempDir
import spock.lang.Unroll
import org.gradle.api.InvalidUserDataException
import org.gradle.testfixtures.ProjectBuilder import org.gradle.testfixtures.ProjectBuilder
import static org.elasticsearch.gradle.internal.test.TestUtils.normalizeString import static org.elasticsearch.gradle.internal.test.TestUtils.normalizeString
@ -21,559 +21,60 @@ class DocSnippetTaskSpec extends Specification {
@TempDir @TempDir
File tempDir File tempDir
def "handling test parsing multiple snippets per file"() { @Unroll
given: def "handling test parsing multiple snippets per #fileType file"() {
def project = ProjectBuilder.builder().build()
def task = project.tasks.register("docSnippetTask", DocSnippetTask).get()
when: when:
def substitutions = [] def snippets = parseFile("example-1.$fileType")
def snippets = task.parseDocFile(
tempDir, docFile(
"""
[[mapper-annotated-text]]
=== Mapper annotated text plugin
experimental[]
The mapper-annotated-text plugin provides the ability to index text that is a
combination of free-text and special markup that is typically used to identify
items of interest such as people or organisations (see NER or Named Entity Recognition
tools).
The elasticsearch markup allows one or more additional tokens to be injected, unchanged, into the token
stream at the same position as the underlying text it annotates.
:plugin_name: mapper-annotated-text
include::install_remove.asciidoc[]
[[mapper-annotated-text-usage]]
==== Using the `annotated-text` field
The `annotated-text` tokenizes text content as per the more common {ref}/text.html[`text`] field (see
"limitations" below) but also injects any marked-up annotation tokens directly into
the search index:
[source,console]
--------------------------
PUT my-index-000001
{
"mappings": {
"properties": {
"my_field": {
"type": "annotated_text"
}
}
}
}
--------------------------
Such a mapping would allow marked-up text eg wikipedia articles to be indexed as both text
and structured tokens. The annotations use a markdown-like syntax using URL encoding of
one or more values separated by the `&` symbol.
We can use the "_analyze" api to test how an example annotation would be stored as tokens
in the search index:
[source,js]
--------------------------
GET my-index-000001/_analyze
{
"field": "my_field",
"text":"Investors in [Apple](Apple+Inc.) rejoiced."
}
--------------------------
// NOTCONSOLE
Response:
[source,js]
--------------------------------------------------
{
"tokens": [
{
"token": "investors",
"start_offset": 0,
"end_offset": 9,
"type": "<ALPHANUM>",
"position": 0
},
{
"token": "in",
"start_offset": 10,
"end_offset": 12,
"type": "<ALPHANUM>",
"position": 1
},
{
"token": "Apple Inc.", <1>
"start_offset": 13,
"end_offset": 18,
"type": "annotation",
"position": 2
},
{
"token": "apple",
"start_offset": 13,
"end_offset": 18,
"type": "<ALPHANUM>",
"position": 2
},
{
"token": "rejoiced",
"start_offset": 19,
"end_offset": 27,
"type": "<ALPHANUM>",
"position": 3
}
]
}
--------------------------------------------------
// NOTCONSOLE
<1> Note the whole annotation token `Apple Inc.` is placed, unchanged as a single token in
the token stream and at the same position (position 2) as the text token (`apple`) it annotates.
We can now perform searches for annotations using regular `term` queries that don't tokenize
the provided search values. Annotations are a more precise way of matching as can be seen
in this example where a search for `Beck` will not match `Jeff Beck` :
[source,console]
--------------------------
# Example documents
PUT my-index-000001/_doc/1
{
"my_field": "[Beck](Beck) announced a new tour"<1>
}
PUT my-index-000001/_doc/2
{
"my_field": "[Jeff Beck](Jeff+Beck&Guitarist) plays a strat"<2>
}
# Example search
GET my-index-000001/_search
{
"query": {
"term": {
"my_field": "Beck" <3>
}
}
}
--------------------------
<1> As well as tokenising the plain text into single words e.g. `beck`, here we
inject the single token value `Beck` at the same position as `beck` in the token stream.
<2> Note annotations can inject multiple tokens at the same position - here we inject both
the very specific value `Jeff Beck` and the broader term `Guitarist`. This enables
broader positional queries e.g. finding mentions of a `Guitarist` near to `strat`.
<3> A benefit of searching with these carefully defined annotation tokens is that a query for
`Beck` will not match document 2 that contains the tokens `jeff`, `beck` and `Jeff Beck`
WARNING: Any use of `=` signs in annotation values eg `[Prince](person=Prince)` will
cause the document to be rejected with a parse failure. In future we hope to have a use for
the equals signs so wil actively reject documents that contain this today.
[[mapper-annotated-text-tips]]
==== Data modelling tips
===== Use structured and unstructured fields
Annotations are normally a way of weaving structured information into unstructured text for
higher-precision search.
`Entity resolution` is a form of document enrichment undertaken by specialist software or people
where references to entities in a document are disambiguated by attaching a canonical ID.
The ID is used to resolve any number of aliases or distinguish between people with the
same name. The hyperlinks connecting Wikipedia's articles are a good example of resolved
entity IDs woven into text.
These IDs can be embedded as annotations in an annotated_text field but it often makes
sense to include them in dedicated structured fields to support discovery via aggregations:
[source,console]
--------------------------
PUT my-index-000001
{
"mappings": {
"properties": {
"my_unstructured_text_field": {
"type": "annotated_text"
},
"my_structured_people_field": {
"type": "text",
"fields": {
"keyword" : {
"type": "keyword"
}
}
}
}
}
}
--------------------------
Applications would then typically provide content and discover it as follows:
[source,console]
--------------------------
# Example documents
PUT my-index-000001/_doc/1
{
"my_unstructured_text_field": "[Shay](%40kimchy) created elasticsearch",
"my_twitter_handles": ["@kimchy"] <1>
}
GET my-index-000001/_search
{
"query": {
"query_string": {
"query": "elasticsearch OR logstash OR kibana",<2>
"default_field": "my_unstructured_text_field"
}
},
"aggregations": {
\t"top_people" :{
\t "significant_terms" : { <3>
\t "field" : "my_twitter_handles.keyword"
\t }
\t}
}
}
--------------------------
<1> Note the `my_twitter_handles` contains a list of the annotation values
also used in the unstructured text. (Note the annotated_text syntax requires escaping).
By repeating the annotation values in a structured field this application has ensured that
the tokens discovered in the structured field can be used for search and highlighting
in the unstructured field.
<2> In this example we search for documents that talk about components of the elastic stack
<3> We use the `my_twitter_handles` field here to discover people who are significantly
associated with the elastic stack.
===== Avoiding over-matching annotations
By design, the regular text tokens and the annotation tokens co-exist in the same indexed
field but in rare cases this can lead to some over-matching.
The value of an annotation often denotes a _named entity_ (a person, place or company).
The tokens for these named entities are inserted untokenized, and differ from typical text
tokens because they are normally:
* Mixed case e.g. `Madonna`
* Multiple words e.g. `Jeff Beck`
* Can have punctuation or numbers e.g. `Apple Inc.` or `@kimchy`
This means, for the most part, a search for a named entity in the annotated text field will
not have any false positives e.g. when selecting `Apple Inc.` from an aggregation result
you can drill down to highlight uses in the text without "over matching" on any text tokens
like the word `apple` in this context:
the apple was very juicy
However, a problem arises if your named entity happens to be a single term and lower-case e.g. the
company `elastic`. In this case, a search on the annotated text field for the token `elastic`
may match a text document such as this:
they fired an elastic band
To avoid such false matches users should consider prefixing annotation values to ensure
they don't name clash with text tokens e.g.
[elastic](Company_elastic) released version 7.0 of the elastic stack today
[[mapper-annotated-text-highlighter]]
==== Using the `annotated` highlighter
The `annotated-text` plugin includes a custom highlighter designed to mark up search hits
in a way which is respectful of the original markup:
[source,console]
--------------------------
# Example documents
PUT my-index-000001/_doc/1
{
"my_field": "The cat sat on the [mat](sku3578)"
}
GET my-index-000001/_search
{
"query": {
"query_string": {
"query": "cats"
}
},
"highlight": {
"fields": {
"my_field": {
"type": "annotated", <1>
"require_field_match": false
}
}
}
}
--------------------------
<1> The `annotated` highlighter type is designed for use with annotated_text fields
The annotated highlighter is based on the `unified` highlighter and supports the same
settings but does not use the `pre_tags` or `post_tags` parameters. Rather than using
html-like markup such as `<em>cat</em>` the annotated highlighter uses the same
markdown-like syntax used for annotations and injects a key=value annotation where `_hit_term`
is the key and the matched search term is the value e.g.
The [cat](_hit_term=cat) sat on the [mat](sku3578)
The annotated highlighter tries to be respectful of any existing markup in the original
text:
* If the search term matches exactly the location of an existing annotation then the
`_hit_term` key is merged into the url-like syntax used in the `(...)` part of the
existing annotation.
* However, if the search term overlaps the span of an existing annotation it would break
the markup formatting so the original annotation is removed in favour of a new annotation
with just the search hit information in the results.
* Any non-overlapping annotations in the original text are preserved in highlighter
selections
[[mapper-annotated-text-limitations]]
==== Limitations
The annotated_text field type supports the same mapping settings as the `text` field type
but with the following exceptions:
* No support for `fielddata` or `fielddata_frequency_filter`
* No support for `index_prefixes` or `index_phrases` indexing
"""
), substitutions
)
then: then:
snippets*.test == [false, false, false, false, false, false, false] snippets*.test == [false, false, false, false, false, false, false]
snippets*.catchPart == [null, null, null, null, null, null, null] snippets*.catchPart == [null, null, null, null, null, null, null]
} snippets*.setup == [null, null, null, null, null, null, null]
snippets*.teardown == [null, null, null, null, null, null, null]
snippets*.testResponse == [false, false, false, false, false, false, false]
snippets*.skip == [null, null, null, null, null, null, null]
snippets*.continued == [false, false, false, false, false, false, false]
snippets*.language == ["console", "js", "js", "console", "console", "console", "console"]
snippets*.contents*.empty == [false, false, false, false, false, false, false]
snippets*.start == expectedSnippetStarts
snippets*.end == expectedSnippetEnds
def "handling test parsing"() { // test two snippet explicitly for content.
when: // More coverage on actual parsing is done in unit tests
def substitutions = [] normalizeString(snippets[0].contents) == """PUT my-index-000001
def snippets = task().parseDocFile(
tempDir, docFile(
"""
[source,console]
----
POST logs-my_app-default/_rollover/
----
// TEST[s/_explain\\/1/_explain\\/1?error_trace=false/ catch:/painless_explain_error/]
"""
), substitutions
)
then:
snippets*.test == [true]
snippets*.catchPart == ["/painless_explain_error/"]
substitutions.size() == 1
substitutions[0].key == "_explain\\/1"
substitutions[0].value == "_explain\\/1?error_trace=false"
when:
substitutions = []
snippets = task().parseDocFile(
tempDir, docFile(
"""
[source,console]
----
PUT _snapshot/my_hdfs_repository
{ {
"type": "hdfs", "mappings": {
"settings": { "properties": {
"uri": "hdfs://namenode:8020/", "my_field": {
"path": "elasticsearch/repositories/my_hdfs_repository", "type": "annotated_text"
"conf.dfs.client.read.shortcircuit": "true"
} }
} }
----
// TEST[skip:we don't have hdfs set up while testing this]
"""
), substitutions
)
then:
snippets*.test == [true]
snippets*.skip == ["we don't have hdfs set up while testing this"]
} }
}"""
def "handling testresponse parsing"() { normalizeString(snippets[1].contents) == """GET my-index-000001/_analyze
when: {
def substitutions = [] "field": "my_field",
def snippets = task().parseDocFile( "text":"Investors in [Apple](Apple+Inc.) rejoiced."
tempDir, docFile( }"""
"""
[source,console]
----
POST logs-my_app-default/_rollover/
----
// TESTRESPONSE[s/\\.\\.\\./"script_stack": \$body.error.caused_by.script_stack, "script": \$body.error.caused_by.script, "lang": \$body.error.caused_by.lang, "position": \$body.error.caused_by.position, "caused_by": \$body.error.caused_by.caused_by, "reason": \$body.error.caused_by.reason/]
"""
), substitutions
)
then:
snippets*.test == [false]
snippets*.testResponse == [true]
substitutions.size() == 1
substitutions[0].key == "\\.\\.\\."
substitutions[0].value ==
"\"script_stack\": \$body.error.caused_by.script_stack, \"script\": \$body.error.caused_by.script, \"lang\": \$body.error.caused_by.lang, \"position\": \$body.error.caused_by.position, \"caused_by\": \$body.error.caused_by.caused_by, \"reason\": \$body.error.caused_by.reason"
when:
snippets = task().parseDocFile(
tempDir, docFile(
"""
[source,console]
----
POST logs-my_app-default/_rollover/
----
// TESTRESPONSE[skip:no setup made for this example yet]
"""
), []
)
then:
snippets*.test == [false]
snippets*.testResponse == [true]
snippets*.skip == ["no setup made for this example yet"]
when:
substitutions = []
snippets = task().parseDocFile(
tempDir, docFile(
"""
[source,txt]
---------------------------------------------------------------------------
my-index-000001 0 p RELOCATING 3014 31.1mb 192.168.56.10 H5dfFeA -> -> 192.168.56.30 bGG90GE
---------------------------------------------------------------------------
// TESTRESPONSE[non_json]
"""
), substitutions
)
then:
snippets*.test == [false]
snippets*.testResponse == [true]
substitutions.size() == 4
}
def "handling console parsing"() {
when:
def snippets = task().parseDocFile(
tempDir, docFile(
"""
[source,console]
----
// $firstToken
----
"""
), []
)
then:
snippets*.console == [firstToken.equals("CONSOLE")]
when:
task().parseDocFile(
tempDir, docFile(
"""
[source,console]
----
// $firstToken
// $secondToken
----
"""
), []
)
then:
def e = thrown(InvalidUserDataException)
e.message == "mapping-charfilter.asciidoc:4: Can't be both CONSOLE and NOTCONSOLE"
when:
task().parseDocFile(
tempDir, docFile(
"""
// $firstToken
// $secondToken
"""
), []
)
then:
e = thrown(InvalidUserDataException)
e.message == "mapping-charfilter.asciidoc:1: $firstToken not paired with a snippet"
where: where:
firstToken << ["CONSOLE", "NOTCONSOLE"] fileType << ["asciidoc", "mdx"]
secondToken << ["NOTCONSOLE", "CONSOLE"] expectedSnippetStarts << [[10, 24, 36, 59, 86, 108, 135], [9, 22, 33, 55, 80, 101, 127]]
expectedSnippetEnds << [[21, 30, 55, 75, 105, 132, 158], [20, 28, 52, 71, 99, 125, 150]]
} }
def "test parsing snippet from doc"() { List<Snippet> parseFile(String fileName) {
def doc = docFile( def task = ProjectBuilder.builder().build().tasks.register("docSnippetTask", DocSnippetTask).get()
""" def docFileToParse = docFile(fileName, DocTestUtils.SAMPLE_TEST_DOCS[fileName])
[source,console] return task.parseDocFile(
---- tempDir, docFileToParse
GET /_analyze
{
"tokenizer": "keyword",
"char_filter": [
{
"type": "mapping",
"mappings": [
"e => 0",
"m => 1",
"p => 2",
"t => 3",
"y => 4"
]
}
],
"text": "My license plate is empty"
}
----
"""
) )
def snippets = task().parseDocFile(tempDir, doc, [])
expect:
snippets[0].start == 3
snippets[0].language == "console"
normalizeString(snippets[0].contents, tempDir) == """GET /_analyze
{
"tokenizer": "keyword",
"char_filter": [
{
"type": "mapping",
"mappings": [
"e => 0",
"m => 1",
"p => 2",
"t => 3",
"y => 4"
]
}
],
"text": "My license plate is empty"
}"""
} }
File docFile(String docContent) { File docFile(String filename, String docContent) {
def file = tempDir.toPath().resolve("mapping-charfilter.asciidoc").toFile() def file = tempDir.toPath().resolve(filename).toFile()
file.text = docContent file.text = docContent
return file return file
} }
private DocSnippetTask task() {
ProjectBuilder.builder().build().tasks.register("docSnippetTask", DocSnippetTask).get()
}
} }

745
build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocTestUtils.groovy Normal file
View file

@ -0,0 +1,745 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.gradle.internal.doc;
class DocTestUtils {
public static Map<String,String> SAMPLE_TEST_DOCS = Map.of(
"example-1.mdx", """
# mapper-annotated-text
### Mapper annotated text plugin
experimental[]
some text
```console
PUT my-index-000001
{
"mappings": {
"properties": {
"my_field": {
"type": "annotated_text"
}
}
}
}
```
```js
GET my-index-000001/_analyze
{
"field": "my_field",
"text":"Investors in [Apple](Apple+Inc.) rejoiced."
}
```
{/* NOTCONSOLE */}
Response:
```js
{
"tokens": [
{
"token": "investors",
"start_offset": 0,
"end_offset": 9,
"type": "<ALPHANUM>",
"position": 0
},
{
"token": "in",
"start_offset": 10,
"end_offset": 12,
"type": "<ALPHANUM>",
"position": 1
}
]
}
```
{/* NOTCONSOLE */}
```console
# Example documents
PUT my-index-000001/_doc/1
{
"my_field": "[Jeff Beck](Jeff+Beck&Guitarist) plays a strat"<2>
}
# Example search
GET my-index-000001/_search
{
"query": {
"term": {
"my_field": "Beck" <3>
}
}
}
```
<1> More text
<2> Even More
<3> More
### Headline
#### a smaller headline
```console
PUT my-index-000001
{
"mappings": {
"properties": {
"my_unstructured_text_field": {
"type": "annotated_text"
},
"my_structured_people_field": {
"type": "text",
"fields": {
"keyword" : {
"type": "keyword"
}
}
}
}
}
}
```
```console
# Example documents
PUT my-index-000001/_doc/1
{
"my_unstructured_text_field": "[Shay](%40kimchy) created elasticsearch",
"my_twitter_handles": ["@kimchy"] <1>
}
GET my-index-000001/_search
{
"query": {
"query_string": {
"query": "elasticsearch OR logstash OR kibana",<2>
"default_field": "my_unstructured_text_field"
}
},
"aggregations": {
\t"top_people" :{
\t "significant_terms" : { <3>
\t "field" : "my_twitter_handles.keyword"
\t }
\t}
}
}
```
```console
# Example documents
PUT my-index-000001/_doc/1
{
"my_field": "The cat sat on the [mat](sku3578)"
}
GET my-index-000001/_search
{
"query": {
"query_string": {
"query": "cats"
}
},
"highlight": {
"fields": {
"my_field": {
"type": "annotated", <1>
"require_field_match": false
}
}
}
}
```
* No support for `fielddata` or `fielddata_frequency_filter`
* No support for `index_prefixes` or `index_phrases` indexing
""",
"example-1.asciidoc", """
[[mapper-annotated-text]]
=== Mapper annotated text plugin
experimental[]
some text
[source,console]
--------------------------
PUT my-index-000001
{
"mappings": {
"properties": {
"my_field": {
"type": "annotated_text"
}
}
}
}
--------------------------
[source,js]
--------------------------
GET my-index-000001/_analyze
{
"field": "my_field",
"text":"Investors in [Apple](Apple+Inc.) rejoiced."
}
--------------------------
// NOTCONSOLE
Response:
[source,js]
--------------------------------------------------
{
"tokens": [
{
"token": "investors",
"start_offset": 0,
"end_offset": 9,
"type": "<ALPHANUM>",
"position": 0
},
{
"token": "in",
"start_offset": 10,
"end_offset": 12,
"type": "<ALPHANUM>",
"position": 1
}
]
}
--------------------------------------------------
// NOTCONSOLE
[source,console]
--------------------------
# Example documents
PUT my-index-000001/_doc/1
{
"my_field": "[Jeff Beck](Jeff+Beck&Guitarist) plays a strat"<2>
}
# Example search
GET my-index-000001/_search
{
"query": {
"term": {
"my_field": "Beck" <3>
}
}
}
--------------------------
<1> More text
<2> Even More
<3> More
[[mapper-annotated-text-tips]]
==== Headline
===== a smaller headline
[source,console]
--------------------------
PUT my-index-000001
{
"mappings": {
"properties": {
"my_unstructured_text_field": {
"type": "annotated_text"
},
"my_structured_people_field": {
"type": "text",
"fields": {
"keyword" : {
"type": "keyword"
}
}
}
}
}
}
--------------------------
[source,console]
--------------------------
# Example documents
PUT my-index-000001/_doc/1
{
"my_unstructured_text_field": "[Shay](%40kimchy) created elasticsearch",
"my_twitter_handles": ["@kimchy"] <1>
}
GET my-index-000001/_search
{
"query": {
"query_string": {
"query": "elasticsearch OR logstash OR kibana",<2>
"default_field": "my_unstructured_text_field"
}
},
"aggregations": {
\t"top_people" :{
\t "significant_terms" : { <3>
\t "field" : "my_twitter_handles.keyword"
\t }
\t}
}
}
--------------------------
[source,console]
--------------------------
# Example documents
PUT my-index-000001/_doc/1
{
"my_field": "The cat sat on the [mat](sku3578)"
}
GET my-index-000001/_search
{
"query": {
"query_string": {
"query": "cats"
}
},
"highlight": {
"fields": {
"my_field": {
"type": "annotated", <1>
"require_field_match": false
}
}
}
}
--------------------------
* No support for `fielddata` or `fielddata_frequency_filter`
* No support for `index_prefixes` or `index_phrases` indexing
""",
"example-2.asciidoc", """
[[example-2]]
=== Field context
Use a Painless script to create a
{ref}/search-fields.html#script-fields[script field] to return
a customized value for each document in the results of a query.
*Variables*
`params` (`Map`, read-only)::
User-defined parameters passed in as part of the query.
`doc` (`Map`, read-only)::
Contains the fields of the specified document where each field is a
`List` of values.
{ref}/mapping-source-field.html[`params['_source']`] (`Map`, read-only)::
Contains extracted JSON in a `Map` and `List` structure for the fields
existing in a stored document.
*Return*
`Object`::
The customized value for each document.
*API*
Both the standard <<painless-api-reference-shared, Painless API>> and
<<painless-api-reference-field, Specialized Field API>> are available.
*Example*
To run this example, first follow the steps in
<<painless-context-examples, context examples>>.
You can then use these two example scripts to compute custom information
for each search hit and output it to two new fields.
The first script gets the doc value for the `datetime` field and calls
the `getDayOfWeekEnum` function to determine the corresponding day of the week.
[source,Painless]
----
doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)
----
The second script calculates the number of actors. Actors' names are stored
as a keyword array in the `actors` field.
[source,Painless]
----
doc['actors'].size() <1>
----
<1> By default, doc values are not available for `text` fields. If `actors` was
a `text` field, you could still calculate the number of actors by extracting
values from `_source` with `params['_source']['actors'].size()`.
The following request returns the calculated day of week and the number of
actors that appear in each play:
[source,console]
----
GET seats/_search
{
"size": 2,
"query": {
"match_all": {}
},
"script_fields": {
"day-of-week": {
"script": {
"source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)"
}
},
"number-of-actors": {
"script": {
"source": "doc['actors'].size()"
}
}
}
}
----
// TEST[setup:seats]
[source,console-result]
----
{
"took" : 68,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 11,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "seats",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"day-of-week" : [
"Thursday"
],
"number-of-actors" : [
4
]
}
},
{
"_index" : "seats",
"_id" : "2",
"_score" : 1.0,
"fields" : {
"day-of-week" : [
"Thursday"
],
"number-of-actors" : [
1
]
}
}
]
}
}
----
// TESTRESPONSE[s/"took" : 68/"took" : "\$body.took"/]
""",
"example-2.mdx", """---
id: enElasticsearchPainlessPainlessFieldContext
slug: /en/elasticsearch/painless/example-2
title: Field context
description: Description to be written
tags: []
---
<div id="example-2"></div>
Use a Painless script to create a
[script field](((ref))/search-fields.html#script-fields) to return
a customized value for each document in the results of a query.
**Variables**
`params` (`Map`, read-only)
: User-defined parameters passed in as part of the query.
`doc` (`Map`, read-only)
: Contains the fields of the specified document where each field is a
`List` of values.
[`params['_source']`](((ref))/mapping-source-field.html) (`Map`, read-only)
: Contains extracted JSON in a `Map` and `List` structure for the fields
existing in a stored document.
**Return**
`Object`
: The customized value for each document.
**API**
Both the standard <DocLink id="enElasticsearchPainlessPainlessApiReferenceShared">Painless API</DocLink> and
<DocLink id="enElasticsearchPainlessPainlessApiReferenceField">Specialized Field API</DocLink> are available.
**Example**
To run this example, first follow the steps in
<DocLink id="enElasticsearchPainlessPainlessContextExamples">context examples</DocLink>.
You can then use these two example scripts to compute custom information
for each search hit and output it to two new fields.
The first script gets the doc value for the `datetime` field and calls
the `getDayOfWeekEnum` function to determine the corresponding day of the week.
```Painless
doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)
```
The second script calculates the number of actors. Actors' names are stored
as a keyword array in the `actors` field.
```Painless
doc['actors'].size() [^1]
```
[^1]: By default, doc values are not available for `text` fields. If `actors` was
a `text` field, you could still calculate the number of actors by extracting
values from `_source` with `params['_source']['actors'].size()`.
The following request returns the calculated day of week and the number of
actors that appear in each play:
```console
GET seats/_search
{
"size": 2,
"query": {
"match_all": {}
},
"script_fields": {
"day-of-week": {
"script": {
"source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)"
}
},
"number-of-actors": {
"script": {
"source": "doc['actors'].size()"
}
}
}
}
```
{/* TEST[setup:seats] */}
```console-result
{
"took" : 68,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 11,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "seats",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"day-of-week" : [
"Thursday"
],
"number-of-actors" : [
4
]
}
},
{
"_index" : "seats",
"_id" : "2",
"_score" : 1.0,
"fields" : {
"day-of-week" : [
"Thursday"
],
"number-of-actors" : [
1
]
}
}
]
}
}
```
{/* TESTRESPONSE[s/"took" : 68/"took" : "\$body.took"/] */}
""",
"example-2-different.mdx", """---
id: enElasticsearchPainlessPainlessFieldContext
slug: /en/elasticsearch/painless/example-2
title: Field context
description: Description to be written
tags: []
---
<div id="example-2"></div>
Use a Painless script to create a
[script field](((ref))/search-fields.html#script-fields) to return
a customized value for each document in the results of a query.
**Variables**
`params` (`Map`, read-only)
: User-defined parameters passed in as part of the query.
`doc` (`Map`, read-only)
: Contains the fields of the specified document where each field is a
`List` of values.
[`params['_source']`](((ref))/mapping-source-field.html) (`Map`, read-only)
: Contains extracted JSON in a `Map` and `List` structure for the fields
existing in a stored document.
**Return**
`Object`
: The customized value for each document.
**API**
Both the standard <DocLink id="enElasticsearchPainlessPainlessApiReferenceShared">Painless API</DocLink> and
<DocLink id="enElasticsearchPainlessPainlessApiReferenceField">Specialized Field API</DocLink> are available.
**Example**
To run this example, first follow the steps in
<DocLink id="enElasticsearchPainlessPainlessContextExamples">context examples</DocLink>.
You can then use these two example scripts to compute custom information
for each search hit and output it to two new fields.
The first script gets the doc value for the `datetime` field and calls
the `getDayOfWeekEnum` function to determine the corresponding day of the week.
```Painless
doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)
```
The second script calculates the number of actors. Actors' names are stored
as a keyword array in the `actors` field.
```Painless
doc['actresses'].size() [^1]
```
[^1]: By default, doc values are not available for `text` fields. If `actors` was
a `text` field, you could still calculate the number of actors by extracting
values from `_source` with `params['_source']['actors'].size()`.
The following request returns the calculated day of week and the number of
actors that appear in each play:
```console
GET seats/_search
{
"size": 2,
"query": {
"match_all": {}
},
"script_fields": {
"day-of-week": {
"script": {
"source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)"
}
},
"number-of-actors": {
"script": {
"source": "doc['actors'].size()"
}
}
}
}
```
{/* TEST[setup:seats] */}
```console-result
{
"took" : 68,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 11,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "seats",
"_id" : "1",
"_score" : 1.0,
"fields" : {
"day-of-week" : [
"Thursday"
],
"number-of-actors" : [
4
]
}
},
{
"_index" : "seats",
"_id" : "2",
"_score" : 1.0,
"fields" : {
"day-of-week" : [
"Thursday"
],
"number-of-actors" : [
1
]
}
}
]
}
}
```
{/* TESTRESPONSE[s/"took" : 68/"took" : "\$body.took"/] */}
""")
}

173
build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/MdxSnippetParserSpec.groovy Normal file
View file

@ -0,0 +1,173 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.gradle.internal.doc
class MdxSnippetParserSpec extends AbstractSnippetParserSpec {
@Override
SnippetParser parser(Map<String,String> defaultSubstitutions = [:]) {
return new MdxSnippetParser(defaultSubstitutions)
}
@Override
String docSnippetWithTest() {
return """```console
PUT /hockey/_doc/1?refresh
{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
POST /hockey/_explain/1
{
"query": {
"script": {
"script": "Debug.explain(doc.goals)"
}
}
}
```
{/* TEST[s/_explain\\/1/_explain\\/1?error_trace=false/ catch:/painless_explain_error/] */}
{/* TEST[teardown:some_teardown] */}
{/* TEST[setup:seats] */}
{/* TEST[warning:some_warning] */}
{/* TEST[skip_shard_failures] */}
"""
}
@Override
String docSnippetWithRepetitiveSubstiutions() {
return """```console
GET /_cat/snapshots/repo1?v=true&s=id
```
{/* TEST[s/^/PUT \\/_snapshot\\/repo1\\/snap1?wait_for_completion=true\\n/] */}
{/* TEST[s/^/PUT \\/_snapshot\\/repo1\\/snap2?wait_for_completion=true\\n/] */}
{/* TEST[s/^/PUT \\/_snapshot\\/repo1\\n{"type": "fs", "settings": {"location": "repo\\/1"}}\\n/] */}
"""
}
@Override
String docSnippetWithConsole() {
return """
```console
{/* CONSOLE */}
```
"""
}
@Override
String docSnippetWithNotConsole() {
return """
```console
{/* NOTCONSOLE */}
```
""" }
@Override
String docSnippetWithMixedConsoleNotConsole() {
return """
```console
{/* CONSOLE */}
{/* NOTCONSOLE */}
```
""" }
@Override
String docSnippetWithTestResponses() {
return """```console-result
{
"docs" : [
{
"processor_results" : [
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field2" : "_value2",
"foo" : "bar"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : "2020-07-30T01:21:24.251836Z"
}
}
},
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field3" : "_value3",
"field2" : "_value2",
"foo" : "bar"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : "2020-07-30T01:21:24.251836Z"
}
}
}
]
},
{
"processor_results" : [
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field2" : "_value2",
"foo" : "rab"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : "2020-07-30T01:21:24.251863Z"
}
}
},
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_id" : "id",
"_version": "-3",
"_source" : {
"field3" : "_value3",
"field2" : "_value2",
"foo" : "rab"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : "2020-07-30T01:21:24.251863Z"
}
}
}
]
}
]
}
```
{/* TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/\$body.docs.0.processor_results.0.doc._ingest.timestamp/] */}
{/* TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/\$body.docs.0.processor_results.1.doc._ingest.timestamp/] */}
{/* TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/\$body.docs.1.processor_results.0.doc._ingest.timestamp/] */}
{/* TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/\$body.docs.1.processor_results.1.doc._ingest.timestamp/] */}
{/* TESTRESPONSE[skip:some_skip_message] */}
"""
}
}

879
build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy
View file

@ -11,9 +11,11 @@ package org.elasticsearch.gradle.internal.doc
import spock.lang.Specification import spock.lang.Specification
import spock.lang.TempDir import spock.lang.TempDir
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException import org.gradle.api.InvalidUserDataException
import org.gradle.testfixtures.ProjectBuilder import org.gradle.testfixtures.ProjectBuilder
import static org.elasticsearch.gradle.internal.doc.DocTestUtils.SAMPLE_TEST_DOCS
import static org.elasticsearch.gradle.internal.doc.RestTestsFromDocSnippetTask.replaceBlockQuote import static org.elasticsearch.gradle.internal.doc.RestTestsFromDocSnippetTask.replaceBlockQuote
import static org.elasticsearch.gradle.internal.doc.RestTestsFromDocSnippetTask.shouldAddShardFailureCheck import static org.elasticsearch.gradle.internal.doc.RestTestsFromDocSnippetTask.shouldAddShardFailureCheck
import static org.elasticsearch.gradle.internal.test.TestUtils.normalizeString import static org.elasticsearch.gradle.internal.test.TestUtils.normalizeString
@ -56,302 +58,138 @@ class RestTestsFromDocSnippetTaskSpec extends Specification {
shouldAddShardFailureCheck("_ml/datafeeds/datafeed-id/_preview") == false shouldAddShardFailureCheck("_ml/datafeeds/datafeed-id/_preview") == false
} }
def "can create rest tests from docs"() { def "can generate tests files from asciidoc and mdx"() {
def build = ProjectBuilder.builder().build()
given: given:
def task = build.tasks.create("restTestFromSnippet", RestTestsFromDocSnippetTask) def build = ProjectBuilder.builder().build()
task.expectedUnconvertedCandidates = ["ml-update-snapshot.asciidoc", "reference/security/authorization/run-as-privilege.asciidoc"] def task = build.tasks.register("restTestFromSnippet", RestTestsFromDocSnippetTask).get()
docs() task.expectedUnconvertedCandidates = []
task.docs = build.fileTree(new File(tempDir, "docs")) task.docs = build.fileTree(new File(tempDir, "docs"))
task.testRoot.convention(build.getLayout().buildDirectory.dir("rest-tests")); task.testRoot.convention(build.getLayout().buildDirectory.dir("rest-tests"));
docFile('docs/example-2-asciidoc.asciidoc', SAMPLE_TEST_DOCS['example-2.asciidoc'])
docFile('docs/example-2-mdx.mdx', SAMPLE_TEST_DOCS['example-2.mdx'])
task.getSetups().put(
"seats", """
'''
- do:
indices.create:
index: seats
body:
settings:
number_of_shards: 1
number_of_replicas: 0
mappings:
properties:
theatre:
type: keyword
"""
)
when: when:
task.getActions().forEach { it.execute(task) } task.getActions().forEach { it.execute(task) }
def restSpec = new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/painless-debugging.yml")
then: then:
restSpec.exists() def restSpecFromAsciidoc = new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2-asciidoc.yml")
normalizeString(restSpec.text, tempDir) == """--- def restSpecFromMdx = new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2-mdx.yml")
"line_22": normalizeRestSpec(restSpecFromAsciidoc.text) == normalizeRestSpec(restSpecFromMdx.text)
- skip: }
features:
- default_shards def "task fails on same doc source file with supported different extension"() {
- stash_in_key given:
- stash_in_path def build = ProjectBuilder.builder().build()
- stash_path_replace def task = build.tasks.register("restTestFromSnippet", RestTestsFromDocSnippetTask).get()
- warnings task.expectedUnconvertedCandidates = []
task.docs = build.fileTree(new File(tempDir, "docs"))
task.testRoot.convention(build.getLayout().buildDirectory.dir("rest-tests"));
docFile('docs/example-2.asciidoc', SAMPLE_TEST_DOCS['example-2.asciidoc'])
docFile('docs/example-2.mdx', SAMPLE_TEST_DOCS['example-2.mdx'])
task.getSetups().put(
"seats", """
'''
- do: - do:
raw: indices.create:
method: PUT index: seats
path: "hockey/_doc/1" body:
refresh: "" settings:
body: | number_of_shards: 1
{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]} number_of_replicas: 0
- is_false: _shards.failures mappings:
properties:
theatre:
type: keyword
"""
)
when:
task.getActions().forEach { it.execute(task) }
then:
def e = thrown(GradleException)
e.message == "Found multiple files with the same name 'example-2' but different extensions: [asciidoc, mdx]"
}
def "can run in migration mode to compare same doc source file with supported different extension"() {
given:
def build = ProjectBuilder.builder().build()
def task = build.tasks.register("restTestFromSnippet", RestTestsFromDocSnippetTask).get()
task.expectedUnconvertedCandidates = []
task.migrationMode = true
task.docs = build.fileTree(new File(tempDir, "docs"))
task.testRoot.convention(build.getLayout().buildDirectory.dir("rest-tests"));
docFile('docs/example-2.asciidoc', SAMPLE_TEST_DOCS['example-2.asciidoc'])
docFile('docs/example-2.mdx', SAMPLE_TEST_DOCS['example-2.mdx'])
task.getSetups().put(
"seats", """
'''
- do: - do:
catch: /painless_explain_error/ indices.create:
raw: index: seats
method: POST body:
path: "hockey/_explain/1" settings:
error_trace: "false" number_of_shards: 1
body: | number_of_replicas: 0
{ mappings:
"query": { properties:
"script": { theatre:
"script": "Debug.explain(doc.goals)" type: keyword
} """
} )
} when:
- is_false: _shards.failures task.getActions().forEach { it.execute(task) }
- match:
\$body: then:
{ new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2.asciidoc.yml").exists()
"error": { new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2.mdx.yml").exists()
"type": "script_exception",
"to_string": "[1, 9, 27]",
"painless_class": "org.elasticsearch.index.fielddata.ScriptDocValues.Longs",
"java_class": "org.elasticsearch.index.fielddata.ScriptDocValues\$Longs",
"script_stack": \$body.error.script_stack, "script": \$body.error.script, "lang": \$body.error.lang, "position": \$body.error.position, "caused_by": \$body.error.caused_by, "root_cause": \$body.error.root_cause, "reason": \$body.error.reason
},
"status": 400
} }
def "fails in migration mode for same doc source file with different extension generates different spec"() {
given:
def build = ProjectBuilder.builder().build()
def task = build.tasks.register("restTestFromSnippet", RestTestsFromDocSnippetTask).get()
task.getMigrationMode().set(true)
task.docs = build.fileTree(new File(tempDir, "docs"))
task.testRoot.convention(build.getLayout().buildDirectory.dir("rest-tests"));
docFile('docs/example-2.asciidoc', SAMPLE_TEST_DOCS['example-2.asciidoc'])
docFile('docs/example-2.mdx', SAMPLE_TEST_DOCS['example-2-different.mdx'])
task.getSetups().put(
"seats", """
'''
- do: - do:
catch: /painless_explain_error/ indices.create:
raw: index: seats
method: POST body:
path: "hockey/_update/1" settings:
error_trace: "false" number_of_shards: 1
body: | number_of_replicas: 0
{ mappings:
"script": "Debug.explain(ctx._source)" properties:
} theatre:
- is_false: _shards.failures type: keyword
- match: """
\$body: )
{ when:
"error" : { task.getActions().forEach { it.execute(task) }
"root_cause": \$body.error.root_cause,
"type": "illegal_argument_exception", then:
"reason": "failed to execute script", new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2.asciidoc.yml").exists()
"caused_by": { new File(task.getTestRoot().get().getAsFile(), "rest-api-spec/test/example-2.mdx.yml").exists()
"type": "script_exception",
"to_string": \$body.error.caused_by.to_string,
"painless_class": "java.util.LinkedHashMap",
"java_class": "java.util.LinkedHashMap",
"script_stack": \$body.error.caused_by.script_stack, "script": \$body.error.caused_by.script, "lang": \$body.error.caused_by.lang, "position": \$body.error.caused_by.position, "caused_by": \$body.error.caused_by.caused_by, "reason": \$body.error.caused_by.reason
}
},
"status": 400
}"""
def restSpec2 = new File(task.testRoot.get().getAsFile(), "rest-api-spec/test/ml-update-snapshot.yml")
restSpec2.exists()
normalizeString(restSpec2.text, tempDir) == """---
"line_50":
- skip:
features:
- default_shards
- stash_in_key
- stash_in_path
- stash_path_replace
- warnings
- always_skip
reason: todo
- do:
raw:
method: POST
path: "_ml/anomaly_detectors/it_ops_new_logs/model_snapshots/1491852978/_update"
body: |
{
"description": "Snapshot 1",
"retain": true
}
- is_false: _shards.failures"""
def restSpec3 = new File(task.testRoot.get().getAsFile(), "rest-api-spec/test/reference/sql/getting-started.yml")
restSpec3.exists()
normalizeString(restSpec3.text, tempDir) == """---
"line_10":
- skip:
features:
- default_shards
- stash_in_key
- stash_in_path
- stash_path_replace
- warnings
- do:
raw:
method: PUT
path: "library/_bulk"
refresh: ""
body: |
{"index":{"_id": "Leviathan Wakes"}}
{"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561}
{"index":{"_id": "Hyperion"}}
{"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482}
{"index":{"_id": "Dune"}}
{"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604}
- is_false: _shards.failures
- do:
raw:
method: POST
path: "_sql"
format: "txt"
body: |
{
"query": "SELECT * FROM library WHERE release_date < '2000-01-01'"
}
- is_false: _shards.failures
- match:
\$body:
/ /s+author /s+/| /s+name /s+/| /s+page_count /s+/| /s+release_date/s*
---------------/+---------------/+---------------/+------------------------/s*
Dan /s+Simmons /s+/|Hyperion /s+/|482 /s+/|1989-05-26T00:00:00.000Z/s*
Frank /s+Herbert /s+/|Dune /s+/|604 /s+/|1965-06-01T00:00:00.000Z/s*/"""
def restSpec4 = new File(task.testRoot.get().getAsFile(), "rest-api-spec/test/reference/security/authorization/run-as-privilege.yml")
restSpec4.exists()
normalizeString(restSpec4.text, tempDir) == """---
"line_51":
- skip:
features:
- default_shards
- stash_in_key
- stash_in_path
- stash_path_replace
- warnings
- do:
raw:
method: POST
path: "_security/role/my_director"
refresh: "true"
body: |
{
"cluster": ["manage"],
"indices": [
{
"names": [ "index1", "index2" ],
"privileges": [ "manage" ]
}
],
"run_as": [ "jacknich", "rdeniro" ],
"metadata" : {
"version" : 1
}
}
- is_false: _shards.failures
---
"line_114":
- skip:
features:
- default_shards
- stash_in_key
- stash_in_path
- stash_path_replace
- warnings
- do:
raw:
method: POST
path: "_security/role/my_admin_role"
refresh: "true"
body: |
{
"cluster": ["manage"],
"indices": [
{
"names": [ "index1", "index2" ],
"privileges": [ "manage" ]
}
],
"applications": [
{
"application": "myapp",
"privileges": [ "admin", "read" ],
"resources": [ "*" ]
}
],
"run_as": [ "analyst_user" ],
"metadata" : {
"version" : 1
}
}
- is_false: _shards.failures
---
"line_143":
- skip:
features:
- default_shards
- stash_in_key
- stash_in_path
- stash_path_replace
- warnings
- do:
raw:
method: POST
path: "_security/role/my_analyst_role"
refresh: "true"
body: |
{
"cluster": [ "monitor"],
"indices": [
{
"names": [ "index1", "index2" ],
"privileges": ["manage"]
}
],
"applications": [
{
"application": "myapp",
"privileges": [ "read" ],
"resources": [ "*" ]
}
],
"metadata" : {
"version" : 1
}
}
- is_false: _shards.failures
---
"line_170":
- skip:
features:
- default_shards
- stash_in_key
- stash_in_path
- stash_path_replace
- warnings
- do:
raw:
method: POST
path: "_security/user/admin_user"
refresh: "true"
body: |
{
"password": "l0ng-r4nd0m-p@ssw0rd",
"roles": [ "my_admin_role" ],
"full_name": "Eirian Zola",
"metadata": { "intelligence" : 7}
}
- is_false: _shards.failures
---
"line_184":
- skip:
features:
- default_shards
- stash_in_key
- stash_in_path
- stash_path_replace
- warnings
- do:
raw:
method: POST
path: "_security/user/analyst_user"
refresh: "true"
body: |
{
"password": "l0nger-r4nd0mer-p@ssw0rd",
"roles": [ "my_analyst_role" ],
"full_name": "Monday Jaffe",
"metadata": { "innovation" : 8}
}
- is_false: _shards.failures"""
} }
File docFile(String fileName, String docContent) { File docFile(String fileName, String docContent) {
@ -361,473 +199,8 @@ class RestTestsFromDocSnippetTaskSpec extends Specification {
return file return file
} }
String normalizeRestSpec(String inputString) {
void docs() { def withNormalizedLines = inputString.replaceAll(/"line_\d+":/, "\"line_0\":")
docFile( return withNormalizedLines
"docs/reference/sql/getting-started.asciidoc", """
[role="xpack"]
[[sql-getting-started]]
== Getting Started with SQL
To start using {es-sql}, create
an index with some data to experiment with:
[source,console]
--------------------------------------------------
PUT /library/_bulk?refresh
{"index":{"_id": "Leviathan Wakes"}}
{"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561}
{"index":{"_id": "Hyperion"}}
{"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482}
{"index":{"_id": "Dune"}}
{"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604}
--------------------------------------------------
And now you can execute SQL using the <<sql-search-api,SQL search API>>:
[source,console]
--------------------------------------------------
POST /_sql?format=txt
{
"query": "SELECT * FROM library WHERE release_date < '2000-01-01'"
}
--------------------------------------------------
// TEST[continued]
Which should return something along the lines of:
[source,text]
--------------------------------------------------
author | name | page_count | release_date
---------------+---------------+---------------+------------------------
Dan Simmons |Hyperion |482 |1989-05-26T00:00:00.000Z
Frank Herbert |Dune |604 |1965-06-01T00:00:00.000Z
--------------------------------------------------
// TESTRESPONSE[s/\\|/\\\\|/ s/\\+/\\\\+/]
// TESTRESPONSE[non_json]
You can also use the <<sql-cli>>. There is a script to start it
shipped in x-pack's bin directory:
[source,bash]
--------------------------------------------------
\$ ./bin/elasticsearch-sql-cli
--------------------------------------------------
From there you can run the same query:
[source,sqlcli]
--------------------------------------------------
sql> SELECT * FROM library WHERE release_date < '2000-01-01';
author | name | page_count | release_date
---------------+---------------+---------------+------------------------
Dan Simmons |Hyperion |482 |1989-05-26T00:00:00.000Z
Frank Herbert |Dune |604 |1965-06-01T00:00:00.000Z
--------------------------------------------------
"""
)
docFile(
"docs/ml-update-snapshot.asciidoc",
"""
[role="xpack"]
[[ml-update-snapshot]]
= Update model snapshots API
++++
<titleabbrev>Update model snapshots</titleabbrev>
++++
Updates certain properties of a snapshot.
[[ml-update-snapshot-request]]
== {api-request-title}
`POST _ml/anomaly_detectors/<job_id>/model_snapshots/<snapshot_id>/_update`
[[ml-update-snapshot-prereqs]]
== {api-prereq-title}
Requires the `manage_ml` cluster privilege. This privilege is included in the
`machine_learning_admin` built-in role.
[[ml-update-snapshot-path-parms]]
== {api-path-parms-title}
`<job_id>`::
(Required, string)
include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
`<snapshot_id>`::
(Required, string)
include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=snapshot-id]
[[ml-update-snapshot-request-body]]
== {api-request-body-title}
The following properties can be updated after the model snapshot is created:
`description`::
(Optional, string) A description of the model snapshot.
`retain`::
(Optional, Boolean)
include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=retain]
[[ml-update-snapshot-example]]
== {api-examples-title}
[source,console]
--------------------------------------------------
POST
_ml/anomaly_detectors/it_ops_new_logs/model_snapshots/1491852978/_update
{
"description": "Snapshot 1",
"retain": true
}
--------------------------------------------------
// TEST[skip:todo]
When the snapshot is updated, you receive the following results:
[source,js]
----
{
"acknowledged": true,
"model": {
"job_id": "it_ops_new_logs",
"timestamp": 1491852978000,
"description": "Snapshot 1",
...
"retain": true
}
}
----
"""
)
docFile(
"docs/painless-debugging.asciidoc",
"""
[[painless-debugging]]
=== Painless Debugging
==== Debug.Explain
Painless doesn't have a
{wikipedia}/Read%E2%80%93eval%E2%80%93print_loop[REPL]
and while it'd be nice for it to have one day, it wouldn't tell you the
whole story around debugging painless scripts embedded in Elasticsearch because
the data that the scripts have access to or "context" is so important. For now
the best way to debug embedded scripts is by throwing exceptions at choice
places. While you can throw your own exceptions
(`throw new Exception('whatever')`), Painless's sandbox prevents you from
accessing useful information like the type of an object. So Painless has a
utility method, `Debug.explain` which throws the exception for you. For
example, you can use {ref}/search-explain.html[`_explain`] to explore the
context available to a {ref}/query-dsl-script-query.html[script query].
[source,console]
---------------------------------------------------------
PUT /hockey/_doc/1?refresh
{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
POST /hockey/_explain/1
{
"query": {
"script": {
"script": "Debug.explain(doc.goals)"
}
}
}
---------------------------------------------------------
// TEST[s/_explain\\/1/_explain\\/1?error_trace=false/ catch:/painless_explain_error/]
// The test system sends error_trace=true by default for easier debugging so
// we have to override it to get a normal shaped response
Which shows that the class of `doc.first` is
`org.elasticsearch.index.fielddata.ScriptDocValues.Longs` by responding with:
[source,console-result]
---------------------------------------------------------
{
"error": {
"type": "script_exception",
"to_string": "[1, 9, 27]",
"painless_class": "org.elasticsearch.index.fielddata.ScriptDocValues.Longs",
"java_class": "org.elasticsearch.index.fielddata.ScriptDocValues\$Longs",
...
},
"status": 400
}
---------------------------------------------------------
// TESTRESPONSE[s/\\.\\.\\./"script_stack": \$body.error.script_stack, "script": \$body.error.script, "lang": \$body.error.lang, "position": \$body.error.position, "caused_by": \$body.error.caused_by, "root_cause": \$body.error.root_cause, "reason": \$body.error.reason/]
You can use the same trick to see that `_source` is a `LinkedHashMap`
in the `_update` API:
[source,console]
---------------------------------------------------------
POST /hockey/_update/1
{
"script": "Debug.explain(ctx._source)"
}
---------------------------------------------------------
// TEST[continued s/_update\\/1/_update\\/1?error_trace=false/ catch:/painless_explain_error/]
The response looks like:
[source,console-result]
---------------------------------------------------------
{
"error" : {
"root_cause": ...,
"type": "illegal_argument_exception",
"reason": "failed to execute script",
"caused_by": {
"type": "script_exception",
"to_string": "{gp=[26, 82, 1], last=gaudreau, assists=[17, 46, 0], first=johnny, goals=[9, 27, 1]}",
"painless_class": "java.util.LinkedHashMap",
"java_class": "java.util.LinkedHashMap",
...
}
},
"status": 400
}
---------------------------------------------------------
// TESTRESPONSE[s/"root_cause": \\.\\.\\./"root_cause": \$body.error.root_cause/]
// TESTRESPONSE[s/\\.\\.\\./"script_stack": \$body.error.caused_by.script_stack, "script": \$body.error.caused_by.script, "lang": \$body.error.caused_by.lang, "position": \$body.error.caused_by.position, "caused_by": \$body.error.caused_by.caused_by, "reason": \$body.error.caused_by.reason/]
// TESTRESPONSE[s/"to_string": ".+"/"to_string": \$body.error.caused_by.to_string/]
Once you have a class you can go to <<painless-api-reference>> to see a list of
available methods.
"""
)
docFile(
"docs/reference/security/authorization/run-as-privilege.asciidoc",
"""[role="xpack"]
[[run-as-privilege]]
= Submitting requests on behalf of other users
{es} roles support a `run_as` privilege that enables an authenticated user to
submit requests on behalf of other users. For example, if your external
application is trusted to authenticate users, {es} can authenticate the external
application and use the _run as_ mechanism to issue authorized requests as
other users without having to re-authenticate each user.
To "run as" (impersonate) another user, the first user (the authenticating user)
must be authenticated by a mechanism that supports run-as delegation. The second
user (the `run_as` user) must be authorized by a mechanism that supports
delegated run-as lookups by username.
The `run_as` privilege essentially operates like a secondary form of
<<authorization_realms,delegated authorization>>. Delegated authorization applies
to the authenticating user, and the `run_as` privilege applies to the user who
is being impersonated.
Authenticating user::
--
For the authenticating user, the following realms (plus API keys) all support
`run_as` delegation: `native`, `file`, Active Directory, JWT, Kerberos, LDAP and
PKI.
Service tokens, the {es} Token Service, SAML 2.0, and OIDC 1.0 do not
support `run_as` delegation.
--
`run_as` user::
--
{es} supports `run_as` for any realm that supports user lookup.
Not all realms support user lookup. Refer to the list of <<user-lookup,supported realms>>
and ensure that the realm you wish to use is configured in a manner that
supports user lookup.
The `run_as` user must be retrieved from a <<realms,realm>> - it is not
possible to run as a
<<service-accounts,service account>>,
<<token-authentication-api-key,API key>> or
<<token-authentication-access-token,access token>>.
--
To submit requests on behalf of other users, you need to have the `run_as`
privilege in your <<defining-roles,roles>>. For example, the following request
creates a `my_director` role that grants permission to submit request on behalf
of `jacknich` or `redeniro`:
[source,console]
----
POST /_security/role/my_director?refresh=true
{
"cluster": ["manage"],
"indices": [
{
"names": [ "index1", "index2" ],
"privileges": [ "manage" ]
}
],
"run_as": [ "jacknich", "rdeniro" ],
"metadata" : {
"version" : 1
}
}
----
To submit a request as another user, you specify the user in the
`es-security-runas-user` request header. For example:
[source,sh]
----
curl -H "es-security-runas-user: jacknich" -u es-admin -X GET http://localhost:9200/
----
The `run_as` user passed in through the `es-security-runas-user` header must be
available from a realm that supports delegated authorization lookup by username.
Realms that don't support user lookup can't be used by `run_as` delegation from
other realms.
For example, JWT realms can authenticate external users specified in JWTs, and
execute requests as a `run_as` user in the `native` realm. {es} will retrieve the
indicated `runas` user and execute the request as that user using their roles.
[[run-as-privilege-apply]]
== Apply the `run_as` privilege to roles
You can apply the `run_as` privilege when creating roles with the
<<security-api-put-role,create or update roles API>>. Users who are assigned
a role that contains the `run_as` privilege inherit all privileges from their
role, and can also submit requests on behalf of the indicated users.
NOTE: Roles for the authenticated user and the `run_as` user are not merged. If
a user authenticates without specifying the `run_as` parameter, only the
authenticated user's roles are used. If a user authenticates and their roles
include the `run_as` parameter, only the `run_as` user's roles are used.
After a user successfully authenticates to {es}, an authorization process determines whether the user behind an incoming request is allowed to run
that request. If the authenticated user has the `run_as` privilege in their list
of permissions and specifies the run-as header, {es} _discards_ the authenticated
user and associated roles. It then looks in each of the configured realms in the
realm chain until it finds the username that's associated with the `run_as` user,
and uses those roles to execute any requests.
Consider an admin role and an analyst role. The admin role has higher privileges,
but might also want to submit requests as another user to test and verify their
permissions.
First, we'll create an admin role named `my_admin_role`. This role has `manage`
<<security-privileges,privileges>> on the entire cluster, and on a subset of
indices. This role also contains the `run_as` privilege, which enables any user
with this role to submit requests on behalf of the specified `analyst_user`.
[source,console]
----
POST /_security/role/my_admin_role?refresh=true
{
"cluster": ["manage"],
"indices": [
{
"names": [ "index1", "index2" ],
"privileges": [ "manage" ]
}
],
"applications": [
{
"application": "myapp",
"privileges": [ "admin", "read" ],
"resources": [ "*" ]
}
],
"run_as": [ "analyst_user" ],
"metadata" : {
"version" : 1
}
}
----
Next, we'll create an analyst role named `my_analyst_role`, which has more
restricted `monitor` cluster privileges and `manage` privileges on a subset of
indices.
[source,console]
----
POST /_security/role/my_analyst_role?refresh=true
{
"cluster": [ "monitor"],
"indices": [
{
"names": [ "index1", "index2" ],
"privileges": ["manage"]
}
],
"applications": [
{
"application": "myapp",
"privileges": [ "read" ],
"resources": [ "*" ]
}
],
"metadata" : {
"version" : 1
}
}
----
We'll create an administrator user and assign them the role named `my_admin_role`,
which allows this user to submit requests as the `analyst_user`.
[source,console]
----
POST /_security/user/admin_user?refresh=true
{
"password": "l0ng-r4nd0m-p@ssw0rd",
"roles": [ "my_admin_role" ],
"full_name": "Eirian Zola",
"metadata": { "intelligence" : 7}
}
----
We can also create an analyst user and assign them the role named
`my_analyst_role`.
[source,console]
----
POST /_security/user/analyst_user?refresh=true
{
"password": "l0nger-r4nd0mer-p@ssw0rd",
"roles": [ "my_analyst_role" ],
"full_name": "Monday Jaffe",
"metadata": { "innovation" : 8}
}
----
You can then authenticate to {es} as the `admin_user` or `analyst_user`. However, the `admin_user` could optionally submit requests on
behalf of the `analyst_user`. The following request authenticates to {es} with a
`Basic` authorization token and submits the request as the `analyst_user`:
[source,sh]
----
curl -s -X GET -H "Authorization: Basic YWRtaW5fdXNlcjpsMG5nLXI0bmQwbS1wQHNzdzByZA==" -H "es-security-runas-user: analyst_user" https://localhost:9200/_security/_authenticate
----
The response indicates that the `analyst_user` submitted this request, using the
`my_analyst_role` that's assigned to that user. When the `admin_user` submitted
the request, {es} authenticated that user, discarded their roles, and then used
the roles of the `run_as` user.
[source,sh]
----
{"username":"analyst_user","roles":["my_analyst_role"],"full_name":"Monday Jaffe","email":null,
"metadata":{"innovation":8},"enabled":true,"authentication_realm":{"name":"native",
"type":"native"},"lookup_realm":{"name":"native","type":"native"},"authentication_type":"realm"}
%
----
The `authentication_realm` and `lookup_realm` in the response both specify
the `native` realm because both the `admin_user` and `analyst_user` are from
that realm. If the two users are in different realms, the values for
`authentication_realm` and `lookup_realm` are different (such as `pki` and
`native`).
"""
)
} }
} }

107
build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/SnippetBuilderSpec.groovy Normal file
View file

@ -0,0 +1,107 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.gradle.internal.doc
import spock.lang.Specification
import spock.lang.Unroll
import org.gradle.api.InvalidUserDataException
class SnippetBuilderSpec extends Specification {
@Unroll
def "checks for valid json for #languageParam"() {
when:
def snippet1 = snippetBuilder().withLanguage(languageParam).withTestResponse(true).withConsole(true)
.withContent(
"""{
"name": "John Doe",
"age": 30,
"isMarried": true,
"address": {
"street": "123 Main Street",
"city": "Springfield",
"state": "IL",
"zip": "62701"
},
"hobbies": ["Reading", "Cooking", "Traveling"]
}"""
).build()
then:
snippet1 != null
when:
snippetBuilder().withLanguage(languageParam).withTestResponse(true).withConsole(true)
.withContent(
"some no valid json"
).build()
then:
def e = thrown(InvalidUserDataException)
e.message.contains("Invalid json in")
when:
def snippet2 = snippetBuilder().withLanguage(languageParam).withTestResponse(true).withConsole(true)
.withSkip("skipping")
.withContent(
"some no valid json"
).build()
then:
snippet2 != null
where:
languageParam << ["js", "console-result"]
}
def "language must be defined"() {
when:
snippetBuilder().withContent("snippet-content").build()
then:
def e = thrown(InvalidUserDataException)
e.message.contains("Snippet missing a language.")
}
def "handles snippets with curl"() {
expect:
snippetBuilder().withLanguage("sh")
.withName("snippet-name-1")
.withContent("curl substDefault subst")
.build()
.curl() == true
}
def "snippet builder handles substitutions"() {
when:
def snippet = snippetBuilder().withLanguage("console").withContent("snippet-content substDefault subst")
.withSubstitutions([substDefault: "\$body", subst: 'substValue']).build()
then:
snippet.contents == "snippet-content \$body substValue"
}
def "test snippets with no curl no console"() {
when:
snippetBuilder()
.withConsole(false)
.withLanguage("shell")
.withContent("hello substDefault subst")
.build()
then:
def e = thrown(InvalidUserDataException)
e.message.contains("No need for NOTCONSOLE if snippet doesn't contain `curl`")
}
SnippetBuilder snippetBuilder() {
return new SnippetBuilder()
}
}

4
build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestUtils.java
View file

@ -14,6 +14,10 @@ import java.util.stream.Collectors;
public class TestUtils { public class TestUtils {
public static String normalizeString(String input) {
return normalizeString(input, new File("."));
}
public static String normalizeString(String input, File projectRootDir) { public static String normalizeString(String input, File projectRootDir) {
try { try {
String canonicalNormalizedPathPrefix = projectRootDir.getCanonicalPath().replace('\\', '/'); String canonicalNormalizedPathPrefix = projectRootDir.getCanonicalPath().replace('\\', '/');

17
docs/build.gradle
View file

@ -1,6 +1,6 @@
import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.info.BuildParams
import org.elasticsearch.gradle.internal.doc.DocSnippetTask
import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT
/* /*
@ -16,6 +16,7 @@ apply plugin: 'elasticsearch.rest-resources'
ext.docsFileTree = fileTree(projectDir) { ext.docsFileTree = fileTree(projectDir) {
include '**/*.asciidoc' include '**/*.asciidoc'
include '**/*.mdx'
// That is where the snippets go, not where they come from! // That is where the snippets go, not where they come from!
exclude 'build/**' exclude 'build/**'
exclude 'build-idea/**' exclude 'build-idea/**'
@ -37,7 +38,7 @@ ext.docsFileTree = fileTree(projectDir) {
/* List of files that have snippets that will not work until platinum tests can occur ... */ /* List of files that have snippets that will not work until platinum tests can occur ... */
tasks.named("buildRestTests").configure { tasks.named("buildRestTests").configure {
expectedUnconvertedCandidates = [ getExpectedUnconvertedCandidates().addAll(
'reference/ml/anomaly-detection/ml-configuring-transform.asciidoc', 'reference/ml/anomaly-detection/ml-configuring-transform.asciidoc',
'reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc', 'reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc',
'reference/ml/anomaly-detection/apis/get-bucket.asciidoc', 'reference/ml/anomaly-detection/apis/get-bucket.asciidoc',
@ -58,7 +59,7 @@ tasks.named("buildRestTests").configure {
'reference/rest-api/watcher/put-watch.asciidoc', 'reference/rest-api/watcher/put-watch.asciidoc',
'reference/rest-api/watcher/stats.asciidoc', 'reference/rest-api/watcher/stats.asciidoc',
'reference/watcher/example-watches/watching-time-series-data.asciidoc' 'reference/watcher/example-watches/watching-time-series-data.asciidoc'
] )
} }
restResources { restResources {
@ -176,15 +177,7 @@ tasks.named("forbiddenPatterns").configure {
exclude '**/*.mmdb' exclude '**/*.mmdb'
} }
tasks.named("buildRestTests").configure { tasks.withType(DocSnippetTask).configureEach {
docs = docsFileTree
}
tasks.named("listSnippets").configure {
docs = docsFileTree
}
tasks.named("listConsoleCandidates").configure {
docs = docsFileTree docs = docsFileTree
} }