diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java index 209d08823192..79b46dfa31f9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java @@ -10,6 +10,7 @@ package org.elasticsearch.client.eql; import org.apache.lucene.search.TotalHits; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.get.GetResult; @@ -25,6 +26,7 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -144,16 +146,19 @@ public class EqlSearchResponse { static final String INDEX = GetResult._INDEX; static final String ID = GetResult._ID; static final String SOURCE = SourceFieldMapper.NAME; + static final String FIELDS = "fields"; } private static final ParseField INDEX = new ParseField(Fields.INDEX); private static final ParseField ID = new ParseField(Fields.ID); private static final ParseField SOURCE = new ParseField(Fields.SOURCE); + private static final ParseField FIELDS = new ParseField(Fields.FIELDS); + @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "eql/search_response_event", true, - args -> new Event((String) args[0], (String) args[1], (BytesReference) args[2]) + args -> new Event((String) args[0], (String) args[1], (BytesReference) args[2], (Map) args[3]) ); static { @@ -165,19 +170,35 @@ public class EqlSearchResponse { return BytesReference.bytes(builder); } }, SOURCE); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> { + Map fields = new HashMap<>(); + while (p.nextToken() != XContentParser.Token.END_OBJECT) { + DocumentField field = DocumentField.fromXContent(p); + fields.put(field.getName(), field); + } + return fields; + }, FIELDS); } private final String index; private final String id; private final BytesReference source; private Map sourceAsMap; + private final Map fetchFields; + @Deprecated public Event(String index, String id, BytesReference source) { + this(index, id, source, null); + } + + private Event(String index, String id, BytesReference source, Map fetchFields) { this.index = index; this.id = id; this.source = source; + this.fetchFields = fetchFields; } + @Deprecated public static Event fromXContent(XContentParser parser) throws IOException { return PARSER.apply(parser, null); } @@ -194,6 +215,10 @@ public class EqlSearchResponse { return source; } + public Map fetchFields() { + return fetchFields; + } + public Map sourceAsMap() { if (source == null) { return null; @@ -208,7 +233,7 @@ public class EqlSearchResponse { @Override public int hashCode() { - return Objects.hash(index, id, source); + return Objects.hash(index, id, source, fetchFields); } @Override @@ -222,7 +247,10 @@ public class EqlSearchResponse { } EqlSearchResponse.Event other = (EqlSearchResponse.Event) obj; - return Objects.equals(index, other.index) && Objects.equals(id, other.id) && Objects.equals(source, other.source); + return Objects.equals(index, other.index) + && Objects.equals(id, other.id) + && Objects.equals(source, other.source) + && Objects.equals(fetchFields, other.fetchFields); } } @@ -262,11 +290,13 @@ public class EqlSearchResponse { private final List joinKeys; private final List events; + @Deprecated public Sequence(List joinKeys, List events) { this.joinKeys = joinKeys == null ? Collections.emptyList() : joinKeys; this.events = events == null ? Collections.emptyList() : events; } + @Deprecated public static Sequence fromXContent(XContentParser parser) { return PARSER.apply(parser, null); } @@ -311,6 +341,7 @@ public class EqlSearchResponse { static final String SEQUENCES = "sequences"; } + @Deprecated public Hits(@Nullable List events, @Nullable List sequences, @Nullable TotalHits totalHits) { this.events = events; this.sequences = sequences; @@ -345,6 +376,7 @@ public class EqlSearchResponse { ); } + @Deprecated public static Hits fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/EqlDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/EqlDocumentationIT.java new file mode 100644 index 000000000000..cb926c61db7e --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/EqlDocumentationIT.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.client.documentation; + +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.client.eql.EqlSearchRequest; +import org.elasticsearch.client.eql.EqlSearchResponse; +import org.elasticsearch.client.eql.EqlSearchResponse.Event; +import org.elasticsearch.client.eql.EqlSearchResponse.Hits; +import org.elasticsearch.client.eql.EqlSearchResponse.Sequence; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xcontent.XContentType.JSON; + +/** + * Documentation for EQL APIs in the high level java client. + * Code wrapped in {@code tag} and {@code end} tags is included in the docs. + */ +@SuppressWarnings("removal") +public class EqlDocumentationIT extends ESRestHighLevelClientTestCase { + + @Before + void setUpIndex() throws IOException { + String index = "my-index"; + CreateIndexResponse createIndexResponse = highLevelClient().indices().create(new CreateIndexRequest(index), RequestOptions.DEFAULT); + assertTrue(createIndexResponse.isAcknowledged()); + BulkRequest bulk = new BulkRequest(index).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulk.add(new IndexRequest().source(JSON, "event_category", "process", "timestamp", "2021-11-23T00:00:00Z", "tie", 1, "host", "A")); + bulk.add(new IndexRequest().source(JSON, "event_category", "process", "timestamp", "2021-11-23T00:00:00Z", "tie", 2, "host", "B")); + bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + highLevelClient().bulk(bulk, RequestOptions.DEFAULT); + } + + public void testEqlSearch() throws Exception { + RestHighLevelClient client = highLevelClient(); + + // tag::eql-search-request + String indices = "my-index"; // <1> + String query = "any where true"; // <2> + EqlSearchRequest request = new EqlSearchRequest(indices, query); + // end::eql-search-request + + // tag::eql-search-request-arguments + request.eventCategoryField("event_category"); // <1> + request.fetchSize(50); // <2> + request.size(15); // <3> + request.tiebreakerField("tie"); // <4> + request.timestampField("timestamp"); // <5> + request.filter(QueryBuilders.matchAllQuery()); // <6> + request.resultPosition("head"); // <7> + + List fields = new ArrayList<>(); + fields.add(new FieldAndFormat("hostname", null)); + request.fetchFields(fields); // <8> + + IndicesOptions op = IndicesOptions.fromOptions(true, true, true, false); + request.indicesOptions(op); // <9> + + Map settings = new HashMap<>(); + settings.put("type", "keyword"); + settings.put("script", "emit(doc['host.keyword'].value)"); + Map field = new HashMap<>(); + field.put("hostname", settings); + request.runtimeMappings(field); // <10> + + request.waitForCompletionTimeout(TimeValue.timeValueMinutes(1)); // <11> + request.keepOnCompletion(true); // <12> + request.keepAlive(TimeValue.timeValueHours(12)); // <13> + // end::eql-search-request-arguments + + // Ignore warning about ignore_throttled being deprecated + RequestOptions options = RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE).build(); + // tag::eql-search-response + EqlSearchResponse response = client.eql().search(request, options); + response.id(); // <1> + response.isPartial(); // <2> + response.isRunning(); // <3> + response.isTimeout(); // <4> + response.took(); // <5> + Hits hits = response.hits(); // <6> + hits.totalHits(); // <7> + List events = hits.events(); // <8> + List sequences = hits.sequences(); // <9> + Map event = events.get(0).sourceAsMap(); + Map fetchField = events.get(0).fetchFields(); + fetchField.get("hostname").getValues(); // <10> + // end::eql-search-response + assertFalse(response.isPartial()); + assertFalse(response.isRunning()); + assertFalse(response.isTimeout()); + assertEquals(2, hits.totalHits().value); + assertEquals(2, events.size()); + assertNull(sequences); + assertEquals(1, fetchField.size()); + assertEquals(1, fetchField.get("hostname").getValues().size()); + assertEquals("A", fetchField.get("hostname").getValues().get(0)); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java index 62743ddd8d4b..489ec822021d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java @@ -93,9 +93,10 @@ public class EqlSearchResponseTests extends AbstractResponseTestCase< Map fetchFields = new HashMap<>(); int fieldsCount = randomIntBetween(0, 5); for (int j = 0; j < fieldsCount; j++) { - fetchFields.put(randomAlphaOfLength(10), randomDocumentField(xType).v1()); + DocumentField doc = randomDocumentField(xType).v2(); + fetchFields.put(doc.getName(), doc); } - if (fetchFields.isEmpty() && randomBoolean()) { + if (fetchFields.isEmpty()) { fetchFields = null; } hits.add( @@ -262,7 +263,10 @@ public class EqlSearchResponseTests extends AbstractResponseTestCase< ) { assertThat(serverEvents.size(), equalTo(clientEvents.size())); for (int j = 0; j < serverEvents.size(); j++) { - assertThat(SourceLookup.sourceAsMap(serverEvents.get(j).source()), is(clientEvents.get(j).sourceAsMap())); + org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event serverEvent = serverEvents.get(j); + EqlSearchResponse.Event clientEvent = clientEvents.get(j); + assertThat(SourceLookup.sourceAsMap(serverEvent.source()), is(clientEvent.sourceAsMap())); + assertThat(serverEvent.fetchFields(), equalTo(clientEvent.fetchFields())); } } } diff --git a/docs/java-rest/high-level/eql/search.asciidoc b/docs/java-rest/high-level/eql/search.asciidoc new file mode 100644 index 000000000000..5307163ec019 --- /dev/null +++ b/docs/java-rest/high-level/eql/search.asciidoc @@ -0,0 +1,71 @@ +-- +:api: eql-search +:request: EqlSearchRequest +:response: EqlSearchResponse +-- + +[role="xpack"] +[id="{upid}-{api}"] +=== EQL Search API + +[id="{upid}-{api}-request"] +==== Request + +A +{request}+ allows to submit an EQL search request. Required arguments are the indices to search against and the query itself: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- +<1> Comma-separated list of data streams, indices, or aliases targeting the local cluster or a remote one, used to limit the request. +Supports wildcards (`*`). To search all data streams and indices, use `*` or `_all`. +<2> The query to execute + +==== Optional arguments +The following arguments can optionally be provided: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-arguments] +-------------------------------------------------- +<1> Field containing the event classification. Defaults to `event.category`, as defined in the Elastic Common Schema (ECS). +<2> Maximum number of events to search at a time for sequence queries (defaults to 1000). +<3> For basic queries, the maximum number of matching events to return. +For sequence queries, the maximum number of matching sequences to return. Defaults to 10. +<4> Field used to sort hits with the same timestamp in ascending order. +<5> Field containing the event timestamp. Defaults to `@timestamp`, as defined in the Elastic Common Schema (ECS). +<6> Query, written in Query DSL, used to filter the events on which the EQL query runs. +<7> Set of matching events or sequences to return. Accepts `tail` (default, return the most recent matches) or `head` (return the earliest matches). +<8> Array of wildcard (*) patterns. The response returns values for field names matching these patterns in the fields property of each hit. +<9> Value of `IndicesOptions` specifying various options for resolving indices names. Defaults to `ignoreUnavailable = true`, +`allowNoIndices = true`, `expandToOpenIndices = true`, `expandToClosedIndices = false`. +<10> Defines one or more runtime fields in the search request. These fields take precedence over mapped fields with the same name. +<11> Timeout duration to wait for the request to finish. Defaults to no timeout, meaning the request waits for complete search results. +If the request does not complete during this period, the search becomes an async search. +<12> If `true`, the search and its results are stored on the cluster. If `false`, the search and its results are stored on the cluster +only if the request does not complete during the period set by the `waitForCompletionTimeout` setting. Defaults to `false`. +<13> Period for which the search and its results are stored on the cluster. Defaults to `5d` (five days). +When this period expires, the search and its results are deleted, even if the search is still ongoing. +If the `keepOnCompletion` setting is `false`, Elasticsearch only stores async searches that do not complete within the period +set by the `waitForCompletionTimeout` setting, regardless of this value. + +[id="{upid}-{api}-response"] +==== Response + +The returned +{response}+ allows to retrieve information about the executed + operation as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- +<1> The id of the async search request, `null` if the response isn't stored. +<2> `true` when the response contains partial results. +<3> `true` when the search is still running. +<4> `true` when the request timed out before completion. +<5> Milliseconds it took Elasticsearch to execute the request. +<6> Contains matching events and sequences. Also contains related metadata. The response will contain either `Event`s or `Sequence`s, not both, depending on the query. +<7> Metadata about the number of matching events or sequences. +<8> Contains events matching the query. Each object represents a matching event. +<9> Contains event sequences matching the query. Each object represents a matching sequence. +<10> Access the value of a runtime field. diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 668f8991fa69..7035c1271c64 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -768,3 +768,15 @@ include::enrich/delete_policy.asciidoc[] include::enrich/get_policy.asciidoc[] include::enrich/stats.asciidoc[] include::enrich/execute_policy.asciidoc[] + +[role="xpack"] +== EQL APIs + +:upid: {mainid}-eql +:doc-tests-file: {doc-tests}/EqlDocumentationIT.java + +The Java High Level REST Client supports the following EQL APIs: + +* <<{upid}-eql-search>> + +include::eql/search.asciidoc[]