mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-24 23:27:25 -04:00
EQL: HLRC documentation (#80979)
This commit is contained in:
Andrei Stefan
GitHub
parent
3798ff6df5
commit
eeb39f8499
5 changed files with 247 additions and 6 deletions
|
|
@ -10,6 +10,7 @@ package org.elasticsearch.client.eql;
|
|||
|
||||
import org.apache.lucene.search.TotalHits;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.document.DocumentField;
|
||||
import org.elasticsearch.common.xcontent.XContentParserUtils;
|
||||
import org.elasticsearch.core.Nullable;
|
||||
import org.elasticsearch.index.get.GetResult;
|
||||
|
|
@ -25,6 +26,7 @@ import org.elasticsearch.xcontent.XContentParser;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
|
@ -144,16 +146,19 @@ public class EqlSearchResponse {
|
|||
static final String INDEX = GetResult._INDEX;
|
||||
static final String ID = GetResult._ID;
|
||||
static final String SOURCE = SourceFieldMapper.NAME;
|
||||
static final String FIELDS = "fields";
|
||||
}
|
||||
|
||||
private static final ParseField INDEX = new ParseField(Fields.INDEX);
|
||||
private static final ParseField ID = new ParseField(Fields.ID);
|
||||
private static final ParseField SOURCE = new ParseField(Fields.SOURCE);
|
||||
private static final ParseField FIELDS = new ParseField(Fields.FIELDS);
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static final ConstructingObjectParser<Event, Void> PARSER = new ConstructingObjectParser<>(
|
||||
"eql/search_response_event",
|
||||
true,
|
||||
args -> new Event((String) args[0], (String) args[1], (BytesReference) args[2])
|
||||
args -> new Event((String) args[0], (String) args[1], (BytesReference) args[2], (Map<String, DocumentField>) args[3])
|
||||
);
|
||||
|
||||
static {
|
||||
|
|
@ -165,19 +170,35 @@ public class EqlSearchResponse {
|
|||
return BytesReference.bytes(builder);
|
||||
}
|
||||
}, SOURCE);
|
||||
PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
|
||||
Map<String, DocumentField> fields = new HashMap<>();
|
||||
while (p.nextToken() != XContentParser.Token.END_OBJECT) {
|
||||
DocumentField field = DocumentField.fromXContent(p);
|
||||
fields.put(field.getName(), field);
|
||||
}
|
||||
return fields;
|
||||
}, FIELDS);
|
||||
}
|
||||
|
||||
private final String index;
|
||||
private final String id;
|
||||
private final BytesReference source;
|
||||
private Map<String, Object> sourceAsMap;
|
||||
private final Map<String, DocumentField> fetchFields;
|
||||
|
||||
@Deprecated
|
||||
public Event(String index, String id, BytesReference source) {
|
||||
this(index, id, source, null);
|
||||
}
|
||||
|
||||
private Event(String index, String id, BytesReference source, Map<String, DocumentField> fetchFields) {
|
||||
this.index = index;
|
||||
this.id = id;
|
||||
this.source = source;
|
||||
this.fetchFields = fetchFields;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public static Event fromXContent(XContentParser parser) throws IOException {
|
||||
return PARSER.apply(parser, null);
|
||||
}
|
||||
|
|
@ -194,6 +215,10 @@ public class EqlSearchResponse {
|
|||
return source;
|
||||
}
|
||||
|
||||
public Map<String, DocumentField> fetchFields() {
|
||||
return fetchFields;
|
||||
}
|
||||
|
||||
public Map<String, Object> sourceAsMap() {
|
||||
if (source == null) {
|
||||
return null;
|
||||
|
|
@ -208,7 +233,7 @@ public class EqlSearchResponse {
|
|||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(index, id, source);
|
||||
return Objects.hash(index, id, source, fetchFields);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -222,7 +247,10 @@ public class EqlSearchResponse {
|
|||
}
|
||||
|
||||
EqlSearchResponse.Event other = (EqlSearchResponse.Event) obj;
|
||||
return Objects.equals(index, other.index) && Objects.equals(id, other.id) && Objects.equals(source, other.source);
|
||||
return Objects.equals(index, other.index)
|
||||
&& Objects.equals(id, other.id)
|
||||
&& Objects.equals(source, other.source)
|
||||
&& Objects.equals(fetchFields, other.fetchFields);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -262,11 +290,13 @@ public class EqlSearchResponse {
|
|||
private final List<Object> joinKeys;
|
||||
private final List<Event> events;
|
||||
|
||||
@Deprecated
|
||||
public Sequence(List<Object> joinKeys, List<Event> events) {
|
||||
this.joinKeys = joinKeys == null ? Collections.emptyList() : joinKeys;
|
||||
this.events = events == null ? Collections.emptyList() : events;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public static Sequence fromXContent(XContentParser parser) {
|
||||
return PARSER.apply(parser, null);
|
||||
}
|
||||
|
|
@ -311,6 +341,7 @@ public class EqlSearchResponse {
|
|||
static final String SEQUENCES = "sequences";
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public Hits(@Nullable List<Event> events, @Nullable List<Sequence> sequences, @Nullable TotalHits totalHits) {
|
||||
this.events = events;
|
||||
this.sequences = sequences;
|
||||
|
|
@ -345,6 +376,7 @@ public class EqlSearchResponse {
|
|||
);
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public static Hits fromXContent(XContentParser parser) throws IOException {
|
||||
return PARSER.parse(parser, null);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,122 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
package org.elasticsearch.client.documentation;
|
||||
|
||||
import org.elasticsearch.action.bulk.BulkRequest;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.support.IndicesOptions;
|
||||
import org.elasticsearch.action.support.WriteRequest;
|
||||
import org.elasticsearch.client.ESRestHighLevelClientTestCase;
|
||||
import org.elasticsearch.client.RequestOptions;
|
||||
import org.elasticsearch.client.RestHighLevelClient;
|
||||
import org.elasticsearch.client.WarningsHandler;
|
||||
import org.elasticsearch.client.eql.EqlSearchRequest;
|
||||
import org.elasticsearch.client.eql.EqlSearchResponse;
|
||||
import org.elasticsearch.client.eql.EqlSearchResponse.Event;
|
||||
import org.elasticsearch.client.eql.EqlSearchResponse.Hits;
|
||||
import org.elasticsearch.client.eql.EqlSearchResponse.Sequence;
|
||||
import org.elasticsearch.client.indices.CreateIndexRequest;
|
||||
import org.elasticsearch.client.indices.CreateIndexResponse;
|
||||
import org.elasticsearch.common.document.DocumentField;
|
||||
import org.elasticsearch.core.TimeValue;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
|
||||
import org.junit.Before;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.elasticsearch.xcontent.XContentType.JSON;
|
||||
|
||||
/**
|
||||
* Documentation for EQL APIs in the high level java client.
|
||||
* Code wrapped in {@code tag} and {@code end} tags is included in the docs.
|
||||
*/
|
||||
@SuppressWarnings("removal")
|
||||
public class EqlDocumentationIT extends ESRestHighLevelClientTestCase {
|
||||
|
||||
@Before
|
||||
void setUpIndex() throws IOException {
|
||||
String index = "my-index";
|
||||
CreateIndexResponse createIndexResponse = highLevelClient().indices().create(new CreateIndexRequest(index), RequestOptions.DEFAULT);
|
||||
assertTrue(createIndexResponse.isAcknowledged());
|
||||
BulkRequest bulk = new BulkRequest(index).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
bulk.add(new IndexRequest().source(JSON, "event_category", "process", "timestamp", "2021-11-23T00:00:00Z", "tie", 1, "host", "A"));
|
||||
bulk.add(new IndexRequest().source(JSON, "event_category", "process", "timestamp", "2021-11-23T00:00:00Z", "tie", 2, "host", "B"));
|
||||
bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
|
||||
}
|
||||
|
||||
public void testEqlSearch() throws Exception {
|
||||
RestHighLevelClient client = highLevelClient();
|
||||
|
||||
// tag::eql-search-request
|
||||
String indices = "my-index"; // <1>
|
||||
String query = "any where true"; // <2>
|
||||
EqlSearchRequest request = new EqlSearchRequest(indices, query);
|
||||
// end::eql-search-request
|
||||
|
||||
// tag::eql-search-request-arguments
|
||||
request.eventCategoryField("event_category"); // <1>
|
||||
request.fetchSize(50); // <2>
|
||||
request.size(15); // <3>
|
||||
request.tiebreakerField("tie"); // <4>
|
||||
request.timestampField("timestamp"); // <5>
|
||||
request.filter(QueryBuilders.matchAllQuery()); // <6>
|
||||
request.resultPosition("head"); // <7>
|
||||
|
||||
List<FieldAndFormat> fields = new ArrayList<>();
|
||||
fields.add(new FieldAndFormat("hostname", null));
|
||||
request.fetchFields(fields); // <8>
|
||||
|
||||
IndicesOptions op = IndicesOptions.fromOptions(true, true, true, false);
|
||||
request.indicesOptions(op); // <9>
|
||||
|
||||
Map<String, Object> settings = new HashMap<>();
|
||||
settings.put("type", "keyword");
|
||||
settings.put("script", "emit(doc['host.keyword'].value)");
|
||||
Map<String, Object> field = new HashMap<>();
|
||||
field.put("hostname", settings);
|
||||
request.runtimeMappings(field); // <10>
|
||||
|
||||
request.waitForCompletionTimeout(TimeValue.timeValueMinutes(1)); // <11>
|
||||
request.keepOnCompletion(true); // <12>
|
||||
request.keepAlive(TimeValue.timeValueHours(12)); // <13>
|
||||
// end::eql-search-request-arguments
|
||||
|
||||
// Ignore warning about ignore_throttled being deprecated
|
||||
RequestOptions options = RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE).build();
|
||||
// tag::eql-search-response
|
||||
EqlSearchResponse response = client.eql().search(request, options);
|
||||
response.id(); // <1>
|
||||
response.isPartial(); // <2>
|
||||
response.isRunning(); // <3>
|
||||
response.isTimeout(); // <4>
|
||||
response.took(); // <5>
|
||||
Hits hits = response.hits(); // <6>
|
||||
hits.totalHits(); // <7>
|
||||
List<Event> events = hits.events(); // <8>
|
||||
List<Sequence> sequences = hits.sequences(); // <9>
|
||||
Map<String, Object> event = events.get(0).sourceAsMap();
|
||||
Map<String, DocumentField> fetchField = events.get(0).fetchFields();
|
||||
fetchField.get("hostname").getValues(); // <10>
|
||||
// end::eql-search-response
|
||||
assertFalse(response.isPartial());
|
||||
assertFalse(response.isRunning());
|
||||
assertFalse(response.isTimeout());
|
||||
assertEquals(2, hits.totalHits().value);
|
||||
assertEquals(2, events.size());
|
||||
assertNull(sequences);
|
||||
assertEquals(1, fetchField.size());
|
||||
assertEquals(1, fetchField.get("hostname").getValues().size());
|
||||
assertEquals("A", fetchField.get("hostname").getValues().get(0));
|
||||
}
|
||||
}
|
||||
|
|
@ -93,9 +93,10 @@ public class EqlSearchResponseTests extends AbstractResponseTestCase<
|
|||
Map<String, DocumentField> fetchFields = new HashMap<>();
|
||||
int fieldsCount = randomIntBetween(0, 5);
|
||||
for (int j = 0; j < fieldsCount; j++) {
|
||||
fetchFields.put(randomAlphaOfLength(10), randomDocumentField(xType).v1());
|
||||
DocumentField doc = randomDocumentField(xType).v2();
|
||||
fetchFields.put(doc.getName(), doc);
|
||||
}
|
||||
if (fetchFields.isEmpty() && randomBoolean()) {
|
||||
if (fetchFields.isEmpty()) {
|
||||
fetchFields = null;
|
||||
}
|
||||
hits.add(
|
||||
|
|
@ -262,7 +263,10 @@ public class EqlSearchResponseTests extends AbstractResponseTestCase<
|
|||
) {
|
||||
assertThat(serverEvents.size(), equalTo(clientEvents.size()));
|
||||
for (int j = 0; j < serverEvents.size(); j++) {
|
||||
assertThat(SourceLookup.sourceAsMap(serverEvents.get(j).source()), is(clientEvents.get(j).sourceAsMap()));
|
||||
org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event serverEvent = serverEvents.get(j);
|
||||
EqlSearchResponse.Event clientEvent = clientEvents.get(j);
|
||||
assertThat(SourceLookup.sourceAsMap(serverEvent.source()), is(clientEvent.sourceAsMap()));
|
||||
assertThat(serverEvent.fetchFields(), equalTo(clientEvent.fetchFields()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
71
docs/java-rest/high-level/eql/search.asciidoc
Normal file
71
docs/java-rest/high-level/eql/search.asciidoc
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
--
|
||||
:api: eql-search
|
||||
:request: EqlSearchRequest
|
||||
:response: EqlSearchResponse
|
||||
--
|
||||
|
||||
[role="xpack"]
|
||||
[id="{upid}-{api}"]
|
||||
=== EQL Search API
|
||||
|
||||
[id="{upid}-{api}-request"]
|
||||
==== Request
|
||||
|
||||
A +{request}+ allows to submit an EQL search request. Required arguments are the indices to search against and the query itself:
|
||||
|
||||
["source","java",subs="attributes,callouts,macros"]
|
||||
--------------------------------------------------
|
||||
include-tagged::{doc-tests-file}[{api}-request]
|
||||
--------------------------------------------------
|
||||
<1> Comma-separated list of data streams, indices, or aliases targeting the local cluster or a remote one, used to limit the request.
|
||||
Supports wildcards (`*`). To search all data streams and indices, use `*` or `_all`.
|
||||
<2> The query to execute
|
||||
|
||||
==== Optional arguments
|
||||
The following arguments can optionally be provided:
|
||||
|
||||
["source","java",subs="attributes,callouts,macros"]
|
||||
--------------------------------------------------
|
||||
include-tagged::{doc-tests-file}[{api}-request-arguments]
|
||||
--------------------------------------------------
|
||||
<1> Field containing the event classification. Defaults to `event.category`, as defined in the Elastic Common Schema (ECS).
|
||||
<2> Maximum number of events to search at a time for sequence queries (defaults to 1000).
|
||||
<3> For basic queries, the maximum number of matching events to return.
|
||||
For sequence queries, the maximum number of matching sequences to return. Defaults to 10.
|
||||
<4> Field used to sort hits with the same timestamp in ascending order.
|
||||
<5> Field containing the event timestamp. Defaults to `@timestamp`, as defined in the Elastic Common Schema (ECS).
|
||||
<6> Query, written in Query DSL, used to filter the events on which the EQL query runs.
|
||||
<7> Set of matching events or sequences to return. Accepts `tail` (default, return the most recent matches) or `head` (return the earliest matches).
|
||||
<8> Array of wildcard (*) patterns. The response returns values for field names matching these patterns in the fields property of each hit.
|
||||
<9> Value of `IndicesOptions` specifying various options for resolving indices names. Defaults to `ignoreUnavailable = true`,
|
||||
`allowNoIndices = true`, `expandToOpenIndices = true`, `expandToClosedIndices = false`.
|
||||
<10> Defines one or more runtime fields in the search request. These fields take precedence over mapped fields with the same name.
|
||||
<11> Timeout duration to wait for the request to finish. Defaults to no timeout, meaning the request waits for complete search results.
|
||||
If the request does not complete during this period, the search becomes an async search.
|
||||
<12> If `true`, the search and its results are stored on the cluster. If `false`, the search and its results are stored on the cluster
|
||||
only if the request does not complete during the period set by the `waitForCompletionTimeout` setting. Defaults to `false`.
|
||||
<13> Period for which the search and its results are stored on the cluster. Defaults to `5d` (five days).
|
||||
When this period expires, the search and its results are deleted, even if the search is still ongoing.
|
||||
If the `keepOnCompletion` setting is `false`, Elasticsearch only stores async searches that do not complete within the period
|
||||
set by the `waitForCompletionTimeout` setting, regardless of this value.
|
||||
|
||||
[id="{upid}-{api}-response"]
|
||||
==== Response
|
||||
|
||||
The returned +{response}+ allows to retrieve information about the executed
|
||||
operation as follows:
|
||||
|
||||
["source","java",subs="attributes,callouts,macros"]
|
||||
--------------------------------------------------
|
||||
include-tagged::{doc-tests-file}[{api}-response]
|
||||
--------------------------------------------------
|
||||
<1> The id of the async search request, `null` if the response isn't stored.
|
||||
<2> `true` when the response contains partial results.
|
||||
<3> `true` when the search is still running.
|
||||
<4> `true` when the request timed out before completion.
|
||||
<5> Milliseconds it took Elasticsearch to execute the request.
|
||||
<6> Contains matching events and sequences. Also contains related metadata. The response will contain either `Event`s or `Sequence`s, not both, depending on the query.
|
||||
<7> Metadata about the number of matching events or sequences.
|
||||
<8> Contains events matching the query. Each object represents a matching event.
|
||||
<9> Contains event sequences matching the query. Each object represents a matching sequence.
|
||||
<10> Access the value of a runtime field.
|
||||
|
|
@ -768,3 +768,15 @@ include::enrich/delete_policy.asciidoc[]
|
|||
include::enrich/get_policy.asciidoc[]
|
||||
include::enrich/stats.asciidoc[]
|
||||
include::enrich/execute_policy.asciidoc[]
|
||||
|
||||
[role="xpack"]
|
||||
== EQL APIs
|
||||
|
||||
:upid: {mainid}-eql
|
||||
:doc-tests-file: {doc-tests}/EqlDocumentationIT.java
|
||||
|
||||
The Java High Level REST Client supports the following EQL APIs:
|
||||
|
||||
* <<{upid}-eql-search>>
|
||||
|
||||
include::eql/search.asciidoc[]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue