mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-25 07:37:19 -04:00
2f320f1bf6
* Revert "Revert "[DOCS] Add docs for EQL missing events (#97372)" (#98028)"
This reverts commit 46c81938d9.
* Changed response for missing events
1286 lines
39 KiB
Text
1286 lines
39 KiB
Text
[role="xpack"]
|
|
[[eql-syntax]]
|
|
== EQL syntax reference
|
|
++++
|
|
<titleabbrev>Syntax reference</titleabbrev>
|
|
++++
|
|
|
|
[discrete]
|
|
[[eql-basic-syntax]]
|
|
=== Basic syntax
|
|
|
|
EQL queries require an event category and a matching condition. The `where`
|
|
keyword connects them.
|
|
|
|
[source,eql]
|
|
----
|
|
event_category where condition
|
|
----
|
|
|
|
An event category is an indexed value of the <<eql-required-fields,event
|
|
category field>>. By default, the <<eql-search-api,EQL search API>> uses the
|
|
`event.category` field from the {ecs-ref}[Elastic Common Schema (ECS)]. You can
|
|
specify another event category field using the API's
|
|
<<specify-a-timestamp-or-event-category-field,`event_category_field`>>
|
|
parameter.
|
|
|
|
For example, the following EQL query matches events with an event category of
|
|
`process` and a `process.name` of `svchost.exe`:
|
|
|
|
[source,eql]
|
|
----
|
|
process where process.name == "svchost.exe"
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-syntax-match-any-event-category]]
|
|
==== Match any event category
|
|
|
|
To match events of any category, use the `any` keyword. You can also use the
|
|
`any` keyword to search for documents without a event category field.
|
|
|
|
For example, the following EQL query matches any documents with a
|
|
`network.protocol` field value of `http`:
|
|
|
|
[source,eql]
|
|
----
|
|
any where network.protocol == "http"
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-syntax-escape-an-event-category]]
|
|
==== Escape an event category
|
|
|
|
Use enclosing double quotes (`"`) or three enclosing double quotes (`"""`) to
|
|
escape event categories that:
|
|
|
|
* Contain a special character, such as a hyphen (`-`) or dot (`.`)
|
|
* Contain a space
|
|
* Start with a numeral
|
|
|
|
[source,eql]
|
|
----
|
|
".my.event.category"
|
|
"my-event-category"
|
|
"my event category"
|
|
"6eventcategory"
|
|
|
|
""".my.event.category"""
|
|
"""my-event-category"""
|
|
"""my event category"""
|
|
"""6eventcategory"""
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-syntax-escape-a-field-name]]
|
|
==== Escape a field name
|
|
|
|
Use enclosing backticks (+++`+++) to escape field names that:
|
|
|
|
* Contain a hyphen (`-`)
|
|
* Contain a space
|
|
* Start with a numeral
|
|
|
|
[source,eql]
|
|
----
|
|
`my-field`
|
|
`my field`
|
|
`6myfield`
|
|
----
|
|
|
|
Use double backticks (+++``+++) to escape any backticks (+++`+++) in the field
|
|
name.
|
|
|
|
[source,eql]
|
|
----
|
|
my`field -> `my``field`
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-syntax-conditions]]
|
|
=== Conditions
|
|
|
|
A condition consists of one or more criteria an event must match.
|
|
You can specify and combine these criteria using the following operators. Most
|
|
EQL operators are case-sensitive by default.
|
|
|
|
[discrete]
|
|
[[eql-syntax-comparison-operators]]
|
|
==== Comparison operators
|
|
|
|
[source,eql]
|
|
----
|
|
< <= == : != >= >
|
|
----
|
|
|
|
`<` (less than)::
|
|
Returns `true` if the value to the left of the operator is less than the value
|
|
to the right. Otherwise returns `false`.
|
|
|
|
`<=` (less than or equal) ::
|
|
Returns `true` if the value to the left of the operator is less than or equal to
|
|
the value to the right. Otherwise returns `false`.
|
|
|
|
`==` (equal, case-sensitive)::
|
|
Returns `true` if the values to the left and right of the operator are equal.
|
|
Otherwise returns `false`. Wildcards are not supported.
|
|
|
|
`:` (equal, case-insensitive)::
|
|
Returns `true` if strings to the left and right of the operator are equal.
|
|
Otherwise returns `false`. Can only be used to compare strings. Supports
|
|
<<eql-syntax-wildcards,wildcards>> and <<eql-syntax-lookup-operators,list
|
|
lookups>>.
|
|
|
|
`!=` (not equal, case-sensitive)::
|
|
Returns `true` if the values to the left and right of the operator are not
|
|
equal. Otherwise returns `false`. Wildcards are not supported.
|
|
|
|
`>=` (greater than or equal) ::
|
|
Returns `true` if the value to the left of the operator is greater than or equal
|
|
to the value to the right. Otherwise returns `false`. When comparing strings,
|
|
the operator uses a case-sensitive lexicographic order.
|
|
|
|
`>` (greater than)::
|
|
Returns `true` if the value to the left of the operator is greater than the
|
|
value to the right. Otherwise returns `false`. When comparing strings,
|
|
the operator uses a case-sensitive lexicographic order.
|
|
|
|
NOTE: `=` is not supported as an equal operator. Use `==` or `:` instead.
|
|
|
|
[discrete]
|
|
[[eql-syntax-pattern-comparison-keywords]]
|
|
==== Pattern comparison keywords
|
|
|
|
[source,eql]
|
|
----
|
|
my_field like "VALUE*" // case-sensitive wildcard matching
|
|
my_field like~ "value*" // case-insensitive wildcard matching
|
|
|
|
my_field regex "VALUE[^Z].?" // case-sensitive regex matching
|
|
my_field regex~ "value[^z].?" // case-insensitive regex matching
|
|
----
|
|
|
|
`like` (case-sensitive)::
|
|
Returns `true` if the string to the left of the keyword matches a
|
|
<<eql-syntax-wildcards,wildcard pattern>> to the right. Supports
|
|
<<eql-syntax-lookup-operators,list lookups>>. Can only be used to compare
|
|
strings. For case-insensitive matching, use `like~`.
|
|
|
|
`regex` (case-sensitive)::
|
|
Returns `true` if the string to the left of the keyword matches a regular
|
|
expression to the right. For supported regular expression syntax, see
|
|
<<regexp-syntax>>. Supports <<eql-syntax-lookup-operators,list lookups>>. Can
|
|
only be used to compare strings. For case-insensitive matching, use `regex~`.
|
|
|
|
[discrete]
|
|
[[limitations-for-comparisons]]
|
|
===== Limitations for comparisons
|
|
|
|
You cannot chain comparisons. Instead, use a
|
|
<<eql-syntax-logical-operators,logical operator>> between comparisons. For
|
|
example, `foo < bar <= baz` is not supported. However, you can rewrite the
|
|
expression as `foo < bar and bar <= baz`, which is supported.
|
|
|
|
You also cannot compare a field to another field, even if the fields are changed
|
|
using a <<eql-functions,function>>.
|
|
|
|
*Example* +
|
|
The following EQL query compares the `process.parent_name` field
|
|
value to a static value, `foo`. This comparison is supported.
|
|
|
|
However, the query also compares the `process.parent.name` field value to the
|
|
`process.name` field. This comparison is not supported and will return an
|
|
error for the entire query.
|
|
|
|
[source,eql]
|
|
----
|
|
process where process.parent.name == "foo" and process.parent.name == process.name
|
|
----
|
|
|
|
Instead, you can rewrite the query to compare both the `process.parent.name`
|
|
and `process.name` fields to static values.
|
|
|
|
[source,eql]
|
|
----
|
|
process where process.parent.name == "foo" and process.name == "foo"
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-syntax-logical-operators]]
|
|
==== Logical operators
|
|
|
|
[source,eql]
|
|
----
|
|
and or not
|
|
----
|
|
|
|
`and`::
|
|
Returns `true` only if the condition to the left and right _both_ return `true`.
|
|
Otherwise returns `false`.
|
|
|
|
`or`::
|
|
Returns `true` if one of the conditions to the left or right `true`.
|
|
Otherwise returns `false`.
|
|
|
|
`not`::
|
|
Returns `true` if the condition to the right is `false`.
|
|
|
|
[discrete]
|
|
[[eql-syntax-lookup-operators]]
|
|
==== Lookup operators
|
|
|
|
[source,eql]
|
|
----
|
|
my_field in ("Value-1", "VALUE2", "VAL3") // case-sensitive
|
|
my_field in~ ("value-1", "value2", "val3") // case-insensitive
|
|
|
|
my_field not in ("Value-1", "VALUE2", "VAL3") // case-sensitive
|
|
my_field not in~ ("value-1", "value2", "val3") // case-insensitive
|
|
|
|
my_field : ("value-1", "value2", "val3") // case-insensitive
|
|
|
|
my_field like ("Value-*", "VALUE2", "VAL?") // case-sensitive
|
|
my_field like~ ("value-*", "value2", "val?") // case-insensitive
|
|
|
|
my_field regex ("[vV]alue-[0-9]", "VALUE[^2].?", "VAL3") // case-sensitive
|
|
my_field regex~ ("value-[0-9]", "value[^2].?", "val3") // case-sensitive
|
|
----
|
|
|
|
`in` (case-sensitive)::
|
|
Returns `true` if the value is contained in the provided list. For
|
|
case-insensitive matching, use `in~`.
|
|
|
|
`not in` (case-sensitive)::
|
|
Returns `true` if the value is not contained in the provided list. For
|
|
case-insensitive matching, use `not in~`.
|
|
|
|
`:` (case-insensitive)::
|
|
Returns `true` if the string is contained in the provided list. Can only be used
|
|
to compare strings.
|
|
|
|
`like` (case-sensitive)::
|
|
Returns `true` if the string matches a <<eql-syntax-wildcards,wildcard pattern>>
|
|
in the provided list. Can only be used to compare strings. For case-insensitive
|
|
matching, use `like~`.
|
|
|
|
`regex` (case-sensitive)::
|
|
Returns `true` if the string matches a regular expression pattern in the
|
|
provided list. For supported regular expression syntax, see <<regexp-syntax>>.
|
|
Can only be used to compare strings. For case-insensitive matching, use
|
|
`regex~`.
|
|
|
|
[discrete]
|
|
[[eql-syntax-math-operators]]
|
|
==== Math operators
|
|
|
|
[source,eql]
|
|
----
|
|
+ - * / %
|
|
----
|
|
|
|
`+` (add)::
|
|
Adds the values to the left and right of the operator.
|
|
|
|
`-` (subtract)::
|
|
Subtracts the value to the right of the operator from the value to the left.
|
|
|
|
`*` (multiply)::
|
|
Multiplies the values to the left and right of the operator.
|
|
|
|
`/` (divide)::
|
|
Divides the value to the left of the operator by the value to the right.
|
|
+
|
|
[[eql-divide-operator-float-rounding]]
|
|
[WARNING]
|
|
====
|
|
If both the dividend and divisor are integers, the divide (`\`) operation
|
|
_rounds down_ any returned floating point numbers to the nearest integer. To
|
|
avoid rounding, convert either the dividend or divisor to a float.
|
|
|
|
*Example* +
|
|
The `process.args_count` field is a <<number,`long`>> integer field containing a
|
|
count of process arguments.
|
|
|
|
A user might expect the following EQL query to only match events with a
|
|
`process.args_count` value of `4`.
|
|
|
|
[source,eql]
|
|
----
|
|
process where ( 4 / process.args_count ) == 1
|
|
----
|
|
|
|
However, the EQL query matches events with a `process.args_count` value of `3`
|
|
or `4`.
|
|
|
|
For events with a `process.args_count` value of `3`, the divide operation
|
|
returns a float of `1.333...`, which is rounded down to `1`.
|
|
|
|
To match only events with a `process.args_count` value of `4`, convert
|
|
either the dividend or divisor to a float.
|
|
|
|
The following EQL query changes the integer `4` to the equivalent float `4.0`.
|
|
|
|
[source,eql]
|
|
----
|
|
process where ( 4.0 / process.args_count ) == 1
|
|
----
|
|
====
|
|
|
|
`%` (modulo)::
|
|
Divides the value to the left of the operator by the value to the right. Returns only the remainder.
|
|
|
|
[discrete]
|
|
[[eql-syntax-match-any-condition]]
|
|
==== Match any condition
|
|
|
|
To match events solely on event category, use the `where true` condition.
|
|
|
|
For example, the following EQL query matches any `file` events:
|
|
|
|
[source,eql]
|
|
----
|
|
file where true
|
|
----
|
|
|
|
To match any event, you can combine the `any` keyword with the `where true`
|
|
condition:
|
|
|
|
[source,eql]
|
|
----
|
|
any where true
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-syntax-optional-fields]]
|
|
=== Optional fields
|
|
|
|
By default, an EQL query can only contain fields that exist in the dataset
|
|
you're searching. A field exists in a dataset if it has an
|
|
<<explicit-mapping,explicit>>, <<dynamic-mapping,dynamic>>, or
|
|
<<eql-use-runtime-fields,runtime>> mapping. If an EQL query contains a field
|
|
that doesn't exist, it returns an error.
|
|
|
|
If you aren't sure if a field exists in a dataset, use the `?` operator to mark
|
|
the field as optional. If an optional field doesn't exist, the query replaces it
|
|
with `null` instead of returning an error.
|
|
|
|
*Example* +
|
|
In the following query, the `user.id` field is optional.
|
|
|
|
[source,eql]
|
|
----
|
|
network where ?user.id != null
|
|
----
|
|
|
|
If the `user.id` field exists in the dataset you're searching, the query matches
|
|
any `network` event that contains a `user.id` value. If the `user.id` field
|
|
doesn't exist in the dataset, EQL interprets the query as:
|
|
|
|
[source,eql]
|
|
----
|
|
network where null != null
|
|
----
|
|
|
|
In this case, the query matches no events.
|
|
|
|
[discrete]
|
|
[[eql-syntax-check-field-exists]]
|
|
==== Check if a field exists
|
|
|
|
To match events containing any value for a field, compare the field to `null`
|
|
using the `!=` operator:
|
|
|
|
[source,eql]
|
|
----
|
|
?my_field != null
|
|
----
|
|
|
|
To match events that do not contain a field value, compare the field to `null`
|
|
using the `==` operator:
|
|
|
|
[source,eql]
|
|
----
|
|
?my_field == null
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-syntax-strings]]
|
|
=== Strings
|
|
|
|
Strings are enclosed in double quotes (`"`).
|
|
|
|
[source,eql]
|
|
----
|
|
"hello world"
|
|
----
|
|
|
|
Strings enclosed in single quotes (`'`) are not supported.
|
|
|
|
[discrete]
|
|
[[eql-syntax-escape-characters]]
|
|
==== Escape characters in a string
|
|
|
|
When used within a string, special characters, such as a carriage return or
|
|
double quote (`"`), must be escaped with a preceding backslash (`\`).
|
|
|
|
[source,eql]
|
|
----
|
|
"example \r of \" escaped \n characters"
|
|
----
|
|
|
|
[options="header"]
|
|
|====
|
|
| Escape sequence | Literal character
|
|
|`\n` | Newline (linefeed)
|
|
|`\r` | Carriage return
|
|
|`\t` | Tab
|
|
|`\\` | Backslash (`\`)
|
|
|`\"` | Double quote (`"`)
|
|
|====
|
|
|
|
You can escape Unicode characters using a hexadecimal `\u{XXXXXXXX}` escape
|
|
sequence. The hexadecimal value can be 2-8 characters and is case-insensitive.
|
|
Values shorter than 8 characters are zero-padded. You can use these escape
|
|
sequences to include non-printable or right-to-left (RTL) characters in your
|
|
strings. For example, you can escape a
|
|
{wikipedia}/Right-to-left_mark[right-to-left mark (RLM)] as `\u{200f}`,
|
|
`\u{200F}`, or `\u{0000200f}`.
|
|
|
|
IMPORTANT: The single quote (`'`) character is reserved for future use. You
|
|
cannot use an escaped single quote (`\'`) for literal strings. Use an escaped
|
|
double quote (`\"`) instead.
|
|
|
|
[discrete]
|
|
[[eql-syntax-raw-strings]]
|
|
==== Raw strings
|
|
|
|
Raw strings treat special characters, such as backslashes (`\`), as literal
|
|
characters. Raw strings are enclosed in three double quotes (`"""`).
|
|
|
|
[source,eql]
|
|
----
|
|
"""Raw string with a literal double quote " and blackslash \ included"""
|
|
----
|
|
|
|
A raw string cannot contain three consecutive double quotes (`"""`). Instead,
|
|
use a regular string with the `\"` escape sequence.
|
|
|
|
[source,eql]
|
|
----
|
|
"String containing \"\"\" three double quotes"
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-syntax-wildcards]]
|
|
==== Wildcards
|
|
|
|
For string comparisons using the `:` operator or `like` keyword, you can use the
|
|
`*` and `?` wildcards to match specific patterns. The `*` wildcard matches zero
|
|
or more characters:
|
|
|
|
[source,eql]
|
|
----
|
|
my_field : "doc*" // Matches "doc", "docs", or "document" but not "DOS"
|
|
my_field : "*doc" // Matches "adoc" or "asciidoc"
|
|
my_field : "d*c" // Matches "doc" or "disc"
|
|
|
|
my_field like "DOC*" // Matches "DOC", "DOCS", "DOCs", or "DOCUMENT" but not "DOS"
|
|
my_field like "D*C" // Matches "DOC", "DISC", or "DisC"
|
|
----
|
|
|
|
The `?` wildcard matches exactly one character:
|
|
|
|
[source,eql]
|
|
----
|
|
my_field : "doc?" // Matches "docs" but not "doc", "document", or "DOS"
|
|
my_field : "?doc" // Matches "adoc" but not "asciidoc"
|
|
my_field : "d?c" // Matches "doc" but not "disc"
|
|
|
|
my_field like "DOC?" // Matches "DOCS" or "DOCs" but not "DOC", "DOCUMENT", or "DOS"
|
|
my_field like "D?c" // Matches "DOC" but not "DISC"
|
|
----
|
|
|
|
The `:` operator and `like` keyword also support wildcards in
|
|
<<eql-syntax-lookup-operators,list lookups>>:
|
|
|
|
[source,eql]
|
|
----
|
|
my_field : ("doc*", "f*o", "ba?", "qux")
|
|
my_field like ("Doc*", "F*O", "BA?", "QUX")
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-sequences]]
|
|
=== Sequences
|
|
|
|
You can use EQL sequences to describe and match an ordered series of events.
|
|
Each item in a sequence is an event category and event condition,
|
|
surrounded by square brackets (`[ ]`). Events are listed in ascending
|
|
chronological order, with the most recent event listed last.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence
|
|
[ event_category_1 where condition_1 ]
|
|
[ event_category_2 where condition_2 ]
|
|
...
|
|
----
|
|
|
|
*Example* +
|
|
The following EQL sequence query matches this series of ordered events:
|
|
|
|
. Start with an event with:
|
|
+
|
|
--
|
|
* An event category of `file`
|
|
* A `file.extension` of `exe`
|
|
--
|
|
. Followed by an event with an event category of `process`
|
|
|
|
[source,eql]
|
|
----
|
|
sequence
|
|
[ file where file.extension == "exe" ]
|
|
[ process where true ]
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-with-maxspan-keywords]]
|
|
==== `with maxspan` statement
|
|
|
|
You can use `with maxspan` to constrain a sequence to a specified timespan. All
|
|
events in a matching sequence must occur within this duration, starting at the
|
|
first event's timestamp.
|
|
|
|
`maxspan` accepts <<time-units,time value>> arguments.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence with maxspan=30s
|
|
[ event_category_1 where condition_1 ] by field_baz
|
|
[ event_category_2 where condition_2 ] by field_bar
|
|
...
|
|
----
|
|
|
|
*Example* +
|
|
The following sequence query uses a `maxspan` value of `15m` (15 minutes).
|
|
Events in a matching sequence must occur within 15 minutes of the first event's
|
|
timestamp.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence with maxspan=15m
|
|
[ file where file.extension == "exe" ]
|
|
[ process where true ]
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-missing-events]]
|
|
==== Missing events
|
|
|
|
Use `!` to match missing events: events in a timespan-constrained sequence that
|
|
do not meet a given condition.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence with maxspan=1h
|
|
[ event_category_1 where condition_1 ]
|
|
![ event_category_2 where condition_2 ]
|
|
[ event_category_3 where condition_3 ]
|
|
...
|
|
----
|
|
|
|
Missing event clauses can be used at the beginning, at the end, and/or in the
|
|
middle of a sequence, in any combination with positive event clauses. A sequence
|
|
can have multiple missing event clauses, but needs to have at least one positive
|
|
clause. <<eql-with-maxspan-keywords,`with maxspan`>> is mandatory when missing
|
|
event clauses are present.
|
|
|
|
|
|
*Example* +
|
|
The following sequence query finds logon events that are not followed within 5
|
|
seconds by a logoff event.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence by host.name, user.name with maxspan=5s
|
|
[ authentication where event.code : "4624" ]
|
|
![ authentication where event.code : "4647" ]
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-by-keyword]]
|
|
==== `by` keyword
|
|
|
|
Use the `by` keyword in a sequence query to only match events that share the
|
|
same values, even if those values are in different fields. These shared values
|
|
are called join keys. If a join key should be in the same field across all
|
|
events, use `sequence by`.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence by field_foo
|
|
[ event_category_1 where condition_1 ] by field_baz
|
|
[ event_category_2 where condition_2 ] by field_bar
|
|
...
|
|
----
|
|
|
|
*Example* +
|
|
The following sequence query uses the `by` keyword to constrain matching events
|
|
to:
|
|
|
|
* Events with the same `user.name` value
|
|
* `file` events with a `file.path` value equal to the following `process`
|
|
event's `process.executable` value.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence
|
|
[ file where file.extension == "exe" ] by user.name, file.path
|
|
[ process where true ] by user.name, process.executable
|
|
----
|
|
|
|
Because the `user.name` field is shared across all events in the sequence, it
|
|
can be included using `sequence by`. The following sequence is equivalent to the
|
|
prior one.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence by user.name
|
|
[ file where file.extension == "exe" ] by file.path
|
|
[ process where true ] by process.executable
|
|
----
|
|
|
|
You can combine `sequence by` and `with maxspan` to constrain a sequence by both
|
|
field values and a timespan.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence by field_foo with maxspan=30s
|
|
[ event_category_1 where condition_1 ]
|
|
[ event_category_2 where condition_2 ]
|
|
...
|
|
----
|
|
|
|
*Example* +
|
|
The following sequence query uses `sequence by` and `with maxspan` to only match
|
|
a sequence of events that:
|
|
|
|
* Share the same `user.name` field values
|
|
* Occur within `15m` (15 minutes) of the first matching event
|
|
|
|
[source,eql]
|
|
----
|
|
sequence by user.name with maxspan=15m
|
|
[ file where file.extension == "exe" ]
|
|
[ process where true ]
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-syntax-optional-by-fields]]
|
|
==== Optional `by` fields
|
|
|
|
By default, a join key must be a non-`null` field value. To allow `null` join
|
|
keys, use the `?` operator to mark the `by` field as
|
|
<<eql-syntax-optional-fields,optional>>. This is also helpful if you aren't sure
|
|
the dataset you're searching contains the `by` field.
|
|
|
|
*Example* +
|
|
The following sequence query uses `sequence by` to constrain matching events
|
|
to:
|
|
|
|
* Events with the same `process.pid` value, excluding `null` values. If the
|
|
`process.pid` field doesn't exist in the dataset you're searching, the query
|
|
returns an error.
|
|
|
|
* Events with the same `process.entity_id` value, including `null` values. If
|
|
an event doesn't contain the `process.entity_id` field, its
|
|
`process.entity_id` value is considered `null`. This applies even if the
|
|
`process.pid` field doesn't exist in the dataset you're searching.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence by process.pid, ?process.entity_id
|
|
[process where process.name == "regsvr32.exe"]
|
|
[network where true]
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-until-keyword]]
|
|
==== `until` keyword
|
|
|
|
You can use the `until` keyword to specify an expiration event for a sequence.
|
|
If this expiration event occurs _between_ matching events in a sequence, the
|
|
sequence expires and is not considered a match. If the expiration event occurs
|
|
_after_ matching events in a sequence, the sequence is still considered a
|
|
match. The expiration event is not included in the results.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence
|
|
[ event_category_1 where condition_1 ]
|
|
[ event_category_2 where condition_2 ]
|
|
...
|
|
until [ event_category_3 where condition_3 ]
|
|
----
|
|
|
|
*Example* +
|
|
A dataset contains the following event sequences, grouped by shared IDs:
|
|
|
|
[source,txt]
|
|
----
|
|
A, B
|
|
A, B, C
|
|
A, C, B
|
|
----
|
|
|
|
The following EQL query searches the dataset for sequences containing
|
|
event `A` followed by event `B`. Event `C` is used as an expiration event.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence by ID
|
|
A
|
|
B
|
|
until C
|
|
----
|
|
|
|
The query matches sequences `A, B` and `A, B, C` but not `A, C, B`.
|
|
|
|
[TIP]
|
|
====
|
|
The `until` keyword can be useful when searching for process sequences in
|
|
Windows event logs.
|
|
|
|
In Windows, a process ID (PID) is unique only while a process is running. After
|
|
a process terminates, its PID can be reused.
|
|
|
|
You can search for a sequence of events with the same PID value using the `by`
|
|
and `sequence by` keywords.
|
|
|
|
*Example* +
|
|
The following EQL query uses the `sequence by` keyword to match a
|
|
sequence of events that share the same `process.pid` value.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence by process.pid
|
|
[ process where event.type == "start" and process.name == "cmd.exe" ]
|
|
[ process where file.extension == "exe" ]
|
|
----
|
|
|
|
However, due to PID reuse, this can result in a matching sequence that
|
|
contains events across unrelated processes. To prevent false positives, you can
|
|
use the `until` keyword to end matching sequences before a process termination
|
|
event.
|
|
|
|
The following EQL query uses the `until` keyword to end sequences before
|
|
`process` events with an `event.type` of `stop`. These events indicate a process
|
|
has been terminated.
|
|
|
|
[source,eql]
|
|
----
|
|
sequence by process.pid
|
|
[ process where event.type == "start" and process.name == "cmd.exe" ]
|
|
[ process where file.extension == "exe" ]
|
|
until [ process where event.type == "stop" ]
|
|
----
|
|
====
|
|
|
|
[discrete]
|
|
[[eql-with-runs-statement]]
|
|
==== `with runs` statement
|
|
|
|
Use a `with runs` statement to run the same event criteria successively within a
|
|
sequence query. For example:
|
|
|
|
[source,eql]
|
|
----
|
|
sequence
|
|
[ process where event.type == "creation" ]
|
|
[ library where process.name == "regsvr32.exe" ] with runs=3
|
|
[ registry where true ]
|
|
----
|
|
|
|
is equivalent to:
|
|
|
|
[source,eql]
|
|
----
|
|
sequence
|
|
[ process where event.type == "creation" ]
|
|
[ library where process.name == "regsvr32.exe" ]
|
|
[ library where process.name == "regsvr32.exe" ]
|
|
[ library where process.name == "regsvr32.exe" ]
|
|
[ registry where true ]
|
|
----
|
|
|
|
The `runs` value must be between `1` and `100` (inclusive).
|
|
|
|
You can use a `with runs` statement with the <<eql-by-keyword,`by` keyword>>.
|
|
For example:
|
|
|
|
[source,eql]
|
|
----
|
|
sequence
|
|
[ process where event.type == "creation" ] by process.executable
|
|
[ library where process.name == "regsvr32.exe" ] by dll.path with runs=3
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-samples]]
|
|
=== Samples
|
|
|
|
You can use EQL samples to describe and match a chronologically unordered series
|
|
of events. All events in a sample share the same value for one or more fields
|
|
that are specified using the <<eql-by-keyword,`by` keyword>> (join keys). Each
|
|
item in a sample is an event category and event condition, surrounded by square
|
|
brackets (`[ ]`). Events are listed in the order of the filters they match.
|
|
|
|
[source,eql]
|
|
----
|
|
sample by join_key
|
|
[ event_category_1 where condition_1 ]
|
|
[ event_category_2 where condition_2 ]
|
|
...
|
|
----
|
|
|
|
*Example* +
|
|
The following EQL sample query returns up to 10 samples with unique values for
|
|
`host`. Each sample consists of two events:
|
|
|
|
. Start with an event with:
|
|
+
|
|
--
|
|
* An event category of `file`
|
|
* A `file.extension` of `exe`
|
|
--
|
|
. Followed by an event with an event category of `process`
|
|
|
|
[source,eql]
|
|
----
|
|
sample by host
|
|
[ file where file.extension == "exe" ]
|
|
[ process where true ]
|
|
----
|
|
|
|
Sample queries do not take into account the chronological ordering of events.
|
|
The `with maxspan` and `with runs` statements as well as the `until` keyword are
|
|
not supported.
|
|
|
|
[discrete]
|
|
[[eql-functions]]
|
|
=== Functions
|
|
|
|
You can use EQL functions to convert data types, perform math, manipulate
|
|
strings, and more. For a list of supported functions, see <<eql-function-ref>>.
|
|
|
|
[discrete]
|
|
[[eql-case-insensitive-functions]]
|
|
==== Case-insensitive functions
|
|
|
|
Most EQL functions are case-sensitive by default. To make a function
|
|
case-insensitive, use the `~` operator after the function name:
|
|
|
|
[source,eql]
|
|
----
|
|
stringContains(process.name,".exe") // Matches ".exe" but not ".EXE" or ".Exe"
|
|
stringContains~(process.name,".exe") // Matches ".exe", ".EXE", or ".Exe"
|
|
----
|
|
|
|
[discrete]
|
|
[[eql-how-functions-impact-search-performance]]
|
|
==== How functions impact search performance
|
|
|
|
Using functions in EQL queries can result in slower search speeds. If you
|
|
often use functions to transform indexed data, you can speed up search by making
|
|
these changes during indexing instead. However, that often means slower index
|
|
speeds.
|
|
|
|
*Example* +
|
|
An index contains the `file.path` field. `file.path` contains the full path to a
|
|
file, including the file extension.
|
|
|
|
When running EQL searches, users often use the `endsWith` function with the
|
|
`file.path` field to match file extensions:
|
|
|
|
[source,eql]
|
|
----
|
|
file where endsWith(file.path,".exe") or endsWith(file.path,".dll")
|
|
----
|
|
|
|
While this works, it can be repetitive to write and can slow search speeds. To
|
|
speed up search, you can do the following instead:
|
|
|
|
. <<indices-put-mapping,Add a new field>>, `file.extension`, to the index. The
|
|
`file.extension` field will contain only the file extension from the
|
|
`file.path` field.
|
|
. Use an <<ingest,ingest pipeline>> containing the <<grok-processor,`grok`>>
|
|
processor or another preprocessor tool to extract the file extension from the
|
|
`file.path` field before indexing.
|
|
. Index the extracted file extension to the `file.extension` field.
|
|
|
|
These changes may slow indexing but allow for faster searches. Users
|
|
can use the `file.extension` field instead of multiple `endsWith` function
|
|
calls:
|
|
|
|
[source,eql]
|
|
----
|
|
file where file.extension in ("exe", "dll")
|
|
----
|
|
|
|
We recommend testing and benchmarking any indexing changes before deploying them
|
|
in production. See <<tune-for-indexing-speed>> and <<tune-for-search-speed>>.
|
|
|
|
[discrete]
|
|
[[eql-pipes]]
|
|
=== Pipes
|
|
|
|
EQL pipes filter, aggregate, and post-process events returned by
|
|
an EQL query. You can use pipes to narrow down EQL query results or make them
|
|
more specific.
|
|
|
|
Pipes are delimited using the pipe (`|`) character.
|
|
|
|
[source,eql]
|
|
----
|
|
event_category where condition | pipe
|
|
----
|
|
|
|
*Example* +
|
|
The following EQL query uses the `tail` pipe to return only the 10 most recent
|
|
events matching the query.
|
|
|
|
[source,eql]
|
|
----
|
|
authentication where agent.id == 4624
|
|
| tail 10
|
|
----
|
|
|
|
You can pass the output of a pipe to another pipe. This lets you use multiple
|
|
pipes with a single query.
|
|
|
|
For a list of supported pipes, see <<eql-pipe-ref>>.
|
|
|
|
[discrete]
|
|
[[eql-syntax-limitations]]
|
|
=== Limitations
|
|
|
|
EQL has the following limitations.
|
|
|
|
[discrete]
|
|
[[eql-uses-fields-parameter]]
|
|
==== EQL uses the `fields` parameter
|
|
|
|
EQL retrieves field values using the search API's <<search-fields-param,`fields`
|
|
parameter>>. Any limitations on the `fields` parameter also apply to EQL
|
|
queries. For example, if `_source` is disabled for any returned fields or at
|
|
index level, the values cannot be retrieved.
|
|
|
|
[discrete]
|
|
[[eql-compare-fields]]
|
|
==== Comparing fields
|
|
|
|
You cannot use EQL comparison operators to compare a field to
|
|
another field. This applies even if the fields are changed using a
|
|
<<eql-functions,function>>.
|
|
|
|
[discrete]
|
|
[[eql-text-fields]]
|
|
==== Text fields are not supported
|
|
|
|
EQL searches do not support <<text,`text`>> fields. To a search a `text` field,
|
|
use the EQL search API's <<eql-search-filter-query-dsl,Query DSL `filter`>>
|
|
parameter.
|
|
|
|
[discrete]
|
|
[[eql-nested-fields]]
|
|
==== EQL search on nested fields
|
|
|
|
You cannot use EQL to search the values of a <<nested,`nested`>> field or the
|
|
sub-fields of a `nested` field. However, data streams and indices containing
|
|
`nested` field mappings are otherwise supported.
|
|
|
|
[discrete]
|
|
[[eql-unsupported-syntax]]
|
|
==== Differences from Endgame EQL syntax
|
|
|
|
{es} EQL differs from the {eql-ref}/index.html[Elastic Endgame EQL syntax] as
|
|
follows:
|
|
|
|
* In {es} EQL, most operators are case-sensitive. For example,
|
|
`process_name == "cmd.exe"` is not equivalent to
|
|
`process_name == "Cmd.exe"`.
|
|
|
|
* In {es} EQL, functions are case-sensitive. To make a function
|
|
case-insensitive, use `~`, such as `endsWith~(process_name, ".exe")`.
|
|
|
|
* For case-insensitive equality comparisons, use the `:` operator. Both `*` and
|
|
`?` are recognized wildcard characters.
|
|
|
|
* The `==` and `!=` operators do not expand wildcard characters. For example,
|
|
`process_name == "cmd*.exe"` interprets `*` as a literal asterisk, not a
|
|
wildcard.
|
|
|
|
* For wildcard matching, use the `like` keyword when case-sensitive and
|
|
`like~` when case-insensitive. The `:` operator is equivalent to `like~`.
|
|
|
|
* For regular expression matching, use `regex` or `regex~`.
|
|
|
|
* `=` cannot be substituted for the `==` operator.
|
|
|
|
* Strings enclosed in single quotes (`'`) are not supported. Enclose strings in
|
|
double quotes (`"`) instead.
|
|
|
|
* `?"` and `?'` do not indicate raw strings. Enclose raw strings in
|
|
three double quotes (`"""`) instead.
|
|
|
|
* {es} EQL does not support:
|
|
|
|
** Array functions:
|
|
*** {eql-ref}/functions.html#arrayContains[`arrayContains`]
|
|
*** {eql-ref}/functions.html#arrayCount[`arrayCount`]
|
|
*** {eql-ref}/functions.html#arraySearch[`arraySearch`]
|
|
|
|
** The {eql-ref}//functions.html#match[`match`] function
|
|
|
|
** {eql-ref}/joins.html[Joins]
|
|
|
|
** {eql-ref}/basic-syntax.html#event-relationships[Lineage-related keywords]:
|
|
*** `child of`
|
|
*** `descendant of`
|
|
*** `event of`
|
|
|
|
** The following {eql-ref}/pipes.html[pipes]:
|
|
*** {eql-ref}/pipes.html#count[`count`]
|
|
*** {eql-ref}/pipes.html#filter[`filter`]
|
|
*** {eql-ref}/pipes.html#sort[`sort`]
|
|
*** {eql-ref}/pipes.html#unique[`unique`]
|
|
*** {eql-ref}/pipes.html#unique-count[`unique_count`]
|
|
|
|
[discrete]
|
|
[[eql-how-sequence-queries-handle-matches]]
|
|
==== How sequence queries handle matches
|
|
|
|
<<eql-sequences,Sequence queries>> don't find all potential matches for a
|
|
sequence. This approach would be too slow and costly for large event data sets.
|
|
Instead, a sequence query handles pending sequence matches as a
|
|
{wikipedia}/Finite-state_machine[state machine]:
|
|
|
|
* Each event item in the sequence query is a state in the machine.
|
|
* Only one pending sequence can be in each state at a time.
|
|
* If two pending sequences are in the same state at the same time, the most
|
|
recent sequence overwrites the older one.
|
|
* If the query includes <<eql-by-keyword,`by` fields>>, the query uses a
|
|
separate state machine for each unique `by` field value.
|
|
|
|
.*Example*
|
|
[%collapsible]
|
|
====
|
|
A data set contains the following `process` events in ascending chronological
|
|
order:
|
|
|
|
[source,js]
|
|
----
|
|
{ "index" : { "_id": "1" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "attrib" }, ...}
|
|
{ "index" : { "_id": "2" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "attrib" }, ...}
|
|
{ "index" : { "_id": "3" } }
|
|
{ "user": { "name": "elkbee" }, "process": { "name": "bash" }, ...}
|
|
{ "index" : { "_id": "4" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "bash" }, ...}
|
|
{ "index" : { "_id": "5" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "bash" }, ...}
|
|
{ "index" : { "_id": "6" } }
|
|
{ "user": { "name": "elkbee" }, "process": { "name": "attrib" }, ...}
|
|
{ "index" : { "_id": "7" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "attrib" }, ...}
|
|
{ "index" : { "_id": "8" } }
|
|
{ "user": { "name": "elkbee" }, "process": { "name": "bash" }, ...}
|
|
{ "index" : { "_id": "9" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "cat" }, ...}
|
|
{ "index" : { "_id": "10" } }
|
|
{ "user": { "name": "elkbee" }, "process": { "name": "cat" }, ...}
|
|
{ "index" : { "_id": "11" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "cat" }, ...}
|
|
----
|
|
// NOTCONSOLE
|
|
|
|
An EQL sequence query searches the data set:
|
|
|
|
[source,eql]
|
|
----
|
|
sequence by user.name
|
|
[process where process.name == "attrib"]
|
|
[process where process.name == "bash"]
|
|
[process where process.name == "cat"]
|
|
----
|
|
|
|
The query's event items correspond to the following states:
|
|
|
|
* State A: `[process where process.name == "attrib"]`
|
|
* State B: `[process where process.name == "bash"]`
|
|
* Complete: `[process where process.name == "cat"]`
|
|
|
|
image::images/eql/sequence-state-machine.svg[align="center"]
|
|
|
|
To find matching sequences, the query uses separate state machines for each
|
|
unique `user.name` value. Based on the data set, you can expect two state
|
|
machines: one for the `root` user and one for `elkbee`.
|
|
|
|
image::images/eql/separate-state-machines.svg[align="center"]
|
|
|
|
Pending sequence matches move through each machine's states as follows:
|
|
|
|
[source,txt]
|
|
----
|
|
{ "index" : { "_id": "1" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "attrib" }, ...}
|
|
// Creates sequence [1] in state A for the "root" user.
|
|
//
|
|
// +------------------------"root"------------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | [1] | | | | | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
|
|
{ "index" : { "_id": "2" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "attrib" }, ...}
|
|
// Creates sequence [2] in state A for "root", overwriting sequence [1].
|
|
//
|
|
// +------------------------"root"------------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | [2] | | | | | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
|
|
{ "index" : { "_id": "3" } }
|
|
{ "user": { "name": "elkbee" }, "process": { "name": "bash" }, ...}
|
|
// Nothing happens. The "elkbee" user has no pending sequence to move
|
|
// from state A to state B.
|
|
//
|
|
// +-----------------------"elkbee"-----------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | | | | | | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
|
|
{ "index" : { "_id": "4" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "bash" }, ...}
|
|
// Sequence [2] moves out of state A for "root".
|
|
// State B for "root" now contains [2, 4].
|
|
// State A for "root" is empty.
|
|
//
|
|
// +------------------------"root"------------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ --> +-----------+ +------------+ |
|
|
// | | | | [2, 4] | | | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
|
|
{ "index" : { "_id": "5" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "bash" }, ...}
|
|
// Nothing happens. State A is empty for "root".
|
|
//
|
|
// +------------------------"root"------------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | | | [2, 4] | | | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
|
|
{ "index" : { "_id": "6" } }
|
|
{ "user": { "name": "elkbee" }, "process": { "name": "attrib" }, ...}
|
|
// Creates sequence [6] in state A for "elkbee".
|
|
//
|
|
// +-----------------------"elkbee"-----------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | [6] | | | | | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
|
|
{ "index" : { "_id": "7" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "attrib" }, ...}
|
|
// Creates sequence [7] in state A for "root".
|
|
// Sequence [2, 4] remains in state B for "root".
|
|
//
|
|
// +------------------------"root"------------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | [7] | | [2, 4] | | | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
|
|
{ "index" : { "_id": "8" } }
|
|
{ "user": { "name": "elkbee" }, "process": { "name": "bash" }, ...}
|
|
// Sequence [6, 8] moves to state B for "elkbee".
|
|
// State A for "elkbee" is now empty.
|
|
//
|
|
// +-----------------------"elkbee"-----------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ --> +-----------+ +------------+ |
|
|
// | | | | [6, 8] | | | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
|
|
{ "index" : { "_id": "9" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "cat" }, ...}
|
|
// Sequence [2, 4, 9] is complete for "root".
|
|
// State B for "root" is now empty.
|
|
// Sequence [7] remains in state A.
|
|
//
|
|
// +------------------------"root"------------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ +-----------+ --> +------------+ |
|
|
// | | [7] | | | | [2, 4, 9] |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
|
|
{ "index" : { "_id": "10" } }
|
|
{ "user": { "name": "elkbee" }, "process": { "name": "cat" }, ...}
|
|
// Sequence [6, 8, 10] is complete for "elkbee".
|
|
// State A and B for "elkbee" are now empty.
|
|
//
|
|
// +-----------------------"elkbee"-----------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ +-----------+ --> +------------+ |
|
|
// | | | | | | [6, 8, 10] |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
|
|
{ "index" : { "_id": "11" } }
|
|
{ "user": { "name": "root" }, "process": { "name": "cat" }, ...}
|
|
// Nothing happens.
|
|
// The machines for "root" and "elkbee" remain the same.
|
|
//
|
|
// +------------------------"root"------------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | [7] | | | | [2, 4, 9] |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
//
|
|
// +-----------------------"elkbee"-----------------------+
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | State A | | State B | | Complete | |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// | | | | | | [6, 8, 10] |
|
|
// | +-----------+ +-----------+ +------------+ |
|
|
// +------------------------------------------------------+
|
|
----
|
|
====
|