Ensure EQL works with RCS (#94265)

The PR allows EQL read action to work with new RCS. It also updates the common CCS test suite to include existing EQL yaml tests. Relates: https://github.com/elastic/elasticsearch/pull/93720
2025-06-28 09:28:55 -04:00 · 2023-03-09 08:51:13 +11:00 · 2023-03-09 08:51:13 +11:00 · a96ffea12e
commit a96ffea12e
parent 7c342f1241
6 changed files with 640 additions and 2 deletions
--- a/qa/ccs-common-rest/build.gradle
+++ b/qa/ccs-common-rest/build.gradle
@ -11,7 +11,7 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 restResources {
  restApi {
    include '_common', 'bulk', 'count', 'cluster', 'field_caps', 'knn_search', 'index', 'indices', 'msearch',
-      'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', 'clear_scroll', 'search_mvt'
+      'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', 'clear_scroll', 'search_mvt', 'eql'
  }
  restTests {
    includeCore 'field_caps', 'msearch', 'search', 'suggest', 'scroll', "indices.resolve_index"
@ -27,6 +27,7 @@ dependencies {
  clusterModules project(':x-pack:plugin:analytics')
  clusterModules project(':x-pack:plugin:vector-tile')
  clusterModules project(':modules:legacy-geo')
+  clusterModules project(':x-pack:plugin:eql')
 }

 tasks.named("yamlRestTest") {
--- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
+++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
@ -75,6 +75,7 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
        .module("mapper-extras")
        .module("vector-tile")
        .module("x-pack-analytics")
+        .module("x-pack-eql")
        .setting("xpack.security.enabled", "false")
        // geohex_grid requires gold license
        .setting("xpack.license.self_generated.type", "trial")
@ -111,6 +112,10 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
        "async_search.get",
        "async_search.status",
        "async_search.delete",
+        "eql.search",
+        "eql.get",
+        "eql.get_status",
+        "eql.delete",
        "open_point_in_time",
        "close_point_in_time"
    );
@ -350,6 +355,9 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
                || apiName.equals("async_search.get")
                || apiName.equals("async_search.delete")
                || apiName.equals("async_search.status")
+                || apiName.equals("eql.get")
+                || apiName.equals("eql.get_status")
+                || apiName.equals("eql.delete")
                || apiName.equals("close_point_in_time")) {
                return false;
            }
--- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java
+++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java
@ -72,6 +72,7 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
        .module("analysis-common")
        .module("vector-tile")
        .module("x-pack-analytics")
+        .module("x-pack-eql")
        .setting("xpack.license.self_generated.type", "trial")
        .setting("xpack.security.enabled", "true")
        .setting("xpack.security.transport.ssl.enabled", "false")
--- a/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml
+++ b/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml
@ -0,0 +1,435 @@
+---
+setup:
+  - do:
+      indices.create:
+          index:  eql_test
+          body:
+            mappings:
+              properties:
+                some_keyword:
+                  type: keyword
+                  ignore_above: 5 # see https://github.com/elastic/elasticsearch/issues/77152
+              runtime:
+                day_of_week:
+                  type: keyword
+                  script:
+                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: eql_test
+              _id:    "1"
+          - event:
+              - category: process
+            "@timestamp": 2020-02-03T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: false
+            some_keyword: longer than normal
+          - index:
+              _index: eql_test
+              _id:    "2"
+          - event:
+              - category: process
+            "@timestamp": 2020-02-04T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: true
+          - index:
+              _index: eql_test
+              _id:    "3"
+          - event:
+              - category: process
+            "@timestamp": 2020-02-05T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: true
+          - index:
+              _index: eql_test
+              _id:    "4"
+          - event:
+              - category: network
+            "@timestamp": 2020-02-06T12:34:56Z
+            user: ADMIN
+            id: 123
+            valid: true
+          - index:
+              _index: eql_test
+              _id:    "5"
+          - event:
+              - category: network
+            "@timestamp": 2020-02-07T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: true
+          - index:
+              _index: eql_test
+              _id:    "6"
+          - event:
+              - category: network
+            "@timestamp": 2020-02-08T12:34:56Z
+            user: ADMIN
+            id: 123
+            valid: true
+          - index:
+              _index: eql_test
+              _id:    "7"
+          - event:
+              - category: network
+            "@timestamp": 2020-02-09T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: true
+
+---
+# Testing round-trip and the basic shape of the response
+"Execute some EQL.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 3}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._source.user: "SYSTEM"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.2._id: "3"}
+
+---
+"Execute EQL events query with fields filtering":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 3}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._source.user: "SYSTEM"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0.fields.@timestamp: ["1580733296000"]}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.valid: [false]}
+  - match: {hits.events.0.fields.day_of_week: ["Monday"]}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.1.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.events.1.fields.id: [123]}
+  - match: {hits.events.1.fields.valid: [true]}
+  - match: {hits.events.1.fields.day_of_week: ["Tuesday"]}
+  - match: {hits.events.2._id: "3"}
+  - match: {hits.events.2.fields.@timestamp: ["1580906096000"]}
+  - match: {hits.events.2.fields.id: [123]}
+  - match: {hits.events.2.fields.valid: [true]}
+  - match: {hits.events.2.fields.day_of_week: ["Wednesday"]}
+
+---
+"Execute EQL events query with wildcard (*) fields filtering.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: [{"field":"*"}]
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 3}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0.fields:{"valid":[false],"@timestamp":["2020-02-03T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Monday"]}}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.1.fields:{"valid":[true],"@timestamp":["2020-02-04T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Tuesday"]}}
+  - match: {hits.events.2._id: "3"}
+  - match: {hits.events.2.fields:{"valid":[true],"@timestamp":["2020-02-05T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Wednesday"]}}
+
+---
+"Execute EQL events query with filter_path":
+  - do:
+      eql.search:
+        index: eql_test
+        filter_path: "hits.events._source.event.category,hits.events.fields.user,hits.events.fields.id"
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","user"]
+
+  - match: {hits.events.0._source.event.0.category: "process"}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.user: ["SYSTEM"]}
+  - match: {hits.events.1._source.event.0.category: "process"}
+  - match: {hits.events.1.fields.id: [123]}
+  - match: {hits.events.1.fields.user: ["SYSTEM"]}
+  - match: {hits.events.2._source.event.0.category: "process"}
+  - match: {hits.events.2.fields.id: [123]}
+  - match: {hits.events.2.fields.user: ["SYSTEM"]}
+
+---
+"Execute EQL sequence with string key.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.1._id: "3"}
+
+---
+"Execute EQL sequence with numeric key.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by id [process where user == "SYSTEM"] [process where true]'
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: 123}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.1.join_keys.0: 123}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.1._id: "3"}
+
+---
+"Execute EQL sequence by default ignores unavailable index.":
+  - do:
+      eql.search:
+        index: eql_test,non_existing
+        body:
+          query: 'sequence by valid [process where user == "SYSTEM"] [process where true]'
+  - match: {timed_out: false}
+  - match: {hits.total.value: 1}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: true}
+  - match: {hits.sequences.0.events.0._id: "2"}
+  - match: {hits.sequences.0.events.1._id: "3"}
+
+---
+"Execute EQL sequence by default ignores unavailable index pattern.":
+  - do:
+      eql.search:
+        index: eql_test,non_existing*
+        body:
+          query: 'sequence by valid [process where user == "SYSTEM"] [process where true]'
+  - match: {timed_out: false}
+  - match: {hits.total.value: 1}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: true}
+  - match: {hits.sequences.0.events.0._id: "2"}
+  - match: {hits.sequences.0.events.1._id: "3"}
+
+---
+"Execute EQL sequence with boolean key.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by valid [process where user == "SYSTEM"] [process where true]'
+  - match: {timed_out: false}
+  - match: {hits.total.value: 1}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: true}
+  - match: {hits.sequences.0.events.0._id: "2"}
+  - match: {hits.sequences.0.events.1._id: "3"}
+
+---
+"Execute EQL sequence with fields filtering.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.0.fields.@timestamp: ["1580733296000"]}
+  - match: {hits.sequences.0.events.0.fields.id: [123]}
+  - match: {hits.sequences.0.events.0.fields.valid: [false]}
+  - match: {hits.sequences.0.events.0.fields.day_of_week: ["Monday"]}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.0.events.1.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.sequences.0.events.1.fields.id: [123]}
+  - match: {hits.sequences.0.events.1.fields.valid: [true]}
+  - match: {hits.sequences.0.events.1.fields.day_of_week: ["Tuesday"]}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.0.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.sequences.1.events.0.fields.id: [123]}
+  - match: {hits.sequences.1.events.0.fields.valid: [true]}
+  - match: {hits.sequences.1.events.0.fields.day_of_week: ["Tuesday"]}
+  - match: {hits.sequences.1.events.1._id: "3"}
+  - match: {hits.sequences.1.events.1.fields.@timestamp: ["1580906096000"]}
+  - match: {hits.sequences.1.events.1.fields.id: [123]}
+  - match: {hits.sequences.1.events.1.fields.valid: [true]}
+  - match: {hits.sequences.1.events.1.fields.day_of_week: ["Wednesday"]}
+
+---
+"Execute EQL sequence with wildcard (*) fields filtering.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
+          fields: [{"field":"*"}]
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.0.fields:{"valid":[false],"@timestamp":["2020-02-03T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Monday"]}}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.0.events.1.fields:{"valid":[true],"@timestamp":["2020-02-04T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Tuesday"]}}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.0.fields:{"valid":[true],"@timestamp":["2020-02-04T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Tuesday"]}}
+  - match: {hits.sequences.1.events.1._id: "3"}
+  - match: {hits.sequences.1.events.1.fields:{"valid":[true],"@timestamp":["2020-02-05T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Wednesday"]}}
+
+---
+"Execute EQL sequence with custom format for timestamp field.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
+          fields: [{"field":"@timestamp","format":"yyyy"},{"field":"day_of_week"}]
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.0.fields:{"@timestamp":["2020"],"day_of_week":["Monday"]}}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.0.events.1.fields:{"@timestamp":["2020"],"day_of_week":["Tuesday"]}}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.0.fields:{"@timestamp":["2020"],"day_of_week":["Tuesday"]}}
+  - match: {hits.sequences.1.events.1._id: "3"}
+  - match: {hits.sequences.1.events.1.fields:{"@timestamp":["2020"],"day_of_week":["Wednesday"]}}
+
+---
+"Execute EQL sequence with filter_path":
+  - do:
+      eql.search:
+        index: eql_test
+        filter_path: "hits.sequences.join_keys,hits.sequences.events.fields.valid"
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
+          fields: ["id","valid"]
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0.fields.valid: [false]}
+  - match: {hits.sequences.0.events.1.fields.valid: [true]}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0.fields.valid: [true]}
+  - match: {hits.sequences.1.events.1.fields.valid: [true]}
+
+---
+"Execute some EQL in async mode.":
+  - do:
+      eql.search:
+        index: eql_test
+        wait_for_completion_timeout: "0ms"
+        keep_on_completion: true
+        body:
+          query: 'process where user == "SYSTEM"'
+
+  - is_true: id
+  - set: {id: id}
+  - gte: {took: 0}
+
+  - do:
+      eql.get:
+        id: $id
+        wait_for_completion_timeout: "10s"
+
+  - match: {is_running: false}
+  - match: {is_partial: false}
+  - match: {timed_out: false}
+  - match: {hits.total.value: 3}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._source.user: "SYSTEM"}
+
+  - do:
+      eql.delete:
+        id: $id
+  - match: {acknowledged: true}
+
+  - do:
+      catch: missing
+      eql.delete:
+        id: $id
+
+---
+"EQL status API":
+
+  - do:
+      eql.search:
+        index: eql_test
+        keep_on_completion: true
+        wait_for_completion_timeout: "0ms"
+        body:
+          query: 'process where user == "SYSTEM"'
+  - is_true: id
+  - set: { id: id }
+
+  # status for eql search that may be completed or not
+  - do:
+      eql.get_status:
+        id: $id
+  - match: { id: $id }
+  - set: { is_running: run_status }
+  - match: { is_partial: $run_status }
+  - is_true: expiration_time_in_millis
+
+  # wait for the certain completion
+  - do:
+      eql.get:
+        id: $id
+        wait_for_completion_timeout: "10s"
+
+  # status for completed eql search
+  - do:
+      eql.get_status:
+        id: $id
+  - match: { id: $id }
+  - match: { is_running: false }
+  - match: { is_partial: false }
+  - is_false: start_time_in_millis
+  - is_true: expiration_time_in_millis
+  - match: { completion_status: 200 }
+
+  - do:
+      eql.delete:
+        id: $id
+  - match: { acknowledged: true }
+
+  # status for a deleted/missing eql search
+  - do:
+      catch: missing
+      eql.get_status:
+        id: $id
+---
+"Sequence checking correct join key ordering.":
+
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by user [network where valid == true] [network where true]'
+  - match: {hits.sequences.0.join_keys.0: "ADMIN"}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
--- a/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml
+++ b/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml
@ -0,0 +1,192 @@
+---
+setup:
+  - do:
+      indices.create:
+          index:  eql_test
+          body:
+            mappings:
+              runtime:
+                day_of_week:
+                  type: keyword
+                  script:
+                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: eql_test
+              _id:    "1"
+          - event:
+              - category: process
+            "@timestamp": "2020-02-03T12:34:56Z"
+            user: SYSTEM
+            id: 123
+            valid: false
+            raw_message: "199.72.81.55 - - [01/Jul/1995:00:00:01 -0400] GET /history/apollo/ HTTP/1.0 200 6245"
+          - index:
+              _index: eql_test
+              _id:    "2"
+          - event:
+              - category: process
+            "@timestamp": "2020-02-04T12:34:56Z"
+            user: SYSTEM
+            id: 123
+            valid: true
+            raw_message: "199.72.81.123 - - [01/Jul/1995:00:00:02 -0400] GET /history/apollo/a HTTP/1.0 200 500"
+          - index:
+              _index: eql_test
+              _id:    "3"
+          - event:
+              - category: process
+            "@timestamp": "2020-02-05T12:34:56Z"
+            user: SYSTEM
+            id: 123
+            valid: true
+            raw_message: "199.72.81.34 - - [01/Jul/1995:00:00:03 -0400] GET /history/apollo/b HTTP/1.0 200 1500"
+          - index:
+              _index: eql_test
+              _id:    "4"
+          - event:
+              - category: process
+            "@timestamp": "2020-02-05T12:34:57Z"
+            user: SYSTEM
+            id: 123
+
+
+---
+"Execute EQL events query with search time keyword runtime field":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where is_valid=="YES"'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","is_valid"]
+          runtime_mappings: {"is_valid": {"type":"keyword","script":"if (doc['valid'].size()==0 || doc['valid'].value == false) emit('NO'); else emit('YES')"}}
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._id: "2"}
+  - match: {hits.events.0.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.valid: [true]}
+  - match: {hits.events.0.fields.is_valid: ["YES"]}
+  - match: {hits.events.1._id: "3"}
+  - match: {hits.events.1.fields.@timestamp: ["1580906096000"]}
+  - match: {hits.events.1.fields.id: [123]}
+  - match: {hits.events.1.fields.valid: [true]}
+  - match: {hits.events.1.fields.is_valid: ["YES"]}
+
+---
+"Execute EQL events query with search time ip runtime field":
+  - do:
+      eql.search:
+        index: eql_test
+        filter_path: "hits.events._source.raw_message,hits.events.fields.address,hits.events._id"
+        body:
+          query: 'process where true'
+          fields: ["address"]
+          runtime_mappings: {"address": {"type": "ip","script": "if (doc[\"raw_message.keyword\"].size() == 0) return; else {Matcher m = /\\d+\\.\\d+\\.\\d+\\.\\d+/.matcher(doc[\"raw_message.keyword\"].value);if (m.find()) emit(m.group());}"}}
+
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0.fields.address: ["199.72.81.55"]}
+  - match: {hits.events.0._source.raw_message: "199.72.81.55 - - [01/Jul/1995:00:00:01 -0400] GET /history/apollo/ HTTP/1.0 200 6245"}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.1.fields.address: ["199.72.81.123"]}
+  - match: {hits.events.1._source.raw_message: "199.72.81.123 - - [01/Jul/1995:00:00:02 -0400] GET /history/apollo/a HTTP/1.0 200 500"}
+  - match: {hits.events.2._id: "3"}
+  - match: {hits.events.2.fields.address: ["199.72.81.34"]}
+  - match: {hits.events.2._source.raw_message: "199.72.81.34 - - [01/Jul/1995:00:00:03 -0400] GET /history/apollo/b HTTP/1.0 200 1500"}
+  - match: {hits.events.3._id: "4"}
+  - match: {hits.events.3.fields.address: null}
+  - match: {hits.events.3._source.raw_message: null}
+
+---
+"Execute EQL events query with search time runtime field overriding mapping level runtime field":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: ["id","day_of_week"]
+          runtime_mappings: {"day_of_week": {"type":"long","script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}}
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 4}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0._source.user: "SYSTEM"}
+  - match: {hits.events.0._source.valid: false}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.day_of_week: [1]}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.1._source.valid: true}
+  - match: {hits.events.1.fields.id: [123]}
+  - match: {hits.events.1.fields.day_of_week: [2]}
+  - match: {hits.events.2._id: "3"}
+  - match: {hits.events.2._source.valid: true}
+  - match: {hits.events.2.fields.id: [123]}
+  - match: {hits.events.2.fields.day_of_week: [3]}
+  - match: {hits.events.3._id: "4"}
+  - match: {hits.events.3.fields.id: [123]}
+  - match: {hits.events.3.fields.day_of_week: [3]}
+
+---
+"Execute EQL sequence with search time runtime fields overriding mapping level runtime field":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true] [process where day_of_week == 3]'
+          fields: ["day_of_week"]
+          runtime_mappings: {"day_of_week": {"type":"long","script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}}
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.0._source.@timestamp: "2020-02-03T12:34:56Z"}
+  - match: {hits.sequences.0.events.0.fields.day_of_week: [1]}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.0.events.1._source.@timestamp: "2020-02-04T12:34:56Z"}
+  - match: {hits.sequences.0.events.1.fields.day_of_week: [2]}
+  - match: {hits.sequences.0.events.2._id: "3"}
+  - match: {hits.sequences.0.events.2._source.@timestamp: "2020-02-05T12:34:56Z"}
+  - match: {hits.sequences.0.events.2.fields.day_of_week: [3]}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.0._source.@timestamp: "2020-02-04T12:34:56Z"}
+  - match: {hits.sequences.1.events.0.fields.day_of_week: [2]}
+  - match: {hits.sequences.1.events.1._id: "3"}
+  - match: {hits.sequences.1.events.1._source.@timestamp: "2020-02-05T12:34:56Z"}
+  - match: {hits.sequences.1.events.1.fields.day_of_week: [3]}
+  - match: {hits.sequences.1.events.2._id: "4"}
+  - match: {hits.sequences.1.events.2._source.@timestamp: "2020-02-05T12:34:57Z"}
+  - match: {hits.sequences.1.events.2.fields.day_of_week: [3]}
+
+---
+"Validate valid runtime mappings request":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: ["id","day_of_week"]
+          runtime_mappings: {"day_of_week": {"script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}}
+      catch: bad_request
+  - match: { error.root_cause.0.type: "action_request_validation_exception" }
+  - match: { error.root_cause.0.reason: "Validation Failed: 1: No type specified for runtime field [day_of_week];" }
+
+
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: ["id","day_of_week"]
+          runtime_mappings: {"day_of_week": [{"type":"long","script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}]}
+      catch: bad_request
+  - match: { error.root_cause.0.type: "action_request_validation_exception" }
+  - match: { error.root_cause.0.reason: "Validation Failed: 1: Expected map for runtime field [day_of_week] definition but got [String];" }
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
@ -102,7 +102,8 @@ public class SecurityServerTransportInterceptor implements TransportInterceptor
            TransportOpenPointInTimeAction.OPEN_SHARD_READER_CONTEXT_NAME,
            ResolveIndexAction.NAME,
            FieldCapabilitiesAction.NAME,
-            FieldCapabilitiesAction.NAME + "[n]"
+            FieldCapabilitiesAction.NAME + "[n]",
+            "indices:data/read/eql"
        );
        REMOTE_ACCESS_ACTION_ALLOWLIST = actions
            // Include action, and proxy equivalent (i.e., with proxy action prefix)