mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
[SIEM][Detection Engine] Fixes critical blocker where signals on signals are not operating
## Summary This fixes halting, infinite creation of signals, and cyclic issues with signals when they are reflected on their own index. Without this fix, you could get a user who looks back at a signals index as both their input and output index and forever generates new signals forever and ever and ever until the heath death of the universe. * Changes the data structure to support parent and ancestors * Adds a check for the parent and ancestors * Adds README.md and in-depth testing of cyclic concepts * Adds README.md and in-depth testing of depth levels of signal concepts * Added unit tests for both use cases * Removed extra console.log statement found in the code base Follow the two README.md's included for testing and explanation of how it works. See `test_cases/signals_on_signals/depth_test` See `test_cases/signals_on_signals/halting_test` ### Checklist Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR. ~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~ ~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~ ~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~ - [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios ~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~ ### For maintainers ~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~ - [x] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)
This commit is contained in:
Frank Hassanabad
GitHub
parent
31d3821598
commit
db1a64da76
29 changed files with 1226 additions and 56 deletions
|
|
@ -114,7 +114,6 @@ async function main() {
|
|||
);
|
||||
return [...accum, parsedLine];
|
||||
} catch (err) {
|
||||
console.log('error parsing a line in this file:', json, line);
|
||||
return accum;
|
||||
}
|
||||
}, []);
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ You should see the new rules created like so:
|
|||
"interval": "5m",
|
||||
"rule_id": "rule-1",
|
||||
"language": "kuery",
|
||||
"output_index": ".siem-signals-frank-hassanabad",
|
||||
"output_index": ".siem-signals-some-name",
|
||||
"max_signals": 100,
|
||||
"risk_score": 1,
|
||||
"name": "Detect Root/Admin Users",
|
||||
|
|
|
|||
|
|
@ -371,7 +371,7 @@ export const getMockPrivileges = () => ({
|
|||
create_snapshot: true,
|
||||
},
|
||||
index: {
|
||||
'.siem-signals-frank-hassanabad-test-space': {
|
||||
'.siem-signals-test-space': {
|
||||
all: false,
|
||||
manage_ilm: true,
|
||||
read: false,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@
|
|||
"properties": {
|
||||
"parent": {
|
||||
"properties": {
|
||||
"rule": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"index": {
|
||||
"type": "keyword"
|
||||
},
|
||||
|
|
@ -19,6 +22,9 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"ancestors": {
|
||||
"type": "object"
|
||||
},
|
||||
"rule": {
|
||||
"properties": {
|
||||
"id": {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
{"created_at":"2020-01-09T01:38:00.740Z","updated_at":"2020-01-09T01:38:00.740Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"6688f367-1aa2-4895-a5a8-b3701eecf57d","immutable":false,"interval":"5m","rule_id":"query-rule-id-1","language":"kuery","output_index":".siem-signals-frank-hassanabad-default","max_signals":100,"risk_score":1,"name":"Query with a rule id Number 1","query":"user.name: root or user.name: admin","references":[],"severity":"high","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
|
||||
{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-2","language":"kuery","output_index":".siem-signals-frank-hassanabad-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
|
||||
{"created_at":"2020-01-09T01:38:00.740Z","updated_at":"2020-01-09T01:38:00.740Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"6688f367-1aa2-4895-a5a8-b3701eecf57d","immutable":false,"interval":"5m","rule_id":"query-rule-id-1","language":"kuery","output_index":".siem-signals-frank-default","max_signals":100,"risk_score":1,"name":"Query with a rule id Number 1","query":"user.name: root or user.name: admin","references":[],"severity":"high","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
|
||||
{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-2","language":"kuery","output_index":".siem-signals-frank-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
|
||||
{"exported_count":2,"missing_rules":[],"missing_rules_count":0}
|
||||
|
|
|
|||
|
|
@ -4,15 +4,3 @@ use these type of rule based messages when writing pure REST API calls. These me
|
|||
more of what you would see "behind the scenes" when you are using Kibana UI which can
|
||||
create rules with additional "meta" data or other implementation details that aren't really
|
||||
a concern for a regular REST API user.
|
||||
|
||||
To post all of them to see in the UI, with the scripts folder as your current working directory:
|
||||
|
||||
```sh
|
||||
./post_rule.sh ./rules/test_cases/*.json
|
||||
```
|
||||
|
||||
To post only one at a time:
|
||||
|
||||
```sh
|
||||
./post_rule.sh ./rules/test_cases/<filename>.json
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"created_at":"2020-01-09T01:38:00.740Z","updated_at":"2020-01-09T01:38:00.740Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"6688f367-1aa2-4895-a5a8-b3701eecf57d","immutable":false,"interval":"5m","rule_id":"query-rule-id-1","language":"kuery","output_index":".siem-signals-frank-hassanabad-default","max_signals":100,"risk_score":1,"name":"Query with a rule id Number 1","query":"user.name: root or user.name: admin","references":[],"severity":"high","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1},
|
||||
{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-2","language":"kuery","output_index":".siem-signals-frank-hassanabad-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
|
||||
{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-3","language":"kuery","output_index":".siem-signals-frank-hassanabad-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
|
||||
{"created_at":"2020-01-09T01:38:00.740Z","updated_at":"2020-01-09T01:38:00.740Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"6688f367-1aa2-4895-a5a8-b3701eecf57d","immutable":false,"interval":"5m","rule_id":"query-rule-id-1","language":"kuery","output_index":".siem-signals-default","max_signals":100,"risk_score":1,"name":"Query with a rule id Number 1","query":"user.name: root or user.name: admin","references":[],"severity":"high","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1},
|
||||
{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-2","language":"kuery","output_index":".siem-signals-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
|
||||
{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-3","language":"kuery","output_index":".siem-signals-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
|
||||
{"exported_count":2,"missing_rules":[],"missing_rules_count":0}
|
||||
|
|
@ -0,0 +1,367 @@
|
|||
This is a depth test which allows users and UI's to create "funnels" of information. You can funnel your data into smaller
|
||||
and smaller data sets using this. For example, you might have 1,000k of events but generate only 100k of signals off
|
||||
of those events. However, you then want to generate signals on top of signals that are only 10k. Likewise you might want
|
||||
signals on top of signals on top of signals to generate only 1k.
|
||||
|
||||
```
|
||||
events from indexes might be 1,000k (no depth)
|
||||
signals -> events would be less such as 100k
|
||||
signals -> signals -> events would be even less (such as 10k)
|
||||
signals -> signals -> events would be even less (such as 1k)
|
||||
```
|
||||
|
||||
This folder contains a rule called
|
||||
|
||||
```sh
|
||||
query_single_id.json
|
||||
```
|
||||
|
||||
which will write a single signal document into the signals index by searching for a single document `"query": "_id: o8G7vm8BvLT8jmu5B1-M"` . Then another rule called
|
||||
|
||||
```sh
|
||||
signal_on_signal_depth_1.json
|
||||
```
|
||||
|
||||
which has this key part of its query: `"query": "signal.parent.depth: 1 and _id: *"` which will only create signals
|
||||
from all signals that point directly to an event (signal -> event).
|
||||
|
||||
Then a second rule called
|
||||
|
||||
```sh
|
||||
signal_on_signal_depth_2.json
|
||||
```
|
||||
|
||||
which will only create signals from all signals that point directly to another signal (signal -> signal) with this query
|
||||
|
||||
```json
|
||||
"query": "signal.parent.depth: 2 and _id: *"
|
||||
```
|
||||
|
||||
## Setup
|
||||
|
||||
You should first get a valid `_id` from the system from the last 24 hours by running any query within timeline
|
||||
or in the system and copying its `_id`. Once you have that `_id` add it to `query_single_id.json`. For example if you have found an `_id`
|
||||
in the last 24 hours of `sQevtW8BvLT8jmu5l0TA` add it to `query_single_id.json` under the key `query` like so:
|
||||
|
||||
```json
|
||||
"query": "_id: sQevtW8BvLT8jmu5l0TA",
|
||||
```
|
||||
|
||||
Then get your current signal index:
|
||||
|
||||
```json
|
||||
./get_signal_index.sh
|
||||
{
|
||||
"name": ".siem-signals-default"
|
||||
}
|
||||
```
|
||||
|
||||
And edit the `signal_on_signal.json` and add that index to the key of `index` so we are running that rule against the signals index:
|
||||
|
||||
```json
|
||||
"index": ".siem-signals-default"
|
||||
```
|
||||
|
||||
Next you want to clear out all of your signals and all rules:
|
||||
|
||||
```sh
|
||||
./hard_reset.sh
|
||||
```
|
||||
|
||||
Finally, insert and start the first the query like so:
|
||||
|
||||
```sh
|
||||
./post_rule.sh ./rules/test_cases/signals_on_signals/depth_test/query_single_id.json
|
||||
```
|
||||
|
||||
Wait 30+ seconds to ensure that the single record shows up in your signals index. You can use dev tools in Kibana
|
||||
to see this by first getting your configured signals index by running:
|
||||
|
||||
```ts
|
||||
./get_signal_index.sh
|
||||
{
|
||||
"name": ".siem-signals-default"
|
||||
}
|
||||
```
|
||||
|
||||
And then you can query against that:
|
||||
|
||||
```ts
|
||||
GET .siem-signals-default/_search
|
||||
```
|
||||
|
||||
Check your parent section of the signal and you will see something like this:
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
The parent and ancestors structure is defined as:
|
||||
|
||||
```
|
||||
rule -> The id of the rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41
|
||||
id -> The original _id of the document
|
||||
type -> The type of the document, it will be either event or signal
|
||||
index -> The original location of the index
|
||||
depth -> The depth of this signal. It will be at least 1 to indicate it is a signal generated from a event. Otherwise 2 or more to indicate a signal on signal and what depth we are at
|
||||
ancestors -> An array tracking all of the parents of this particular signal. As depth increases this will too.
|
||||
```
|
||||
|
||||
This is indicating that you have a single parent of an event from the signal (signal -> event) and this document has a single
|
||||
ancestor of that event. Each 30 seconds that goes it will use de-duplication techniques to ensure that this signal is not re-inserted. If after
|
||||
each 30 seconds you DO SEE multiple signals then the bug is a de-duplication bug and a critical bug. If you ever see a duplicate rule in the
|
||||
ancestors array then that is another CRITICAL bug which needs to be fixed.
|
||||
|
||||
After this is ensured, the next step is to run a single signal on top of a signal by posting once
|
||||
|
||||
```sh
|
||||
./post_rule.sh ./rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json
|
||||
```
|
||||
|
||||
Notice in `signal_on_signal_depth_1.json` we do NOT have a `rule_id` set. This is intentional and is to make it so we can test N rules
|
||||
running in the system which are generating signals on top of signals. After 30 seconds have gone by you should see that you now have two
|
||||
documents in the signals index. The first signal is our original (signal -> event) document with a rule id:
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
and the second document is a signal on top of a signal like so:
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c",
|
||||
"id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
{
|
||||
"rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c",
|
||||
"id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Notice that the depth indicates it is at level 2 and its parent is that of a signal. Also notice that the ancestors is an array of size 2
|
||||
indicating that this signal terminates at an event. Each and every signal ancestors array should terminate at an event and should ONLY contain 1
|
||||
event and NEVER 2 or more events. After 30+ seconds you should NOT see any new documents being created and you should be stable
|
||||
at 2. Otherwise we have AND/OR a de-duplication issue, signal on signal issue.
|
||||
|
||||
Now, post this same rule a second time as a second instance which is going to run against these two documents.
|
||||
|
||||
```sh
|
||||
./post_rule.sh ./rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json
|
||||
```
|
||||
|
||||
If you were to look at the number of rules you have:
|
||||
|
||||
```sh
|
||||
./find_rules.sh
|
||||
```
|
||||
|
||||
You should see that you have 3 rules running concurrently at this point. Write down the `id` to keep track of them
|
||||
|
||||
- 1 event rule which is always finding the same event continuously (id: 74e0dd0c-4609-416f-b65e-90f8b2564612)
|
||||
- 1 signal rule which is finding ALL signals at depth 1 (id: 1d3b3735-66ef-4e53-b7f5-4340026cc40c)
|
||||
- 1 signal rule which is finding ALL signals at depth 1 (id: c93ddb57-e7e9-4973-9886-72ddefb4d22e)
|
||||
|
||||
The expected behavior is that eventually you will get 3 total documents but not additional ones after 1+ minutes. These will be:
|
||||
|
||||
The original event rule 74e0dd0c-4609-416f-b65e-90f8b2564612 (event -> signal)
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
The first signal to signal rule 1d3b3735-66ef-4e53-b7f5-4340026cc40c (signal -> event)
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c",
|
||||
"id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
{
|
||||
"rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c",
|
||||
"id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Then our second signal to signal rule c93ddb57-e7e9-4973-9886-72ddefb4d22e (signal -> event) which finds the same thing as the first
|
||||
signal to signal
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e",
|
||||
"id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
{
|
||||
"rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e",
|
||||
"id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
We should be able to post this depth level as many times as we want and get only 1 new document each time. If we decide though to
|
||||
post `signal_on_signal_depth_2.json` like so:
|
||||
|
||||
```sh
|
||||
./post_rule.sh ./rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json
|
||||
```
|
||||
|
||||
The expectation is that a document for each of the previous depth 1 documents would be produced. Since we have 2 instances of
|
||||
depth 1 rules running then the signals at depth 2 will produce two new ones and those two will look like so:
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "a1f7b520-5bfd-451d-af59-428f60753fee",
|
||||
"id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 3
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
{
|
||||
"rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c",
|
||||
"id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
},
|
||||
{
|
||||
"rule" : "a1f7b520-5bfd-451d-af59-428f60753fee",
|
||||
"id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 3
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "a1f7b520-5bfd-451d-af59-428f60753fee",
|
||||
"id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 3
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
{
|
||||
"rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e",
|
||||
"id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
},
|
||||
{
|
||||
"rule" : "a1f7b520-5bfd-451d-af59-428f60753fee",
|
||||
"id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 3
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
The total number of documents should be 5 at this point. If you were to post this same rule a second time to get a second instance
|
||||
running you will end up with 7 documents as it will only re-report the first 2 and not interfere with the other rules.
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"name": "Queries single id",
|
||||
"description": "Finds only one id below to create a single signal. Change the query to your exact _id you want to test with",
|
||||
"risk_score": 1,
|
||||
"severity": "high",
|
||||
"type": "query",
|
||||
"from": "now-1d",
|
||||
"interval": "30s",
|
||||
"to": "now",
|
||||
"query": "_id: o8G7vm8BvLT8jmu5B1-M",
|
||||
"enabled": true
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"name": "Signal on Signals Rule 1 Depth 1",
|
||||
"description": "Example Signal on Signal where it reports everything as a signal at depth 1",
|
||||
"risk_score": 1,
|
||||
"severity": "high",
|
||||
"type": "query",
|
||||
"from": "now-1d",
|
||||
"interval": "30s",
|
||||
"to": "now",
|
||||
"query": "signal.parent.depth: 1 and _id: *",
|
||||
"enabled": true,
|
||||
"index": ".siem-signals-default"
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"name": "Signal on Signals Rule 1 Depth 2",
|
||||
"description": "Example Signal on Signal where it reports everything as a signal at Depth 2",
|
||||
"risk_score": 1,
|
||||
"severity": "high",
|
||||
"type": "query",
|
||||
"from": "now-1d",
|
||||
"interval": "30s",
|
||||
"to": "now",
|
||||
"query": "signal.parent.depth: 2 and _id: *",
|
||||
"enabled": true,
|
||||
"index": ".siem-signals-default"
|
||||
}
|
||||
|
|
@ -0,0 +1,375 @@
|
|||
This test is to ensure that signals will "halt" eventually when they are run against themselves. This isn't how anyone should setup
|
||||
signals on signals but rather how we will eventually "halt" given the worst case situations where users are running signals on top of signals
|
||||
that are duplicates of each other and going very far back in time.
|
||||
|
||||
It contains a rule called
|
||||
|
||||
```sh
|
||||
query_single_id.json
|
||||
```
|
||||
|
||||
which will write a single signal document into the signals index by searching for a single document `"query": "_id: o8G7vm8BvLT8jmu5B1-M"` . Then another rule called
|
||||
|
||||
```sh
|
||||
signal_on_signal.json
|
||||
```
|
||||
|
||||
which will always generate a signal for EVERY single document it sees `"query": "_id: *"`
|
||||
|
||||
## Setup
|
||||
|
||||
You should first get a valid `_id` from the system from the last 24 hours by running any query within timeline
|
||||
or in the system and copying its `_id`. Once you have that `_id` add it to `query_single_id.json`. For example if you have found an `_id`
|
||||
in the last 24 hours of `sQevtW8BvLT8jmu5l0TA` add it to `query_single_id.json` under the key `query` like so:
|
||||
|
||||
```json
|
||||
"query": "_id: sQevtW8BvLT8jmu5l0TA",
|
||||
```
|
||||
|
||||
Then get your current signal index:
|
||||
|
||||
```json
|
||||
./get_signal_index.sh
|
||||
{
|
||||
"name": ".siem-signals-default"
|
||||
}
|
||||
```
|
||||
|
||||
And edit the `signal_on_signal.json` and add that index to the key of `index` so we are running that rule against the signals index:
|
||||
|
||||
```json
|
||||
"index": ".siem-signals-default"
|
||||
```
|
||||
|
||||
Next you want to clear out all of your signals and all rules:
|
||||
|
||||
```sh
|
||||
./hard_reset.sh
|
||||
```
|
||||
|
||||
Finally, insert and start the first the query like so:
|
||||
|
||||
```sh
|
||||
./post_rule.sh ./rules/test_cases/signals_on_signals/halting_test/query_single_id.json
|
||||
```
|
||||
|
||||
Wait 30+ seconds to ensure that the single record shows up in your signals index. You can use dev tools in Kibana
|
||||
to see this by first getting your configured signals index by running:
|
||||
|
||||
```ts
|
||||
./get_signal_index.sh
|
||||
{
|
||||
"name": ".siem-signals-default"
|
||||
}
|
||||
```
|
||||
|
||||
And then you can query against that:
|
||||
|
||||
```ts
|
||||
GET .siem-signals-default/_search
|
||||
```
|
||||
|
||||
Check your parent section of the signal and you will see something like this:
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
The parent and ancestors structure is defined as:
|
||||
|
||||
```
|
||||
rule -> The id of the rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41
|
||||
id -> The original _id of the document
|
||||
type -> The type of the document, it will be either event or signal
|
||||
index -> The original location of the index
|
||||
depth -> The depth of this signal. It will be at least 1 to indicate it is a signal generated from a event. Otherwise 2 or more to indicate a signal on signal and what depth we are at
|
||||
ancestors -> An array tracking all of the parents of this particular signal. As depth increases this will too.
|
||||
```
|
||||
|
||||
This is indicating that you have a single parent of an event from the signal (signal -> event) and this document has a single
|
||||
ancestor of that event. Each 30 seconds that goes it will use de-duplication techniques to ensure that this signal is not re-inserted. If after
|
||||
each 30 seconds you DO SEE multiple signals then the bug is a de-duplication bug and a critical bug. If you ever see a duplicate rule in the
|
||||
ancestors array then that is another CRITICAL bug which needs to be fixed.
|
||||
|
||||
After this is ensured, the next step is to run a single signal on top of a signal by posting once
|
||||
|
||||
```sh
|
||||
./post_rule.sh ./rules/test_cases/signals_on_signals/halting_test/signal_on_signal.json
|
||||
```
|
||||
|
||||
Notice in `signal_on_signal.json` we do NOT have a `rule_id` set. This is intentional and is to make it so we can test N rules
|
||||
running in the system which are generating signals on top of signals. After 30 seconds have gone by you should see that you now have two
|
||||
documents in the signals index. The first signal is our original (signal -> event) document with a rule id:
|
||||
|
||||
(signal -> event)
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
and the second document is a signal on top of a signal like so:
|
||||
|
||||
(signal -> signal -> event)
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904",
|
||||
"id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
{
|
||||
"rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904",
|
||||
"id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Notice that the depth indicates it is at level 2 and its parent is that of a signal. Also notice that the ancestors is an array of size 2
|
||||
indicating that this signal terminates at an event. Each and every signal ancestors array should terminate at an event and should ONLY contain 1
|
||||
event and NEVER 2 or more events. After 30+ seconds you should NOT see any new documents being created and you should be stable
|
||||
at 2. Otherwise we have AND/OR a de-duplication issue, signal on signal issue.
|
||||
|
||||
Now, post a second signal that is going to run against these two documents.
|
||||
|
||||
```sh
|
||||
./post_rule.sh ./rules/test_cases/signals_on_signals/halting_test/signal_on_signal.json
|
||||
```
|
||||
|
||||
If you were to look at the number of rules you have:
|
||||
|
||||
```sh
|
||||
./find_rules.sh
|
||||
```
|
||||
|
||||
You should see that you have 3 rules running concurrently at this point. Write down the `id` to keep track of them
|
||||
|
||||
- 1 event rule which is always finding the same event continuously (id: ded57b36-9c4e-4ee4-805d-be4e92033e41)
|
||||
- 1 signal rule which is finding ALL signals (id: 161fa5b8-0b96-4985-b066-0d99b2bcb904)
|
||||
- 1 signal rule which is finding ALL signals (id: f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406)
|
||||
|
||||
The expected behavior is that eventually you will get 5 total documents but not additional ones after 1+ minutes. These will be:
|
||||
|
||||
The original event rule ded57b36-9c4e-4ee4-805d-be4e92033e41 (event -> signal)
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
The first signal to signal rule 161fa5b8-0b96-4985-b066-0d99b2bcb904 (signal -> event)
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904",
|
||||
"id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
{
|
||||
"rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904",
|
||||
"id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Then our second signal to signal rule f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406 (signal -> event) which finds the same thing as the first
|
||||
signal to signal
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406",
|
||||
"id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
{
|
||||
"rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406",
|
||||
"id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
But then f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406 also finds the first signal to signal rule from 161fa5b8-0b96-4985-b066-0d99b2bcb904
|
||||
and writes that document out with a depth of 3. (signal -> signal -> event)
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406",
|
||||
"id" : "c627e5e2576f1b10952c6c57249947e89b6153b763a59fb9e391d0b56be8e7fe",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 3
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
{
|
||||
"rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904",
|
||||
"id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
},
|
||||
{
|
||||
"rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406",
|
||||
"id" : "c627e5e2576f1b10952c6c57249947e89b6153b763a59fb9e391d0b56be8e7fe",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 3
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Since it wrote that document, the first signal to signal 161fa5b8-0b96-4985-b066-0d99b2bcb904 writes out it found this newly created signal
|
||||
(signal -> signal -> event)
|
||||
|
||||
```json
|
||||
"parent" : {
|
||||
"rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904",
|
||||
"id" : "efbe514e8d806a5ef3da7658cfa73961e25befefc84f622e963b45dcac798868",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 3
|
||||
},
|
||||
"ancestors" : [
|
||||
{
|
||||
"rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41",
|
||||
"id" : "o8G7vm8BvLT8jmu5B1-M",
|
||||
"type" : "event",
|
||||
"index" : "filebeat-8.0.0-2019.12.18-000001",
|
||||
"depth" : 1
|
||||
},
|
||||
{
|
||||
"rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406",
|
||||
"id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 2
|
||||
},
|
||||
{
|
||||
"rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904",
|
||||
"id" : "efbe514e8d806a5ef3da7658cfa73961e25befefc84f622e963b45dcac798868",
|
||||
"type" : "signal",
|
||||
"index" : ".siem-signals-default-000001",
|
||||
"depth" : 3
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
You will be "halted" at this point as the signal ancestry and de-duplication ensures that we do not report twice on signals and that we do not
|
||||
create additional duplications. So what happens if we create a 3rd rule which does a signal on a signal?
|
||||
|
||||
```sh
|
||||
./post_rule.sh ./rules/test_cases/signals_on_signals/halting_test/signal_on_signal.json
|
||||
```
|
||||
|
||||
That 3rd signal should find all previous 5 signals and write them out. So that's 5 more. Then each signal will report on those 5 giving a depth of
|
||||
4 . Grand total will be 16. You can repeat this as many times as you want and should always see an eventual constant stop time of the signals. They should
|
||||
never keep increasing for this test.
|
||||
|
||||
What about ordering the adding of rules between the query of the document and the signals? This order should not matter and you should get the same
|
||||
results regardless of if you add the signals -> signals rules first or the query a signal event document first. The same number of documents should also
|
||||
be outputted.
|
||||
|
||||
Why does it take sometimes several minutes before things become stable? This is because a rule can write a signal back to the index, then another rule
|
||||
wakes up and writes its document, and the previous rules on next run see this one and creates another chain. This continues until the ancestor detection
|
||||
part of the code realizes that it is going to create a cyclic if it adds the same rule a second time and you no longer have a DAG (Directed Acyclic Graph)
|
||||
at which point it terminates.
|
||||
|
||||
What would happen if I changed the rule look-back from `"from": "now-1d"` to something smaller such as `"from": "now-30s"`? Then you won't get the same
|
||||
number potentially and things are indeterministic because depending on when your rule runs it might find a previous signal and it might not. This is ok
|
||||
and normal as you are then running signals on signals at the same interval as each other and the rules at the moment. A signal on a signal does not detect
|
||||
that another signal has written something and it needs to re-run within the same scheduled time period. It also does not detect that another rule has just
|
||||
written something and does not re-schedule its self to re-run again or against that document.
|
||||
|
||||
How do I then solve the ordering problem event and signal rules writing at the same time? See the `depth_test` folder for more tests around that but you
|
||||
have a few options. You can run your event rules at 5 minute intervals + 5 minute look back, then your signals rule at a 10 minute interval + 10 minute look
|
||||
back which will cause it to check the latest run and the previous run for signals to signals depth of 2. For expected signals that should operate at a depth
|
||||
of 3, you would increase it by another 10 minute look back for a 20 minute interval + 20 minute look back. For level 4, you would increase that to 40 minute
|
||||
look back and adjust your queries accordingly to check the depth for more efficiency in querying. See `depth_test` for more information.
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"name": "Queries single id",
|
||||
"description": "Finds only one id below to create a single signal. Change the query to your exact _id you want to test with",
|
||||
"risk_score": 1,
|
||||
"severity": "high",
|
||||
"type": "query",
|
||||
"from": "now-1d",
|
||||
"interval": "30s",
|
||||
"to": "now",
|
||||
"query": "_id: o8G7vm8BvLT8jmu5B1-M",
|
||||
"enabled": true
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"name": "Signal on Signals Rule 1",
|
||||
"description": "Example Signal on Signal where it reports everything as a signal",
|
||||
"risk_score": 1,
|
||||
"severity": "high",
|
||||
"type": "query",
|
||||
"from": "now-1d",
|
||||
"interval": "30s",
|
||||
"to": "now",
|
||||
"query": "_id: *",
|
||||
"enabled": true,
|
||||
"index": ".siem-signals-default"
|
||||
}
|
||||
|
|
@ -75,7 +75,7 @@ export const sampleDocWithSortId = (someUuid: string = sampleIdGuid): SignalSour
|
|||
sort: ['1234567891111'],
|
||||
});
|
||||
|
||||
export const sampleEmptyDocSearchResults: SignalSearchResponse = {
|
||||
export const sampleEmptyDocSearchResults = (): SignalSearchResponse => ({
|
||||
took: 10,
|
||||
timed_out: false,
|
||||
_shards: {
|
||||
|
|
@ -89,6 +89,44 @@ export const sampleEmptyDocSearchResults: SignalSearchResponse = {
|
|||
max_score: 100,
|
||||
hits: [],
|
||||
},
|
||||
});
|
||||
|
||||
export const sampleDocWithAncestors = (): SignalSearchResponse => {
|
||||
const sampleDoc = sampleDocNoSortId();
|
||||
sampleDoc._source.signal = {
|
||||
parent: {
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
ancestors: [
|
||||
{
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
return {
|
||||
took: 10,
|
||||
timed_out: false,
|
||||
_shards: {
|
||||
total: 10,
|
||||
successful: 10,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
},
|
||||
hits: {
|
||||
total: 0,
|
||||
max_score: 100,
|
||||
hits: [sampleDoc],
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const sampleBulkCreateDuplicateResult = {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import {
|
|||
sampleIdGuid,
|
||||
} from './__mocks__/es_results';
|
||||
import { buildBulkBody } from './build_bulk_body';
|
||||
import { SignalHit } from './types';
|
||||
|
||||
describe('buildBulkBody', () => {
|
||||
beforeEach(() => {
|
||||
|
|
@ -32,18 +33,28 @@ describe('buildBulkBody', () => {
|
|||
});
|
||||
// Timestamp will potentially always be different so remove it for the test
|
||||
delete fakeSignalSourceHit['@timestamp'];
|
||||
expect(fakeSignalSourceHit).toEqual({
|
||||
const expected: Omit<SignalHit, '@timestamp'> & { someKey: 'someValue' } = {
|
||||
someKey: 'someValue',
|
||||
event: {
|
||||
kind: 'signal',
|
||||
},
|
||||
signal: {
|
||||
parent: {
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: sampleIdGuid,
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
ancestors: [
|
||||
{
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: sampleIdGuid,
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
],
|
||||
original_time: 'someTimeStamp',
|
||||
status: 'open',
|
||||
rule: {
|
||||
|
|
@ -74,7 +85,8 @@ describe('buildBulkBody', () => {
|
|||
updated_at: fakeSignalSourceHit.signal.rule?.updated_at,
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
expect(fakeSignalSourceHit).toEqual(expected);
|
||||
});
|
||||
|
||||
test('if bulk body builds original_event if it exists on the event to begin with', () => {
|
||||
|
|
@ -99,7 +111,7 @@ describe('buildBulkBody', () => {
|
|||
});
|
||||
// Timestamp will potentially always be different so remove it for the test
|
||||
delete fakeSignalSourceHit['@timestamp'];
|
||||
expect(fakeSignalSourceHit).toEqual({
|
||||
const expected: Omit<SignalHit, '@timestamp'> & { someKey: 'someValue' } = {
|
||||
someKey: 'someValue',
|
||||
event: {
|
||||
action: 'socket_opened',
|
||||
|
|
@ -115,11 +127,21 @@ describe('buildBulkBody', () => {
|
|||
module: 'system',
|
||||
},
|
||||
parent: {
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: sampleIdGuid,
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
ancestors: [
|
||||
{
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: sampleIdGuid,
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
],
|
||||
original_time: 'someTimeStamp',
|
||||
status: 'open',
|
||||
rule: {
|
||||
|
|
@ -150,7 +172,8 @@ describe('buildBulkBody', () => {
|
|||
updated_at: fakeSignalSourceHit.signal.rule?.updated_at,
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
expect(fakeSignalSourceHit).toEqual(expected);
|
||||
});
|
||||
|
||||
test('if bulk body builds original_event if it exists on the event to begin with but no kind information', () => {
|
||||
|
|
@ -174,7 +197,7 @@ describe('buildBulkBody', () => {
|
|||
});
|
||||
// Timestamp will potentially always be different so remove it for the test
|
||||
delete fakeSignalSourceHit['@timestamp'];
|
||||
expect(fakeSignalSourceHit).toEqual({
|
||||
const expected: Omit<SignalHit, '@timestamp'> & { someKey: 'someValue' } = {
|
||||
someKey: 'someValue',
|
||||
event: {
|
||||
action: 'socket_opened',
|
||||
|
|
@ -189,11 +212,21 @@ describe('buildBulkBody', () => {
|
|||
module: 'system',
|
||||
},
|
||||
parent: {
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: sampleIdGuid,
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
ancestors: [
|
||||
{
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: sampleIdGuid,
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
],
|
||||
original_time: 'someTimeStamp',
|
||||
status: 'open',
|
||||
rule: {
|
||||
|
|
@ -224,7 +257,8 @@ describe('buildBulkBody', () => {
|
|||
updated_at: fakeSignalSourceHit.signal.rule?.updated_at,
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
expect(fakeSignalSourceHit).toEqual(expected);
|
||||
});
|
||||
|
||||
test('if bulk body builds original_event if it exists on the event to begin with with only kind information', () => {
|
||||
|
|
@ -246,7 +280,7 @@ describe('buildBulkBody', () => {
|
|||
});
|
||||
// Timestamp will potentially always be different so remove it for the test
|
||||
delete fakeSignalSourceHit['@timestamp'];
|
||||
expect(fakeSignalSourceHit).toEqual({
|
||||
const expected: Omit<SignalHit, '@timestamp'> & { someKey: 'someValue' } = {
|
||||
someKey: 'someValue',
|
||||
event: {
|
||||
kind: 'signal',
|
||||
|
|
@ -256,11 +290,21 @@ describe('buildBulkBody', () => {
|
|||
kind: 'event',
|
||||
},
|
||||
parent: {
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: sampleIdGuid,
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
ancestors: [
|
||||
{
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: sampleIdGuid,
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
],
|
||||
original_time: 'someTimeStamp',
|
||||
status: 'open',
|
||||
rule: {
|
||||
|
|
@ -291,6 +335,7 @@ describe('buildBulkBody', () => {
|
|||
created_at: fakeSignalSourceHit.signal.rule?.created_at,
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
expect(fakeSignalSourceHit).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,9 +5,8 @@
|
|||
*/
|
||||
|
||||
import { sampleDocNoSortId, sampleRule } from './__mocks__/es_results';
|
||||
import { buildSignal } from './build_signal';
|
||||
import { OutputRuleAlertRest } from '../types';
|
||||
import { Signal } from './types';
|
||||
import { buildSignal, buildAncestor, buildAncestorsSignal } from './build_signal';
|
||||
import { Signal, Ancestor } from './types';
|
||||
|
||||
describe('buildSignal', () => {
|
||||
beforeEach(() => {
|
||||
|
|
@ -17,15 +16,25 @@ describe('buildSignal', () => {
|
|||
test('it builds a signal as expected without original_event if event does not exist', () => {
|
||||
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
|
||||
delete doc._source.event;
|
||||
const rule: Partial<OutputRuleAlertRest> = sampleRule();
|
||||
const rule = sampleRule();
|
||||
const signal = buildSignal(doc, rule);
|
||||
const expected: Signal = {
|
||||
parent: {
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
ancestors: [
|
||||
{
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
],
|
||||
original_time: 'someTimeStamp',
|
||||
status: 'open',
|
||||
rule: {
|
||||
|
|
@ -66,15 +75,25 @@ describe('buildSignal', () => {
|
|||
kind: 'event',
|
||||
module: 'system',
|
||||
};
|
||||
const rule: Partial<OutputRuleAlertRest> = sampleRule();
|
||||
const rule = sampleRule();
|
||||
const signal = buildSignal(doc, rule);
|
||||
const expected: Signal = {
|
||||
parent: {
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
ancestors: [
|
||||
{
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
],
|
||||
original_time: 'someTimeStamp',
|
||||
original_event: {
|
||||
action: 'socket_opened',
|
||||
|
|
@ -112,4 +131,131 @@ describe('buildSignal', () => {
|
|||
};
|
||||
expect(signal).toEqual(expected);
|
||||
});
|
||||
|
||||
test('it builds a ancestor correctly if the parent does not exist', () => {
|
||||
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
|
||||
doc._source.event = {
|
||||
action: 'socket_opened',
|
||||
dataset: 'socket',
|
||||
kind: 'event',
|
||||
module: 'system',
|
||||
};
|
||||
const rule = sampleRule();
|
||||
const signal = buildAncestor(doc, rule);
|
||||
const expected: Ancestor = {
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
};
|
||||
expect(signal).toEqual(expected);
|
||||
});
|
||||
|
||||
test('it builds a ancestor correctly if the parent does exist', () => {
|
||||
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
|
||||
doc._source.event = {
|
||||
action: 'socket_opened',
|
||||
dataset: 'socket',
|
||||
kind: 'event',
|
||||
module: 'system',
|
||||
};
|
||||
doc._source.signal = {
|
||||
parent: {
|
||||
rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
|
||||
id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
ancestors: [
|
||||
{
|
||||
rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
|
||||
id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
const rule = sampleRule();
|
||||
const signal = buildAncestor(doc, rule);
|
||||
const expected: Ancestor = {
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'signal',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 2,
|
||||
};
|
||||
expect(signal).toEqual(expected);
|
||||
});
|
||||
|
||||
test('it builds a signal ancestor correctly if the parent does not exist', () => {
|
||||
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
|
||||
doc._source.event = {
|
||||
action: 'socket_opened',
|
||||
dataset: 'socket',
|
||||
kind: 'event',
|
||||
module: 'system',
|
||||
};
|
||||
const rule = sampleRule();
|
||||
const signal = buildAncestorsSignal(doc, rule);
|
||||
const expected: Ancestor[] = [
|
||||
{
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
];
|
||||
expect(signal).toEqual(expected);
|
||||
});
|
||||
|
||||
test('it builds a signal ancestor correctly if the parent does exist', () => {
|
||||
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
|
||||
doc._source.event = {
|
||||
action: 'socket_opened',
|
||||
dataset: 'socket',
|
||||
kind: 'event',
|
||||
module: 'system',
|
||||
};
|
||||
doc._source.signal = {
|
||||
parent: {
|
||||
rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
|
||||
id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
ancestors: [
|
||||
{
|
||||
rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
|
||||
id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
const rule = sampleRule();
|
||||
const signal = buildAncestorsSignal(doc, rule);
|
||||
const expected: Ancestor[] = [
|
||||
{
|
||||
rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
|
||||
id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
{
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'signal',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 2,
|
||||
},
|
||||
];
|
||||
expect(signal).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,17 +4,52 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { SignalSourceHit, Signal } from './types';
|
||||
import { SignalSourceHit, Signal, Ancestor } from './types';
|
||||
import { OutputRuleAlertRest } from '../types';
|
||||
|
||||
export const buildSignal = (doc: SignalSourceHit, rule: Partial<OutputRuleAlertRest>): Signal => {
|
||||
const signal: Signal = {
|
||||
parent: {
|
||||
export const buildAncestor = (
|
||||
doc: SignalSourceHit,
|
||||
rule: Partial<OutputRuleAlertRest>
|
||||
): Ancestor => {
|
||||
const existingSignal = doc._source.signal?.parent;
|
||||
if (existingSignal != null) {
|
||||
return {
|
||||
rule: rule.id != null ? rule.id : '',
|
||||
id: doc._id,
|
||||
type: 'signal',
|
||||
index: doc._index,
|
||||
depth: existingSignal.depth + 1,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
rule: rule.id != null ? rule.id : '',
|
||||
id: doc._id,
|
||||
type: 'event',
|
||||
index: doc._index,
|
||||
depth: 1,
|
||||
},
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export const buildAncestorsSignal = (
|
||||
doc: SignalSourceHit,
|
||||
rule: Partial<OutputRuleAlertRest>
|
||||
): Signal['ancestors'] => {
|
||||
const newAncestor = buildAncestor(doc, rule);
|
||||
const existingAncestors = doc._source.signal?.ancestors;
|
||||
if (existingAncestors != null) {
|
||||
return [...existingAncestors, newAncestor];
|
||||
} else {
|
||||
return [newAncestor];
|
||||
}
|
||||
};
|
||||
|
||||
export const buildSignal = (doc: SignalSourceHit, rule: Partial<OutputRuleAlertRest>): Signal => {
|
||||
const parent = buildAncestor(doc, rule);
|
||||
const ancestors = buildAncestorsSignal(doc, rule);
|
||||
const signal: Signal = {
|
||||
parent,
|
||||
ancestors,
|
||||
original_time: doc._source['@timestamp'],
|
||||
status: 'open',
|
||||
rule,
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
test('if successful with empty search results', async () => {
|
||||
const sampleParams = sampleRuleAlertParams();
|
||||
const result = await searchAfterAndBulkCreate({
|
||||
someResult: sampleEmptyDocSearchResults,
|
||||
someResult: sampleEmptyDocSearchResults(),
|
||||
ruleParams: sampleParams,
|
||||
services: mockService,
|
||||
logger: mockLogger,
|
||||
|
|
@ -51,6 +51,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
expect(mockService.callCluster).toHaveBeenCalledTimes(0);
|
||||
expect(result).toEqual(true);
|
||||
});
|
||||
|
||||
test('if successful iteration of while loop with maxDocs', async () => {
|
||||
const sampleParams = sampleRuleAlertParams(30);
|
||||
const someGuids = Array.from({ length: 13 }).map(x => uuid.v4());
|
||||
|
|
@ -103,6 +104,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
expect(mockService.callCluster).toHaveBeenCalledTimes(5);
|
||||
expect(result).toEqual(true);
|
||||
});
|
||||
|
||||
test('if unsuccessful first bulk create', async () => {
|
||||
const someGuids = Array.from({ length: 4 }).map(x => uuid.v4());
|
||||
const sampleParams = sampleRuleAlertParams(10);
|
||||
|
|
@ -126,6 +128,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
expect(mockLogger.error).toHaveBeenCalled();
|
||||
expect(result).toEqual(false);
|
||||
});
|
||||
|
||||
test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids', async () => {
|
||||
const sampleParams = sampleRuleAlertParams();
|
||||
mockService.callCluster.mockReturnValueOnce({
|
||||
|
|
@ -156,6 +159,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
expect(mockLogger.error).toHaveBeenCalled();
|
||||
expect(result).toEqual(false);
|
||||
});
|
||||
|
||||
test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids and 0 total hits', async () => {
|
||||
const sampleParams = sampleRuleAlertParams();
|
||||
mockService.callCluster.mockReturnValueOnce({
|
||||
|
|
@ -185,6 +189,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
});
|
||||
expect(result).toEqual(true);
|
||||
});
|
||||
|
||||
test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => {
|
||||
const sampleParams = sampleRuleAlertParams(10);
|
||||
const someGuids = Array.from({ length: 4 }).map(x => uuid.v4());
|
||||
|
|
@ -217,6 +222,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
});
|
||||
expect(result).toEqual(true);
|
||||
});
|
||||
|
||||
test('if successful iteration of while loop with maxDocs and search after returns empty results with no sort ids', async () => {
|
||||
const sampleParams = sampleRuleAlertParams(10);
|
||||
const someGuids = Array.from({ length: 4 }).map(x => uuid.v4());
|
||||
|
|
@ -230,7 +236,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
})
|
||||
.mockReturnValueOnce(sampleEmptyDocSearchResults);
|
||||
.mockReturnValueOnce(sampleEmptyDocSearchResults());
|
||||
const result = await searchAfterAndBulkCreate({
|
||||
someResult: repeatedSearchResultsWithSortId(4, 1, someGuids),
|
||||
ruleParams: sampleParams,
|
||||
|
|
@ -249,6 +255,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
});
|
||||
expect(result).toEqual(true);
|
||||
});
|
||||
|
||||
test('if returns false when singleSearchAfter throws an exception', async () => {
|
||||
const sampleParams = sampleRuleAlertParams(10);
|
||||
const someGuids = Array.from({ length: 4 }).map(x => uuid.v4());
|
||||
|
|
|
|||
|
|
@ -246,7 +246,7 @@ export const signalRulesAlertType = ({
|
|||
// TODO: Error handling and writing of errors into a signal that has error
|
||||
// handling/conditions
|
||||
logger.error(
|
||||
`Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
|
||||
`Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", ${err.message}`
|
||||
);
|
||||
const sDate = new Date().toISOString();
|
||||
currentStatusSavedObject.attributes.status = 'failed';
|
||||
|
|
|
|||
|
|
@ -14,10 +14,11 @@ import {
|
|||
sampleEmptyDocSearchResults,
|
||||
sampleBulkCreateDuplicateResult,
|
||||
sampleBulkCreateErrorResult,
|
||||
sampleDocWithAncestors,
|
||||
} from './__mocks__/es_results';
|
||||
import { savedObjectsClientMock } from 'src/core/server/mocks';
|
||||
import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants';
|
||||
import { singleBulkCreate } from './single_bulk_create';
|
||||
import { singleBulkCreate, filterDuplicateRules } from './single_bulk_create';
|
||||
|
||||
export const mockService = {
|
||||
callCluster: jest.fn(),
|
||||
|
|
@ -131,9 +132,9 @@ describe('singleBulkCreate', () => {
|
|||
expect(firstHash).not.toEqual(secondHash);
|
||||
});
|
||||
});
|
||||
|
||||
test('create successful bulk create', async () => {
|
||||
const sampleParams = sampleRuleAlertParams();
|
||||
const sampleSearchResult = sampleDocSearchResultsNoSortId;
|
||||
mockService.callCluster.mockReturnValueOnce({
|
||||
took: 100,
|
||||
errors: false,
|
||||
|
|
@ -144,7 +145,7 @@ describe('singleBulkCreate', () => {
|
|||
],
|
||||
});
|
||||
const successfulsingleBulkCreate = await singleBulkCreate({
|
||||
someResult: sampleSearchResult(),
|
||||
someResult: sampleDocSearchResultsNoSortId(),
|
||||
ruleParams: sampleParams,
|
||||
services: mockService,
|
||||
logger: mockLogger,
|
||||
|
|
@ -159,9 +160,9 @@ describe('singleBulkCreate', () => {
|
|||
});
|
||||
expect(successfulsingleBulkCreate).toEqual(true);
|
||||
});
|
||||
|
||||
test('create successful bulk create with docs with no versioning', async () => {
|
||||
const sampleParams = sampleRuleAlertParams();
|
||||
const sampleSearchResult = sampleDocSearchResultsNoSortIdNoVersion;
|
||||
mockService.callCluster.mockReturnValueOnce({
|
||||
took: 100,
|
||||
errors: false,
|
||||
|
|
@ -172,7 +173,7 @@ describe('singleBulkCreate', () => {
|
|||
],
|
||||
});
|
||||
const successfulsingleBulkCreate = await singleBulkCreate({
|
||||
someResult: sampleSearchResult(),
|
||||
someResult: sampleDocSearchResultsNoSortIdNoVersion(),
|
||||
ruleParams: sampleParams,
|
||||
services: mockService,
|
||||
logger: mockLogger,
|
||||
|
|
@ -187,12 +188,12 @@ describe('singleBulkCreate', () => {
|
|||
});
|
||||
expect(successfulsingleBulkCreate).toEqual(true);
|
||||
});
|
||||
|
||||
test('create unsuccessful bulk create due to empty search results', async () => {
|
||||
const sampleParams = sampleRuleAlertParams();
|
||||
const sampleSearchResult = sampleEmptyDocSearchResults;
|
||||
mockService.callCluster.mockReturnValue(false);
|
||||
const successfulsingleBulkCreate = await singleBulkCreate({
|
||||
someResult: sampleSearchResult,
|
||||
someResult: sampleEmptyDocSearchResults(),
|
||||
ruleParams: sampleParams,
|
||||
services: mockService,
|
||||
logger: mockLogger,
|
||||
|
|
@ -253,4 +254,71 @@ describe('singleBulkCreate', () => {
|
|||
expect(mockLogger.error).toHaveBeenCalled();
|
||||
expect(successfulsingleBulkCreate).toEqual(true);
|
||||
});
|
||||
|
||||
test('filter duplicate rules will return an empty array given an empty array', () => {
|
||||
const filtered = filterDuplicateRules(
|
||||
'04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
sampleEmptyDocSearchResults()
|
||||
);
|
||||
expect(filtered).toEqual([]);
|
||||
});
|
||||
|
||||
test('filter duplicate rules will return nothing filtered when the two rule ids do not match with each other', () => {
|
||||
const filtered = filterDuplicateRules('some id', sampleDocWithAncestors());
|
||||
expect(filtered).toEqual([
|
||||
{
|
||||
_index: 'myFakeSignalIndex',
|
||||
_type: 'doc',
|
||||
_score: 100,
|
||||
_version: 1,
|
||||
_id: 'e1e08ddc-5e37-49ff-a258-5393aa44435a',
|
||||
_source: {
|
||||
someKey: 'someValue',
|
||||
'@timestamp': 'someTimeStamp',
|
||||
signal: {
|
||||
parent: {
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
ancestors: [
|
||||
{
|
||||
rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('filters duplicate rules will return empty array when the two rule ids match each other', () => {
|
||||
const filtered = filterDuplicateRules(
|
||||
'04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
sampleDocWithAncestors()
|
||||
);
|
||||
expect(filtered).toEqual([]);
|
||||
});
|
||||
|
||||
test('filter duplicate rules will return back search responses if they do not have a signal and will NOT filter the source out', () => {
|
||||
const ancestors = sampleDocWithAncestors();
|
||||
ancestors.hits.hits[0]._source = { '@timestamp': 'some timestamp' };
|
||||
const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', ancestors);
|
||||
expect(filtered).toEqual([
|
||||
{
|
||||
_index: 'myFakeSignalIndex',
|
||||
_type: 'doc',
|
||||
_score: 100,
|
||||
_version: 1,
|
||||
_id: 'e1e08ddc-5e37-49ff-a258-5393aa44435a',
|
||||
_source: { '@timestamp': 'some timestamp' },
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -28,6 +28,28 @@ interface SingleBulkCreateParams {
|
|||
tags: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* This is for signals on signals to work correctly. If given a rule id this will check if
|
||||
* that rule id already exists in the ancestor tree of each signal search response and remove
|
||||
* those documents so they cannot be created as a signal since we do not want a rule id to
|
||||
* ever be capable of re-writing the same signal continuously if both the _input_ and _output_
|
||||
* of the signals index happens to be the same index.
|
||||
* @param ruleId The rule id
|
||||
* @param signalSearchResponse The search response that has all the documents
|
||||
*/
|
||||
export const filterDuplicateRules = (
|
||||
ruleId: string,
|
||||
signalSearchResponse: SignalSearchResponse
|
||||
) => {
|
||||
return signalSearchResponse.hits.hits.filter(doc => {
|
||||
if (doc._source.signal == null) {
|
||||
return true;
|
||||
} else {
|
||||
return !doc._source.signal.ancestors.some(ancestor => ancestor.rule === ruleId);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Bulk Index documents.
|
||||
export const singleBulkCreate = async ({
|
||||
someResult,
|
||||
|
|
@ -43,6 +65,8 @@ export const singleBulkCreate = async ({
|
|||
enabled,
|
||||
tags,
|
||||
}: SingleBulkCreateParams): Promise<boolean> => {
|
||||
someResult.hits.hits = filterDuplicateRules(id, someResult);
|
||||
|
||||
if (someResult.hits.hits.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -51,11 +51,16 @@ export type SearchTypes =
|
|||
| boolean
|
||||
| boolean[]
|
||||
| object
|
||||
| object[];
|
||||
| object[]
|
||||
| undefined;
|
||||
|
||||
export interface SignalSource {
|
||||
[key: string]: SearchTypes;
|
||||
'@timestamp': string;
|
||||
signal?: {
|
||||
parent: Ancestor;
|
||||
ancestors: Ancestor[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface BulkResponse {
|
||||
|
|
@ -123,14 +128,18 @@ export type SignalRuleAlertTypeDefinition = Omit<AlertType, 'executor'> & {
|
|||
executor: ({ services, params, state }: RuleExecutorOptions) => Promise<State | void>;
|
||||
};
|
||||
|
||||
export interface Ancestor {
|
||||
rule: string;
|
||||
id: string;
|
||||
type: string;
|
||||
index: string;
|
||||
depth: number;
|
||||
}
|
||||
|
||||
export interface Signal {
|
||||
rule: Partial<OutputRuleAlertRest>;
|
||||
parent: {
|
||||
id: string;
|
||||
type: string;
|
||||
index: string;
|
||||
depth: number;
|
||||
};
|
||||
parent: Ancestor;
|
||||
ancestors: Ancestor[];
|
||||
original_time: string;
|
||||
original_event?: SearchTypes;
|
||||
status: 'open' | 'closed';
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue