github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[SIEM][Detection Engine] - Update DE to work with new exceptions schema (#69715)

Browse source 
* Updates list entry schema, exposes exception list client, updates tests

* create new de list schema and unit tests

* updated route unit tests and types to match new list schema

* updated existing DE exceptions code so it should now work as is with updated schema

* test and types cleanup

* cleanup

* update unit test

* updates per feedback
This commit is contained in:
Yara Tercero 2020-06-25 09:47:05 -04:00 • committed by GitHub
parent 7a557822f3
commit f7acbbe7a1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
71 changed files with 2528 additions and 2194 deletions
Download patch file Download diff file Expand all files Collapse all files

6
x-pack/plugins/lists/README.md
View file

@ -157,12 +157,14 @@ And you can attach exception list items like so:
{
"field": "actingProcess.file.signer",
"operator": "included",
"match": "Elastic, N.V."
"type": "match",
"value": "Elastic, N.V."
},
{
"field": "event.category",
"operator": "included",
"match_any": [
"type": "match_any",
"value": [
"process",
"malware"
]

6
x-pack/plugins/lists/common/constants.mock.ts
View file

@ -46,10 +46,8 @@ export const EXISTS = 'exists';
export const NESTED = 'nested';
export const ENTRIES: EntriesArray = [
{
entries: [
{ field: 'some.not.nested.field', operator: 'included', type: 'match', value: 'some value' },
],
field: 'some.field',
entries: [{ field: 'nested.field', operator: 'included', type: 'match', value: 'some value' }],
field: 'some.parentField',
type: 'nested',
},
{ field: 'some.not.nested.field', operator: 'included', type: 'match', value: 'some value' },

3
x-pack/plugins/lists/common/schemas/types/default_entries_array.test.ts
View file

@ -17,7 +17,8 @@ import { getEntriesArrayMock, getEntryMatchMock, getEntryNestedMock } from './en
// it checks against every item in that union. Since entries consist of 5
// different entry types, it returns 5 of these. To make more readable,
// extracted here.
const returnedSchemaError = `"Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "list", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "exists" |} | {| entries: Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "list", value: DefaultStringArray |} | {| field: string, operator: "excluded" | "included", type: "exists" |})>, field: string, type: "nested" |})>"`;
const returnedSchemaError =
'"Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, list: {| id: string, type: "ip" | "keyword" |}, operator: "excluded" | "included", type: "list" |} | {| field: string, operator: "excluded" | "included", type: "exists" |} | {| entries: Array<{| field: string, operator: "excluded" | "included", type: "match", value: string |}>, field: string, type: "nested" |})>"';
describe('default_entries_array', () => {
test('it should validate an empty array', () => {

61
x-pack/plugins/lists/common/schemas/types/default_namespace.test.ts Normal file
View file

@ -0,0 +1,61 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { pipe } from 'fp-ts/lib/pipeable';
import { left } from 'fp-ts/lib/Either';
import { foldLeftRight, getPaths } from '../../siem_common_deps';
import { DefaultNamespace } from './default_namespace';
describe('default_namespace', () => {
test('it should validate "single"', () => {
const payload = 'single';
const decoded = DefaultNamespace.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
test('it should validate "agnostic"', () => {
const payload = 'agnostic';
const decoded = DefaultNamespace.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
test('it defaults to "single" if "undefined"', () => {
const payload = undefined;
const decoded = DefaultNamespace.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual('single');
});
test('it defaults to "single" if "null"', () => {
const payload = null;
const decoded = DefaultNamespace.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual('single');
});
test('it should NOT validate if not "single" or "agnostic"', () => {
const payload = 'something else';
const decoded = DefaultNamespace.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
`Invalid value "something else" supplied to "DefaultNamespace"`,
]);
expect(message.schema).toEqual({});
});
});

2
x-pack/plugins/lists/common/schemas/types/default_namespace.ts
View file

@ -7,7 +7,7 @@
import * as t from 'io-ts';
import { Either } from 'fp-ts/lib/Either';
const namespaceType = t.keyof({ agnostic: null, single: null });
export const namespaceType = t.keyof({ agnostic: null, single: null });
type NamespaceType = t.TypeOf<typeof namespaceType>;

6
x-pack/plugins/lists/common/schemas/types/entries.mock.ts
View file

@ -9,10 +9,12 @@ import {
EXISTS,
FIELD,
LIST,
LIST_ID,
MATCH,
MATCH_ANY,
NESTED,
OPERATOR,
TYPE,
} from '../../constants.mock';
import {
@ -40,9 +42,9 @@ export const getEntryMatchAnyMock = (): EntryMatchAny => ({
export const getEntryListMock = (): EntryList => ({
field: FIELD,
list: { id: LIST_ID, type: TYPE },
operator: OPERATOR,
type: LIST,
value: [ENTRY_VALUE],
});
export const getEntryExistsMock = (): EntryExists => ({
@ -52,7 +54,7 @@ export const getEntryExistsMock = (): EntryExists => ({
});
export const getEntryNestedMock = (): EntryNested => ({
entries: [getEntryMatchMock(), getEntryExistsMock()],
entries: [getEntryMatchMock(), getEntryMatchMock()],
field: FIELD,
type: NESTED,
});

22
x-pack/plugins/lists/common/schemas/types/entries.test.ts
View file

@ -251,16 +251,16 @@ describe('Entries', () => {
expect(message.schema).toEqual(payload);
});
test('it should not validate when "value" is not string array', () => {
const payload: Omit<EntryList, 'value'> & { value: string } = {
test('it should not validate when "list" is not expected value', () => {
const payload: Omit<EntryList, 'list'> & { list: string } = {
...getEntryListMock(),
value: 'someListId',
list: 'someListId',
};
const decoded = entriesList.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "someListId" supplied to "value"',
'Invalid value "someListId" supplied to "list"',
]);
expect(message.schema).toEqual({});
});
@ -338,6 +338,20 @@ describe('Entries', () => {
expect(message.schema).toEqual({});
});
test('it should NOT validate when "entries" contains an entry item that is not type "match"', () => {
const payload: Omit<EntryNested, 'entries'> & {
entries: EntryMatchAny[];
} = { ...getEntryNestedMock(), entries: [getEntryMatchAnyMock()] };
const decoded = entriesNested.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "match_any" supplied to "entries,type"',
'Invalid value "["some host name"]" supplied to "entries,value"',
]);
expect(message.schema).toEqual({});
});
test('it should strip out extra keys', () => {
const payload: EntryNested & {
extraKey?: string;

6
x-pack/plugins/lists/common/schemas/types/entries.ts
View file

@ -8,7 +8,7 @@
import * as t from 'io-ts';
import { operator } from '../common/schemas';
import { operator, type } from '../common/schemas';
import { DefaultStringArray } from '../../siem_common_deps';
export const entriesMatch = t.exact(
@ -34,9 +34,9 @@ export type EntryMatchAny = t.TypeOf<typeof entriesMatchAny>;
export const entriesList = t.exact(
t.type({
field: t.string,
list: t.exact(t.type({ id: t.string, type })),
operator,
type: t.keyof({ list: null }),
value: DefaultStringArray,
})
);
export type EntryList = t.TypeOf<typeof entriesList>;
@ -52,7 +52,7 @@ export type EntryExists = t.TypeOf<typeof entriesExists>;
export const entriesNested = t.exact(
t.type({
entries: t.array(t.union([entriesMatch, entriesMatchAny, entriesList, entriesExists])),
entries: t.array(entriesMatch),
field: t.string,
type: t.keyof({ nested: null }),
})

1
x-pack/plugins/lists/common/schemas/types/index.ts
View file

@ -5,5 +5,6 @@
*/
export * from './default_comments_array';
export * from './default_entries_array';
export * from './default_namespace';
export * from './comments';
export * from './entries';

1
x-pack/plugins/lists/server/index.ts
View file

@ -11,6 +11,7 @@ import { ListPlugin } from './plugin';
// exporting these since its required at top level in siem plugin
export { ListClient } from './services/lists/list_client';
export { ExceptionListClient } from './services/exception_lists/exception_list_client';
export { ListPluginSetup } from './types';
export const config = { schema: ConfigSchema };

10
x-pack/plugins/lists/server/saved_objects/exception_list.ts
View file

@ -105,6 +105,16 @@ export const exceptionListItemMapping: SavedObjectsType['mappings'] = {
field: {
type: 'keyword',
},
list: {
properties: {
id: {
type: 'keyword',
},
type: {
type: 'keyword',
},
},
},
operator: {
type: 'keyword',
},

24
x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item_with_list.json Normal file
View file

@ -0,0 +1,24 @@
{
"list_id": "endpoint_list",
"item_id": "endpoint_list_item_lg_val_list",
"_tags": ["endpoint", "process", "malware", "os:windows"],
"tags": ["user added string for a tag", "malware"],
"type": "simple",
"description": "This is a sample exception list item with a large value list included",
"name": "Sample Endpoint Exception List Item with large value list",
"comments": [],
"entries": [
{
"field": "event.module",
"operator": "excluded",
"type": "match_any",
"value": ["zeek"]
},
{
"field": "source.ip",
"operator": "excluded",
"type": "list",
"list": { "id": "list-ip", "type": "ip" }
}
]
}

2
x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json
View file

@ -1,5 +1,5 @@
{
"id": "hand_inserted_item_id",
"list_id": "list-ip",
"value": "127.0.0.1"
"value": "10.4.2.140"
}

7
x-pack/plugins/security_solution/common/detection_engine/lists_common_deps.ts Normal file
View file

@ -0,0 +1,7 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export { EntriesArray, namespaceType } from '../../../lists/common/schemas';

37
x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts
View file

@ -341,40 +341,3 @@ export type Note = t.TypeOf<typeof note>;
export const noteOrUndefined = t.union([note, t.undefined]);
export type NoteOrUndefined = t.TypeOf<typeof noteOrUndefined>;
// NOTE: Experimental list support not being shipped currently and behind a feature flag
// TODO: Remove this comment once we lists have passed testing and is ready for the release
export const list_field = t.string;
export const list_values_operator = t.keyof({ included: null, excluded: null });
export const list_values_type = t.keyof({ match: null, match_all: null, list: null, exists: null });
export const list_values = t.exact(
t.intersection([
t.type({
name: t.string,
}),
t.partial({
id: t.string,
description: t.string,
created_at,
}),
])
);
export const list = t.exact(
t.intersection([
t.type({
field: t.string,
values_operator: list_values_operator,
values_type: list_values_type,
}),
t.partial({ values: t.array(list_values) }),
])
);
export const list_and = t.intersection([
list,
t.partial({
and: t.array(list),
}),
]);
export const listAndOrUndefined = t.union([t.array(list_and), t.undefined]);
export type ListAndOrUndefined = t.TypeOf<typeof listAndOrUndefined>;

27
x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts
View file

@ -40,16 +40,19 @@ import {
} from '../common/schemas';
/* eslint-enable @typescript-eslint/camelcase */
import { DefaultStringArray } from '../types/default_string_array';
import { DefaultActionsArray } from '../types/default_actions_array';
import { DefaultBooleanFalse } from '../types/default_boolean_false';
import { DefaultFromString } from '../types/default_from_string';
import { DefaultIntervalString } from '../types/default_interval_string';
import { DefaultMaxSignalsNumber } from '../types/default_max_signals_number';
import { DefaultToString } from '../types/default_to_string';
import { DefaultThreatArray } from '../types/default_threat_array';
import { DefaultThrottleNull } from '../types/default_throttle_null';
import { ListsDefaultArray, ListsDefaultArraySchema } from '../types/lists_default_array';
import {
DefaultStringArray,
DefaultActionsArray,
DefaultBooleanFalse,
DefaultFromString,
DefaultIntervalString,
DefaultMaxSignalsNumber,
DefaultToString,
DefaultThreatArray,
DefaultThrottleNull,
DefaultListArray,
ListArray,
} from '../types';
/**
* Big differences between this schema and the createRulesSchema
@ -96,7 +99,7 @@ export const addPrepackagedRulesSchema = t.intersection([
throttle: DefaultThrottleNull, // defaults to "null" if not set during decode
references: DefaultStringArray, // defaults to empty array of strings if not set during decode
note, // defaults to "undefined" if not set during decode
exceptions_list: ListsDefaultArray, // defaults to empty array if not set during decode
exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
})
),
]);
@ -130,5 +133,5 @@ export type AddPrepackagedRulesSchemaDecoded = Omit<
to: To;
threat: Threat;
throttle: ThrottleOrNull;
exceptions_list: ListsDefaultArraySchema;
exceptions_list: ListArray;
};

188
x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts
View file

@ -19,6 +19,7 @@ import {
getAddPrepackagedRulesSchemaDecodedMock,
} from './add_prepackaged_rules_schema.mock';
import { DEFAULT_MAX_SIGNALS } from '../../../constants';
import { getListArrayMock } from '../types/lists.mock';
describe('add prepackaged rules schema', () => {
test('empty objects do not validate', () => {
@ -1379,14 +1380,189 @@ describe('add prepackaged rules schema', () => {
});
});
// TODO: The exception_list tests are skipped and empty until we re-integrate it from the lists plugin
describe.skip('exception_list', () => {
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and exceptions_list] does validate', () => {});
describe('exception_list', () => {
test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, version, and exceptions_list] does validate', () => {
const payload: AddPrepackagedRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
version: 1,
exceptions_list: getListArrayMock(),
};
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => {});
const decoded = addPrepackagedRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: AddPrepackagedRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: false,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
version: 1,
filters: [],
exceptions_list: [
{
id: 'some_uuid',
namespace_type: 'single',
},
{
id: 'some_uuid',
namespace_type: 'agnostic',
},
],
};
expect(message.schema).toEqual(expected);
});
test('rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and invalid exceptions_list] does NOT validate', () => {});
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, version, and empty exceptions_list] does validate', () => {
const payload: AddPrepackagedRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
version: 1,
note: '# some markdown',
exceptions_list: [],
};
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => {});
const decoded = addPrepackagedRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: AddPrepackagedRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: false,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
version: 1,
filters: [],
exceptions_list: [],
};
expect(message.schema).toEqual(expected);
});
test('rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, version, and invalid exceptions_list] does NOT validate', () => {
const payload: Omit<AddPrepackagedRulesSchema, 'exceptions_list'> & {
exceptions_list: Array<{ id: string; namespace_type: string }>;
} = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
version: 1,
note: '# some markdown',
exceptions_list: [{ id: 'uuid_here', namespace_type: 'not a namespace type' }],
};
const decoded = addPrepackagedRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "not a namespace type" supplied to "exceptions_list,namespace_type"',
]);
expect(message.schema).toEqual({});
});
test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, version, and non-existent exceptions_list] does validate with empty exceptions_list', () => {
const payload: AddPrepackagedRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
version: 1,
note: '# some markdown',
};
const decoded = addPrepackagedRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: AddPrepackagedRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: false,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
version: 1,
exceptions_list: [],
filters: [],
};
expect(message.schema).toEqual(expected);
});
});
});

184
x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts
View file

@ -18,6 +18,7 @@ import {
getCreateRulesSchemaDecodedMock,
} from './create_rules_schema.mock';
import { DEFAULT_MAX_SIGNALS } from '../../../constants';
import { getListArrayMock } from '../types/lists.mock';
describe('create rules schema', () => {
test('empty objects do not validate', () => {
@ -1435,14 +1436,185 @@ describe('create rules schema', () => {
);
});
// TODO: The exception_list tests are skipped and empty until we re-integrate it from the lists plugin
describe.skip('exception_list', () => {
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and exceptions_list] does validate', () => {});
describe('exception_list', () => {
test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and exceptions_list] does validate', () => {
const payload: CreateRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
exceptions_list: getListArrayMock(),
};
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => {});
const decoded = createRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: CreateRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: true,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
version: 1,
filters: [],
exceptions_list: [
{
id: 'some_uuid',
namespace_type: 'single',
},
{
id: 'some_uuid',
namespace_type: 'agnostic',
},
],
};
expect(message.schema).toEqual(expected);
});
test('rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and invalid exceptions_list] does NOT validate', () => {});
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => {
const payload: CreateRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
exceptions_list: [],
};
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => {});
const decoded = createRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: CreateRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: true,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
version: 1,
filters: [],
exceptions_list: [],
};
expect(message.schema).toEqual(expected);
});
test('rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and invalid exceptions_list] does NOT validate', () => {
const payload: Omit<CreateRulesSchema, 'exceptions_list'> & {
exceptions_list: Array<{ id: string; namespace_type: string }>;
} = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
exceptions_list: [{ id: 'uuid_here', namespace_type: 'not a namespace type' }],
};
const decoded = createRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "not a namespace type" supplied to "exceptions_list,namespace_type"',
]);
expect(message.schema).toEqual({});
});
test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => {
const payload: CreateRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
};
const decoded = createRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: CreateRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: true,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
version: 1,
exceptions_list: [],
filters: [],
};
expect(message.schema).toEqual(expected);
});
});
});

31
x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts
View file

@ -41,18 +41,21 @@ import {
} from '../common/schemas';
/* eslint-enable @typescript-eslint/camelcase */
import { DefaultStringArray } from '../types/default_string_array';
import { DefaultActionsArray } from '../types/default_actions_array';
import { DefaultBooleanTrue } from '../types/default_boolean_true';
import { DefaultFromString } from '../types/default_from_string';
import { DefaultIntervalString } from '../types/default_interval_string';
import { DefaultMaxSignalsNumber } from '../types/default_max_signals_number';
import { DefaultToString } from '../types/default_to_string';
import { DefaultThreatArray } from '../types/default_threat_array';
import { DefaultThrottleNull } from '../types/default_throttle_null';
import { DefaultVersionNumber } from '../types/default_version_number';
import { ListsDefaultArray, ListsDefaultArraySchema } from '../types/lists_default_array';
import { DefaultUuid } from '../types/default_uuid';
import {
DefaultStringArray,
DefaultActionsArray,
DefaultBooleanTrue,
DefaultFromString,
DefaultIntervalString,
DefaultMaxSignalsNumber,
DefaultToString,
DefaultThreatArray,
DefaultThrottleNull,
DefaultVersionNumber,
DefaultListArray,
ListArray,
DefaultUuid,
} from '../types';
export const createRulesSchema = t.intersection([
t.exact(
@ -92,7 +95,7 @@ export const createRulesSchema = t.intersection([
references: DefaultStringArray, // defaults to empty array of strings if not set during decode
note, // defaults to "undefined" if not set during decode
version: DefaultVersionNumber, // defaults to 1 if not set during decode
exceptions_list: ListsDefaultArray, // defaults to empty array if not set during decode
exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
})
),
]);
@ -129,6 +132,6 @@ export type CreateRulesSchemaDecoded = Omit<
threat: Threat;
throttle: ThrottleOrNull;
version: Version;
exceptions_list: ListsDefaultArraySchema;
exceptions_list: ListArray;
rule_id: RuleId;
};

187
x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts
View file

@ -22,6 +22,7 @@ import {
getImportRulesSchemaDecodedMock,
} from './import_rules_schema.mock';
import { DEFAULT_MAX_SIGNALS } from '../../../constants';
import { getListArrayMock } from '../types/lists.mock';
describe('import rules schema', () => {
test('empty objects do not validate', () => {
@ -1569,14 +1570,188 @@ describe('import rules schema', () => {
});
});
// TODO: The exception_list tests are skipped and empty until we re-integrate it from the lists plugin
describe.skip('exception_list', () => {
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and exceptions_list] does validate', () => {});
describe('exception_list', () => {
test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and exceptions_list] does validate', () => {
const payload: ImportRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
exceptions_list: getListArrayMock(),
};
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => {});
const decoded = importRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: ImportRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: true,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
version: 1,
filters: [],
immutable: false,
exceptions_list: [
{
id: 'some_uuid',
namespace_type: 'single',
},
{
id: 'some_uuid',
namespace_type: 'agnostic',
},
],
};
expect(message.schema).toEqual(expected);
});
test('rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and invalid exceptions_list] does NOT validate', () => {});
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => {
const payload: ImportRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
exceptions_list: [],
};
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => {});
const decoded = importRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: ImportRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: true,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
version: 1,
immutable: false,
filters: [],
exceptions_list: [],
};
expect(message.schema).toEqual(expected);
});
test('rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and invalid exceptions_list] does NOT validate', () => {
const payload: Omit<ImportRulesSchema, 'exceptions_list'> & {
exceptions_list: Array<{ id: string; namespace_type: string }>;
} = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
exceptions_list: [{ id: 'uuid_here', namespace_type: 'not a namespace type' }],
};
const decoded = importRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "not a namespace type" supplied to "exceptions_list,namespace_type"',
]);
expect(message.schema).toEqual({});
});
test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => {
const payload: ImportRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
};
const decoded = importRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: ImportRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: true,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
version: 1,
immutable: false,
exceptions_list: [],
filters: [],
};
expect(message.schema).toEqual(expected);
});
});
});

33
x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts
View file

@ -47,19 +47,22 @@ import {
} from '../common/schemas';
/* eslint-enable @typescript-eslint/camelcase */
import { DefaultStringArray } from '../types/default_string_array';
import { DefaultActionsArray } from '../types/default_actions_array';
import { DefaultBooleanTrue } from '../types/default_boolean_true';
import { DefaultFromString } from '../types/default_from_string';
import { DefaultIntervalString } from '../types/default_interval_string';
import { DefaultMaxSignalsNumber } from '../types/default_max_signals_number';
import { DefaultToString } from '../types/default_to_string';
import { DefaultThreatArray } from '../types/default_threat_array';
import { DefaultThrottleNull } from '../types/default_throttle_null';
import { DefaultVersionNumber } from '../types/default_version_number';
import { ListsDefaultArray, ListsDefaultArraySchema } from '../types/lists_default_array';
import { OnlyFalseAllowed } from '../types/only_false_allowed';
import { DefaultStringBooleanFalse } from '../types/default_string_boolean_false';
import {
DefaultStringArray,
DefaultActionsArray,
DefaultBooleanTrue,
DefaultFromString,
DefaultIntervalString,
DefaultMaxSignalsNumber,
DefaultToString,
DefaultThreatArray,
DefaultThrottleNull,
DefaultVersionNumber,
OnlyFalseAllowed,
DefaultStringBooleanFalse,
DefaultListArray,
ListArray,
} from '../types';
/**
* Differences from this and the createRulesSchema are
@ -111,7 +114,7 @@ export const importRulesSchema = t.intersection([
references: DefaultStringArray, // defaults to empty array of strings if not set during decode
note, // defaults to "undefined" if not set during decode
version: DefaultVersionNumber, // defaults to 1 if not set during decode
exceptions_list: ListsDefaultArray, // defaults to empty array if not set during decode
exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
created_at, // defaults "undefined" if not set during decode
updated_at, // defaults "undefined" if not set during decode
created_by, // defaults "undefined" if not set during decode
@ -153,7 +156,7 @@ export type ImportRulesSchemaDecoded = Omit<
threat: Threat;
throttle: ThrottleOrNull;
version: Version;
exceptions_list: ListsDefaultArraySchema;
exceptions_list: ListArray;
rule_id: RuleId;
immutable: false;
};

155
x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.test.ts
View file

@ -10,6 +10,7 @@ import { exactCheck } from '../../../exact_check';
import { pipe } from 'fp-ts/lib/pipeable';
import { foldLeftRight, getPaths } from '../../../test_utils';
import { left } from 'fp-ts/lib/Either';
import { getListArrayMock } from '../types/lists.mock';
describe('patch_rules_schema', () => {
test('made up values do not validate', () => {
@ -1139,14 +1140,156 @@ describe('patch_rules_schema', () => {
expect(message.schema).toEqual({});
});
// TODO: The exception_list tests are skipped and empty until we re-integrate it from the lists plugin
describe.skip('exception_list', () => {
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and exceptions_list] does validate', () => {});
describe('exception_list', () => {
test('[rule_id, description, from, to, index, name, severity, interval, type, filters, note, and exceptions_list] does validate', () => {
const payload: PatchRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
note: '# some documentation markdown',
exceptions_list: getListArrayMock(),
};
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => {});
const decoded = patchRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: PatchRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
note: '# some documentation markdown',
exceptions_list: [
{
id: 'some_uuid',
namespace_type: 'single',
},
{
id: 'some_uuid',
namespace_type: 'agnostic',
},
],
};
expect(message.schema).toEqual(expected);
});
test('rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and invalid exceptions_list] does NOT validate', () => {});
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => {
const payload: PatchRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
exceptions_list: [],
};
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => {});
const decoded = patchRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: PatchRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
exceptions_list: [],
};
expect(message.schema).toEqual(expected);
});
test('rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and invalid exceptions_list] does NOT validate', () => {
const payload: Omit<PatchRulesSchema, 'exceptions_list'> & {
exceptions_list: Array<{ id: string; namespace_type: string }>;
} = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
exceptions_list: [{ id: 'uuid_here', namespace_type: 'not a namespace type' }],
};
const decoded = patchRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "not a namespace type" supplied to "exceptions_list,namespace_type"',
'Invalid value "[{"id":"uuid_here","namespace_type":"not a namespace type"}]" supplied to "exceptions_list"',
]);
expect(message.schema).toEqual({});
});
test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => {
const payload: PatchRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
};
const decoded = patchRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: PatchRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
filters: [],
risk_score: 50,
note: '# some markdown',
};
expect(message.schema).toEqual(expected);
});
});
});

4
x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts
View file

@ -37,10 +37,10 @@ import {
references,
to,
language,
listAndOrUndefined,
query,
id,
} from '../common/schemas';
import { listArrayOrUndefined } from '../types/lists';
/* eslint-enable @typescript-eslint/camelcase */
/**
@ -80,7 +80,7 @@ export const patchRulesSchema = t.exact(
references,
note,
version,
exceptions_list: listAndOrUndefined,
exceptions_list: listArrayOrUndefined,
})
);

181
x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.test.ts
View file

@ -18,6 +18,7 @@ import {
getUpdateRulesSchemaDecodedMock,
} from './update_rules_schema.mock';
import { DEFAULT_MAX_SIGNALS } from '../../../constants';
import { getListArrayMock } from '../types/lists.mock';
describe('update rules schema', () => {
test('empty objects do not validate', () => {
@ -1377,14 +1378,182 @@ describe('update rules schema', () => {
});
});
// TODO: The exception_list tests are skipped and empty until we re-integrate it from the lists plugin
describe.skip('exception_list', () => {
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and exceptions_list] does validate', () => {});
describe('exception_list', () => {
test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and exceptions_list] does validate', () => {
const payload: UpdateRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
filters: [],
note: '# some markdown',
exceptions_list: getListArrayMock(),
};
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => {});
const decoded = updateRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: UpdateRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: true,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
filters: [],
exceptions_list: [
{
id: 'some_uuid',
namespace_type: 'single',
},
{
id: 'some_uuid',
namespace_type: 'agnostic',
},
],
};
expect(message.schema).toEqual(expected);
});
test('rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and invalid exceptions_list] does NOT validate', () => {});
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => {
const payload: UpdateRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
filters: [],
note: '# some markdown',
exceptions_list: [],
};
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => {});
const decoded = updateRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: UpdateRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: true,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
filters: [],
exceptions_list: [],
};
expect(message.schema).toEqual(expected);
});
test('rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and invalid exceptions_list] does NOT validate', () => {
const payload: Omit<UpdateRulesSchema, 'exceptions_list'> & {
exceptions_list: Array<{ id: string; namespace_type: string }>;
} = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
filters: [],
note: '# some markdown',
exceptions_list: [{ id: 'uuid_here', namespace_type: 'not a namespace type' }],
};
const decoded = updateRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "not a namespace type" supplied to "exceptions_list,namespace_type"',
]);
expect(message.schema).toEqual({});
});
test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => {
const payload: UpdateRulesSchema = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
filters: [],
note: '# some markdown',
};
const decoded = updateRulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
const expected: UpdateRulesSchemaDecoded = {
rule_id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
risk_score: 50,
note: '# some markdown',
references: [],
actions: [],
enabled: true,
false_positives: [],
max_signals: DEFAULT_MAX_SIGNALS,
tags: [],
threat: [],
throttle: null,
exceptions_list: [],
filters: [],
};
expect(message.schema).toEqual(expected);
});
});
});

27
x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts
View file

@ -43,16 +43,19 @@ import {
} from '../common/schemas';
/* eslint-enable @typescript-eslint/camelcase */
import { DefaultStringArray } from '../types/default_string_array';
import { DefaultActionsArray } from '../types/default_actions_array';
import { DefaultBooleanTrue } from '../types/default_boolean_true';
import { DefaultFromString } from '../types/default_from_string';
import { DefaultIntervalString } from '../types/default_interval_string';
import { DefaultMaxSignalsNumber } from '../types/default_max_signals_number';
import { DefaultToString } from '../types/default_to_string';
import { DefaultThreatArray } from '../types/default_threat_array';
import { DefaultThrottleNull } from '../types/default_throttle_null';
import { ListsDefaultArray, ListsDefaultArraySchema } from '../types/lists_default_array';
import {
DefaultStringArray,
DefaultActionsArray,
DefaultBooleanTrue,
DefaultFromString,
DefaultIntervalString,
DefaultMaxSignalsNumber,
DefaultToString,
DefaultThreatArray,
DefaultThrottleNull,
DefaultListArray,
ListArray,
} from '../types';
/**
* This almost identical to the create_rules_schema except for a few details.
@ -100,7 +103,7 @@ export const updateRulesSchema = t.intersection([
references: DefaultStringArray, // defaults to empty array of strings if not set during decode
note, // defaults to "undefined" if not set during decode
version, // defaults to "undefined" if not set during decode
exceptions_list: ListsDefaultArray, // defaults to empty array if not set during decode
exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
})
),
]);
@ -135,6 +138,6 @@ export type UpdateRulesSchemaDecoded = Omit<
to: To;
threat: Threat;
throttle: ThrottleOrNull;
exceptions_list: ListsDefaultArraySchema;
exceptions_list: ListArray;
rule_id: RuleId;
};

34
x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts
View file

@ -3,6 +3,7 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { getListArrayMock } from '../types/lists.mock';
import { RulesSchema } from './rules_schema';
@ -64,38 +65,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem
language: 'kuery',
rule_id: 'query-rule-id',
interval: '5m',
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptions_list: getListArrayMock(),
});
export const getRulesMlSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => {

44
x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts
View file

@ -22,6 +22,7 @@ import { exactCheck } from '../../../exact_check';
import { foldLeftRight, getPaths } from '../../../test_utils';
import { TypeAndTimelineOnly } from './type_timeline_only_schema';
import { getRulesSchemaMock, getRulesMlSchemaMock } from './rules_schema.mocks';
import { ListArray } from '../types/lists';
export const ANCHOR_DATE = '2020-02-20T03:57:54.037Z';
@ -650,4 +651,47 @@ describe('rules_schema', () => {
expect(fields.length).toEqual(2);
});
});
describe('exceptions_list', () => {
test('it should validate an empty array for "exceptions_list"', () => {
const payload = getRulesSchemaMock();
payload.exceptions_list = [];
const decoded = rulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
const expected = getRulesSchemaMock();
expected.exceptions_list = [];
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(expected);
});
test('it should NOT validate when "exceptions_list" is not expected type', () => {
const payload: Omit<RulesSchema, 'exceptions_list'> & {
exceptions_list?: string;
} = { ...getRulesSchemaMock(), exceptions_list: 'invalid_data' };
const decoded = rulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "invalid_data" supplied to "exceptions_list"',
]);
expect(message.schema).toEqual({});
});
test('it should default to empty array if "exceptions_list" is undefined ', () => {
const payload: Omit<RulesSchema, 'exceptions_list'> & {
exceptions_list?: ListArray;
} = getRulesSchemaMock();
payload.exceptions_list = undefined;
const decoded = rulesSchema.decode(payload);
const checked = exactCheck(payload, decoded);
const message = pipe(checked, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual({ ...payload, exceptions_list: [] });
});
});
});

4
x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts
View file

@ -56,7 +56,7 @@ import {
meta,
note,
} from '../common/schemas';
import { ListsDefaultArray } from '../types/lists_default_array';
import { DefaultListArray } from '../types/lists_default_array';
/**
* This is the required fields for the rules schema response. Put all required properties on
@ -87,7 +87,7 @@ export const requiredRulesSchema = t.type({
updated_at,
created_by,
version,
exceptions_list: ListsDefaultArray,
exceptions_list: DefaultListArray,
});
export type RequiredRulesSchema = t.TypeOf<typeof requiredRulesSchema>;

34
x-pack/plugins/security_solution/common/detection_engine/schemas/types/index.ts Normal file
View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export * from './default_actions_array';
export * from './default_boolean_false';
export * from './default_boolean_true';
export * from './default_empty_string';
export * from './default_export_file_name';
export * from './default_from_string';
export * from './default_interval_string';
export * from './default_language_string';
export * from './default_max_signals_number';
export * from './default_page';
export * from './default_per_page';
export * from './default_string_array';
export * from './default_string_boolean_false';
export * from './default_threat_array';
export * from './default_throttle_null';
export * from './default_to_string';
export * from './default_uuid';
export * from './default_version_number';
export * from './iso_date_string';
export * from './lists';
export * from './lists_default_array';
export * from './non_empty_string';
export * from './only_false_allowed';
export * from './positive_integer';
export * from './positive_integer_greater_than_zero';
export * from './references_default_array';
export * from './risk_score';
export * from './uuid';

18
x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.mock.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { List, ListArray } from './lists';
export const getListMock = (): List => ({
id: 'some_uuid',
namespace_type: 'single',
});
export const getListAgnosticMock = (): List => ({
id: 'some_uuid',
namespace_type: 'agnostic',
});
export const getListArrayMock = (): ListArray => [getListMock(), getListAgnosticMock()];

131
x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.test.ts Normal file
View file

@ -0,0 +1,131 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { pipe } from 'fp-ts/lib/pipeable';
import { left } from 'fp-ts/lib/Either';
import { foldLeftRight, getPaths } from '../../../test_utils';
import { getListAgnosticMock, getListMock, getListArrayMock } from './lists.mock';
import {
List,
ListArray,
ListArrayOrUndefined,
list,
listArray,
listArrayOrUndefined,
} from './lists';
describe('Lists', () => {
describe('list', () => {
test('it should validate a list', () => {
const payload = getListMock();
const decoded = list.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
test('it should validate a list with "namespace_type" of"agnostic"', () => {
const payload = getListAgnosticMock();
const decoded = list.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
test('it should NOT validate a list without an "id"', () => {
const payload = getListMock();
delete payload.id;
const decoded = list.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "undefined" supplied to "id"',
]);
expect(message.schema).toEqual({});
});
test('it should NOT validate a list without "namespace_type"', () => {
const payload = getListMock();
delete payload.namespace_type;
const decoded = list.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "undefined" supplied to "namespace_type"',
]);
expect(message.schema).toEqual({});
});
test('it should strip out extra keys', () => {
const payload: List & {
extraKey?: string;
} = getListMock();
payload.extraKey = 'some value';
const decoded = list.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(getListMock());
});
});
describe('listArray', () => {
test('it should validate an array of lists', () => {
const payload = getListArrayMock();
const decoded = listArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
test('it should not validate when unexpected type found in array', () => {
const payload = ([1] as unknown) as ListArray;
const decoded = listArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "1" supplied to "Array<{| id: string, namespace_type: "agnostic" | "single" |}>"',
]);
expect(message.schema).toEqual({});
});
});
describe('listArrayOrUndefined', () => {
test('it should validate an array of lists', () => {
const payload = getListArrayMock();
const decoded = listArrayOrUndefined.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
test('it should validate when undefined', () => {
const payload = undefined;
const decoded = listArrayOrUndefined.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
test('it should not allow an item that is not of type "list" in array', () => {
const payload = ([1] as unknown) as ListArrayOrUndefined;
const decoded = listArrayOrUndefined.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "1" supplied to "(Array<{| id: string, namespace_type: "agnostic" | "single" |}> | undefined)"',
'Invalid value "[1]" supplied to "(Array<{| id: string, namespace_type: "agnostic" | "single" |}> | undefined)"',
]);
expect(message.schema).toEqual({});
});
});
});

22
x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.ts Normal file
View file

@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import * as t from 'io-ts';
import { namespaceType } from '../../lists_common_deps';
export const list = t.exact(
t.type({
id: t.string,
namespace_type: namespaceType,
})
);
export type List = t.TypeOf<typeof list>;
export const listArray = t.array(list);
export type ListArray = t.TypeOf<typeof listArray>;
export const listArrayOrUndefined = t.union([listArray, t.undefined]);
export type ListArrayOrUndefined = t.TypeOf<typeof listArrayOrUndefined>;

187
x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists_default_array.test.ts
View file

@ -4,15 +4,36 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { ListsDefaultArray } from './lists_default_array';
import { pipe } from 'fp-ts/lib/pipeable';
import { left } from 'fp-ts/lib/Either';
import { foldLeftRight, getPaths } from '../../../test_utils';
import { DefaultListArray, DefaultListArrayC } from './lists_default_array';
import { getListArrayMock } from './lists.mock';
describe('lists_default_array', () => {
test('it should return a default array when null', () => {
const payload = null;
const decoded = DefaultListArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual([]);
});
test('it should return a default array when undefined', () => {
const payload = undefined;
const decoded = DefaultListArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual([]);
});
test('it should validate an empty array', () => {
const payload: string[] = [];
const decoded = ListsDefaultArray.decode(payload);
const decoded = DefaultListArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
@ -20,171 +41,23 @@ describe('lists_default_array', () => {
});
test('it should validate an array of lists', () => {
const payload = [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
];
const decoded = ListsDefaultArray.decode(payload);
const payload = getListArrayMock();
const decoded = DefaultListArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
test('it should not validate an array of lists that includes a values_operator other than included or excluded', () => {
const payload = [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'exists',
},
{
field: 'host.hostname',
values_operator: 'jibber jabber',
values_type: 'exists',
},
];
const decoded = ListsDefaultArray.decode(payload);
test('it should not validate an array of non accepted types', () => {
// Terrible casting for purpose of tests
const payload = ([1] as unknown) as DefaultListArrayC;
const decoded = DefaultListArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "jibber jabber" supplied to "values_operator"',
'Invalid value "1" supplied to "DefaultListArray"',
]);
expect(message.schema).toEqual({});
});
// TODO - this scenario should never come up, as the values key is forbidden when values_type is "exists" in the incoming schema - need to find a good way to do this in io-ts
test('it will validate an array of lists that includes "values" when "values_type" is "exists"', () => {
const payload = [
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'exists',
values: [
{
name: '127.0.0.1',
},
],
},
];
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
// TODO - this scenario should never come up, as the values key is required when values_type is "match" in the incoming schema - need to find a good way to do this in io-ts
test('it will validate an array of lists that does not include "values" when "values_type" is "match"', () => {
const payload = [
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
},
];
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
// TODO - this scenario should never come up, as the values key is required when values_type is "match_all" in the incoming schema - need to find a good way to do this in io-ts
test('it will validate an array of lists that does not include "values" when "values_type" is "match_all"', () => {
const payload = [
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match_all',
},
];
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
// TODO - this scenario should never come up, as the values key is required when values_type is "list" in the incoming schema - need to find a good way to do this in io-ts
test('it should not validate an array of lists that does not include "values" when "values_type" is "list"', () => {
const payload = [
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'list',
},
];
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
test('it should not validate an array with a number', () => {
const payload = [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
values: [
{
name: '127.0.0.1',
},
],
},
5,
];
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "5" supplied to "listsWithDefaultArray"',
'Invalid value "5" supplied to "listsWithDefaultArray"',
]);
expect(message.schema).toEqual({});
});
test('it should return a default array entry', () => {
const payload = null;
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual([]);
});
});

28
x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists_default_array.ts
View file

@ -7,28 +7,18 @@
import * as t from 'io-ts';
import { Either } from 'fp-ts/lib/Either';
import {
list_and as listAnd,
list_values as listValues,
list_values_operator as listOperator,
} from '../common/schemas';
import { ListArray, list } from './lists';
export type List = t.TypeOf<typeof listAnd>;
export type ListValues = t.TypeOf<typeof listValues>;
export type ListOperator = t.TypeOf<typeof listOperator>;
export type DefaultListArrayC = t.Type<ListArray, ListArray, unknown>;
/**
* Types the ListsDefaultArray as:
* - If null or undefined, then a default array will be set for the list
* Types the DefaultListArray as:
* - If null or undefined, then a default array of type list will be set
*/
export const ListsDefaultArray = new t.Type<List[], List[], unknown>(
'listsWithDefaultArray',
t.array(listAnd).is,
(input, context): Either<t.Errors, List[]> =>
input == null ? t.success([]) : t.array(listAnd).validate(input, context),
export const DefaultListArray: DefaultListArrayC = new t.Type<ListArray, ListArray, unknown>(
'DefaultListArray',
t.array(list).is,
(input, context): Either<t.Errors, ListArray> =>
input == null ? t.success([]) : t.array(list).validate(input, context),
t.identity
);
export type ListsDefaultArrayC = typeof ListsDefaultArray;
export type ListsDefaultArraySchema = t.TypeOf<typeof ListsDefaultArray>;

6
x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx
View file

@ -215,7 +215,7 @@ describe('Exception helpers', () => {
fieldName: 'host.name',
isNested: false,
operator: 'is in list',
value: ['some host name'],
value: 'some-list-id',
},
{
fieldName: 'host.name',
@ -238,8 +238,8 @@ describe('Exception helpers', () => {
{
fieldName: 'host.name.host.name',
isNested: true,
operator: 'exists',
value: null,
operator: 'is',
value: 'some host name',
},
];
expect(result).toEqual(expected);

13
x-pack/plugins/security_solution/public/common/components/exceptions/helpers.tsx
View file

@ -19,6 +19,7 @@ import {
OperatorTypeEnum,
entriesNested,
entriesExists,
entriesList,
} from '../../../lists_plugin_deps';
/**
@ -87,6 +88,16 @@ export const getFormattedEntries = (entries: EntriesArray): FormattedEntry[] =>
return formattedEntries.flat();
};
export const getEntryValue = (entry: Entry): string | string[] | null => {
if (entriesList.is(entry)) {
return entry.list.id;
} else if (entriesExists.is(entry)) {
return null;
} else {
return entry.value;
}
};
/**
* Helper method for `getFormattedEntries`
*/
@ -100,7 +111,7 @@ export const formatEntry = ({
item: Entry;
}): FormattedEntry => {
const operator = getExceptionOperatorSelect(item);
const value = !entriesExists.is(item) ? item.value : null;
const value = getEntryValue(item);
return {
fieldName: isNested ? `${parent}.${item.field}` : item.field,

1
x-pack/plugins/security_solution/public/lists_plugin_deps.ts
View file

@ -27,4 +27,5 @@ export {
OperatorTypeEnum,
entriesNested,
entriesExists,
entriesList,
} from '../../lists/common/schemas';

34
x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts
View file

@ -27,6 +27,7 @@ import { RuleNotificationAlertType } from '../../notifications/types';
import { QuerySignalsSchemaDecoded } from '../../../../../common/detection_engine/schemas/request/query_signals_index_schema';
import { SetSignalsStatusSchemaDecoded } from '../../../../../common/detection_engine/schemas/request/set_signal_status_schema';
import { getCreateRulesSchemaMock } from '../../../../../common/detection_engine/schemas/request/create_rules_schema.mock';
import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock';
export const typicalSetStatusSignalByIdsPayload = (): SetSignalsStatusSchemaDecoded => ({
signal_ids: ['somefakeid1', 'somefakeid2'],
@ -390,38 +391,7 @@ export const getResult = (): RuleAlertType => ({
references: ['http://www.example.com', 'https://ww.example.com'],
note: '# Investigative notes',
version: 1,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptionsList: getListArrayMock(),
},
createdAt: new Date('2019-12-13T16:40:33.400Z'),
updatedAt: new Date('2019-12-13T16:40:33.400Z'),

34
x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts
View file

@ -8,6 +8,7 @@ import { Readable } from 'stream';
import { HapiReadableStream } from '../../rules/types';
import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema';
import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock';
/**
* Given a string, builds a hapi stream as our
@ -76,38 +77,7 @@ export const getOutputRuleAlertForRest = (): Omit<
],
},
],
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptions_list: getListArrayMock(),
filters: [
{
query: {

34
x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.test.ts
View file

@ -14,6 +14,7 @@ import { FindResult } from '../../../../../../alerts/server';
import { BulkError } from '../utils';
import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags';
import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema';
import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock';
export const ruleOutput: RulesSchema = {
actions: [],
@ -68,38 +69,7 @@ export const ruleOutput: RulesSchema = {
},
},
],
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptions_list: getListArrayMock(),
index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
meta: {
someMeta: 'someField',

32
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.test.ts
View file

@ -80,36 +80,8 @@ describe('getExportAll', () => {
note: '# Investigative notes',
version: 1,
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
{ id: 'some_uuid', namespace_type: 'single' },
{ id: 'some_uuid', namespace_type: 'agnostic' },
],
})}\n`,
exportDetails: `${JSON.stringify({

64
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.test.ts
View file

@ -88,36 +88,8 @@ describe('get_export_by_object_ids', () => {
note: '# Investigative notes',
version: 1,
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
{ id: 'some_uuid', namespace_type: 'single' },
{ id: 'some_uuid', namespace_type: 'agnostic' },
],
})}\n`,
exportDetails: `${JSON.stringify({
@ -216,36 +188,8 @@ describe('get_export_by_object_ids', () => {
note: '# Investigative notes',
version: 1,
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
{ id: 'some_uuid', namespace_type: 'single' },
{ id: 'some_uuid', namespace_type: 'agnostic' },
],
},
],

9
x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts
View file

@ -14,7 +14,6 @@ import {
SavedObjectsClientContract,
} from 'kibana/server';
import { RuleAlertAction } from '../../../../common/detection_engine/types';
import { ListsDefaultArraySchema } from '../../../../common/detection_engine/schemas/types/lists_default_array';
import {
FalsePositives,
From,
@ -62,7 +61,6 @@ import {
ThreatOrUndefined,
TypeOrUndefined,
ReferencesOrUndefined,
ListAndOrUndefined,
PerPageOrUndefined,
PageOrUndefined,
SortFieldOrUndefined,
@ -80,6 +78,7 @@ import { AlertsClient, PartialAlert } from '../../../../../alerts/server';
import { Alert, SanitizedAlert } from '../../../../../alerts/common';
import { SIGNALS_ID } from '../../../../common/constants';
import { RuleTypeParams, PartialFilter } from '../types';
import { ListArrayOrUndefined, ListArray } from '../../../../common/detection_engine/schemas/types';
export interface RuleAlertType extends Alert {
params: RuleTypeParams;
@ -194,7 +193,7 @@ export interface CreateRulesOptions {
references: References;
note: NoteOrUndefined;
version: Version;
exceptionsList: ListsDefaultArraySchema;
exceptionsList: ListArray;
actions: RuleAlertAction[];
}
@ -230,7 +229,7 @@ export interface UpdateRulesOptions {
references: References;
note: NoteOrUndefined;
version: VersionOrUndefined;
exceptionsList: ListsDefaultArraySchema;
exceptionsList: ListArray;
actions: RuleAlertAction[];
}
@ -264,7 +263,7 @@ export interface PatchRulesOptions {
references: ReferencesOrUndefined;
note: NoteOrUndefined;
version: VersionOrUndefined;
exceptionsList: ListAndOrUndefined;
exceptionsList: ListArrayOrUndefined;
actions: RuleAlertAction[] | undefined;
rule: SanitizedAlert | null;
}

4
x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts
View file

@ -31,9 +31,9 @@ import {
ThreatOrUndefined,
TypeOrUndefined,
ReferencesOrUndefined,
ListAndOrUndefined,
} from '../../../../common/detection_engine/schemas/common/schemas';
import { PartialFilter } from '../types';
import { ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types';
export const calculateInterval = (
interval: string | undefined,
@ -74,7 +74,7 @@ export interface UpdateProperties {
references: ReferencesOrUndefined;
note: NoteOrUndefined;
version: VersionOrUndefined;
exceptionsList: ListAndOrUndefined;
exceptionsList: ListArrayOrUndefined;
anomalyThreshold: AnomalyThresholdOrUndefined;
}

27
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/patches/update_list.json
View file

@ -2,31 +2,8 @@
"rule_id": "query-with-list",
"exceptions_list": [
{
"field": "source.ip",
"values_operator": "excluded",
"values_type": "exists"
},
{
"field": "host.name",
"values_operator": "included",
"values_type": "match",
"values": [
{
"name": "rock01"
}
],
"and": [
{
"field": "host.id",
"values_operator": "included",
"values_type": "match_all",
"values": [
{
"name": "123456"
}
]
}
]
"id": "some_updated_fake_id",
"namespace_type": "single"
}
]
}

35
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/lists/query_with_and.json
View file

@ -1,35 +0,0 @@
{
"name": "List - and",
"description": "Query with a list that includes and. This rule should only produce signals when host.name exists and when both event.module is endgame and event.category is anything other than file",
"rule_id": "query-with-list-and",
"risk_score": 1,
"severity": "high",
"type": "query",
"query": "host.name: *",
"interval": "30s",
"language": "kuery",
"exceptions_list": [
{
"field": "event.module",
"values_operator": "excluded",
"values_type": "match",
"values": [
{
"name": "endgame"
}
],
"and": [
{
"field": "event.category",
"values_operator": "included",
"values_type": "match",
"values": [
{
"name": "file"
}
]
}
]
}
]
}

23
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/lists/query_with_excluded.json
View file

@ -1,23 +0,0 @@
{
"name": "List - excluded",
"description": "Query with a list of values_operator excluded. This rule should only produce signals when host.name exists and event.module is suricata",
"rule_id": "query-with-list-excluded",
"risk_score": 1,
"severity": "high",
"type": "query",
"query": "host.name: *",
"interval": "30s",
"language": "kuery",
"exceptions_list": [
{
"field": "event.module",
"values_operator": "excluded",
"values_type": "match",
"values": [
{
"name": "suricata"
}
]
}
]
}

18
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/lists/query_with_exists.json
View file

@ -1,18 +0,0 @@
{
"name": "List - exists",
"description": "Query with a list that includes exists. This rule should only produce signals when host.name exists and event.action does not exist",
"rule_id": "query-with-list-exists",
"risk_score": 1,
"severity": "high",
"type": "query",
"query": "host.name: *",
"interval": "30s",
"language": "kuery",
"exceptions_list": [
{
"field": "event.action",
"values_operator": "included",
"values_type": "exists"
}
]
}

54
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list.json
View file

@ -1,54 +0,0 @@
{
"name": "Query with a list",
"description": "Query with a list. This rule should only produce signals when either host.name exists and event.module is system and user.name is zeek or gdm OR when host.name exists and event.module is not endgame or zeek or system.",
"rule_id": "query-with-list",
"risk_score": 2,
"severity": "high",
"type": "query",
"query": "host.name: *",
"interval": "30s",
"language": "kuery",
"exceptions_list": [
{
"field": "event.module",
"values_operator": "excluded",
"values_type": "match",
"values": [
{
"name": "system"
}
],
"and": [
{
"field": "user.name",
"values_operator": "excluded",
"values_type": "match_all",
"values": [
{
"name": "zeek"
},
{
"name": "gdm"
}
]
}
]
},
{
"field": "event.module",
"values_operator": "included",
"values_type": "match_all",
"values": [
{
"name": "endgame"
},
{
"name": "zeek"
},
{
"name": "system"
}
]
}
]
}

24
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list_plugin.json
View file

@ -1,24 +0,0 @@
{
"name": "Query with a list",
"description": "Query with a list only generate signals if source.ip is not in list",
"rule_id": "query-with-list",
"risk_score": 2,
"severity": "high",
"type": "query",
"query": "host.name: *",
"interval": "30s",
"language": "kuery",
"exceptions_list": [
{
"field": "source.ip",
"values_operator": "excluded",
"values_type": "list",
"values": [
{
"id": "ci-badguys.txt",
"name": "ip"
}
]
}
]
}

23
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/lists/query_with_match.json
View file

@ -1,23 +0,0 @@
{
"name": "List - match",
"description": "Query with a list that includes match. This rule should only produce signals when host.name exists and event.module is not suricata",
"rule_id": "query-with-list-match",
"risk_score": 1,
"severity": "high",
"type": "query",
"query": "host.name: *",
"interval": "30s",
"language": "kuery",
"exceptions_list": [
{
"field": "event.module",
"values_operator": "included",
"values_type": "match",
"values": [
{
"name": "suricata"
}
]
}
]
}

26
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/lists/query_with_match_all.json
View file

@ -1,26 +0,0 @@
{
"name": "List - match_all",
"description": "Query with a list that includes match_all. This rule should only produce signals when host.name exists and event.module is not suricata or auditd",
"rule_id": "query-with-list-match-all",
"risk_score": 1,
"severity": "high",
"type": "query",
"query": "host.name: *",
"interval": "30s",
"language": "kuery",
"exceptions_list": [
{
"field": "event.module",
"values_operator": "included",
"values_type": "match_all",
"values": [
{
"name": "suricata"
},
{
"name": "auditd"
}
]
}
]
}

32
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/lists/query_with_or.json
View file

@ -1,32 +0,0 @@
{
"name": "List - or",
"description": "Query with a list that includes or. This rule should only produce signals when host.name exists and event.module is suricata OR when host.name exists and event.category is file",
"rule_id": "query-with-list-or",
"risk_score": 1,
"severity": "high",
"type": "query",
"query": "host.name: *",
"interval": "30s",
"exceptions_list": [
{
"field": "event.module",
"values_operator": "excluded",
"values_type": "match",
"values": [
{
"name": "suricata"
}
]
},
{
"field": "event.category",
"values_operator": "excluded",
"values_type": "match",
"values": [
{
"name": "file"
}
]
}
]
}

10
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_list.json Normal file
View file

@ -0,0 +1,10 @@
{
"name": "Rule w exceptions",
"description": "Sample rule with exception list",
"risk_score": 1,
"severity": "high",
"type": "query",
"query": "host.name: *",
"interval": "30s",
"exceptions_list": [{ "id": "endpoint_list", "namespace_type": "single" }]
}

30
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/updates/update_list.json
View file

@ -6,33 +6,5 @@
"severity": "high",
"type": "query",
"query": "user.name: root or user.name: admin",
"exceptions_list": [
{
"field": "source.ip",
"values_operator": "excluded",
"values_type": "exists"
},
{
"field": "host.name",
"values_operator": "included",
"values_type": "match",
"values": [
{
"name": "rock01"
}
],
"and": [
{
"field": "host.id",
"values_operator": "included",
"values_type": "match_all",
"values": [
{
"name": "123456"
}
]
}
]
}
]
"exceptions_list": [{ "id": "some_updated_fake_id", "namespace_type": "single" }]
}

34
x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
View file

@ -14,6 +14,7 @@ import { loggingSystemMock } from '../../../../../../../../src/core/server/mocks
import { RuleTypeParams } from '../../types';
import { IRuleStatusAttributes } from '../../rules/types';
import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings';
import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock';
export const sampleRuleAlertParams = (
maxSignals?: number | undefined,
@ -44,38 +45,7 @@ export const sampleRuleAlertParams = (
meta: undefined,
threat: undefined,
version: 1,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptionsList: getListArrayMock(),
});
export const sampleDocNoSortId = (someUuid: string = sampleIdGuid): SignalSourceHit => ({

133
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts
View file

@ -12,6 +12,7 @@ import {
} from './__mocks__/es_results';
import { buildBulkBody } from './build_bulk_body';
import { SignalHit } from './types';
import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock';
describe('buildBulkBody', () => {
beforeEach(() => {
@ -91,38 +92,7 @@ describe('buildBulkBody', () => {
version: 1,
created_at: fakeSignalSourceHit.signal.rule?.created_at,
updated_at: fakeSignalSourceHit.signal.rule?.updated_at,
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptions_list: getListArrayMock(),
},
},
};
@ -218,38 +188,7 @@ describe('buildBulkBody', () => {
updated_at: fakeSignalSourceHit.signal.rule?.updated_at,
throttle: 'no_actions',
threat: [],
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptions_list: getListArrayMock(),
},
},
};
@ -343,38 +282,7 @@ describe('buildBulkBody', () => {
created_at: fakeSignalSourceHit.signal.rule?.created_at,
updated_at: fakeSignalSourceHit.signal.rule?.updated_at,
throttle: 'no_actions',
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptions_list: getListArrayMock(),
},
},
};
@ -461,38 +369,7 @@ describe('buildBulkBody', () => {
updated_at: fakeSignalSourceHit.signal.rule?.updated_at,
created_at: fakeSignalSourceHit.signal.rule?.created_at,
throttle: 'no_actions',
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptions_list: getListArrayMock(),
},
},
};

1215
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_exceptions_query.test.ts
View file

File diff suppressed because it is too large Load diff

160
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_exceptions_query.ts
View file

@ -3,17 +3,23 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import {
ListAndOrUndefined,
Language,
Query,
} from '../../../../common/detection_engine/schemas/common/schemas';
import {
ListOperator,
ListValues,
List,
} from '../../../../common/detection_engine/schemas/types/lists_default_array';
import { Language, Query } from '../../../../common/detection_engine/schemas/common/schemas';
import { Query as DataQuery } from '../../../../../../../src/plugins/data/server';
import {
Entry,
ExceptionListItemSchema,
EntryMatch,
EntryMatchAny,
EntryNested,
EntryExists,
EntriesArray,
Operator,
entriesMatchAny,
entriesExists,
entriesMatch,
entriesNested,
entriesList,
} from '../../../../../lists/common/schemas';
type Operators = 'and' | 'or' | 'not';
type LuceneOperators = 'AND' | 'OR' | 'NOT';
@ -41,37 +47,30 @@ export const operatorBuilder = ({
operator,
language,
}: {
operator: ListOperator;
operator: Operator;
language: Language;
}): string => {
const and = getLanguageBooleanOperator({
language,
value: 'and',
});
const or = getLanguageBooleanOperator({
const not = getLanguageBooleanOperator({
language,
value: 'not',
});
switch (operator) {
case 'excluded':
return ` ${and} `;
case 'included':
return ` ${and} ${or} `;
return `${not} `;
default:
return '';
}
};
export const buildExists = ({
operator,
field,
item,
language,
}: {
operator: ListOperator;
field: string;
item: EntryExists;
language: Language;
}): string => {
const { operator, field } = item;
const exceptionOperator = operatorBuilder({ operator, language });
switch (language) {
@ -85,64 +84,70 @@ export const buildExists = ({
};
export const buildMatch = ({
operator,
field,
values,
item,
language,
}: {
operator: ListOperator;
field: string;
values: ListValues[];
item: EntryMatch;
language: Language;
}): string => {
if (values.length > 0) {
const exceptionOperator = operatorBuilder({ operator, language });
const [exception] = values;
const { value, operator, field } = item;
const exceptionOperator = operatorBuilder({ operator, language });
return `${exceptionOperator}${field}:${exception.name}`;
} else {
return '';
}
return `${exceptionOperator}${field}:${value}`;
};
export const buildMatchAll = ({
operator,
field,
values,
export const buildMatchAny = ({
item,
language,
}: {
operator: ListOperator;
field: string;
values: ListValues[];
item: EntryMatchAny;
language: Language;
}): string => {
switch (values.length) {
const { value, operator, field } = item;
switch (value.length) {
case 0:
return '';
case 1:
return buildMatch({ operator, field, values, language });
default:
const or = getLanguageBooleanOperator({ language, value: 'or' });
const exceptionOperator = operatorBuilder({ operator, language });
const matchAllValues = values.map((value) => {
return value.name;
});
const matchAnyValues = value.map((v) => v);
return `${exceptionOperator}${field}:(${matchAllValues.join(` ${or} `)})`;
return `${exceptionOperator}${field}:(${matchAnyValues.join(` ${or} `)})`;
}
};
export const evaluateValues = ({ list, language }: { list: List; language: Language }): string => {
const { values_operator: operator, values_type: type, field, values } = list;
switch (type) {
case 'exists':
return buildExists({ operator, field, language });
case 'match':
return buildMatch({ operator, field, values: values ?? [], language });
case 'match_all':
return buildMatchAll({ operator, field, values: values ?? [], language });
default:
return '';
export const buildNested = ({
item,
language,
}: {
item: EntryNested;
language: Language;
}): string => {
const { field, entries } = item;
const and = getLanguageBooleanOperator({ language, value: 'and' });
const values = entries.map((entry) => `${entry.field}:${entry.value}`);
return `${field}:{ ${values.join(` ${and} `)} }`;
};
export const evaluateValues = ({
item,
language,
}: {
item: Entry | EntryNested;
language: Language;
}): string => {
if (entriesExists.is(item)) {
return buildExists({ item, language });
} else if (entriesMatch.is(item)) {
return buildMatch({ item, language });
} else if (entriesMatchAny.is(item)) {
return buildMatchAny({ item, language });
} else if (entriesNested.is(item)) {
return buildNested({ item, language });
} else {
return '';
}
};
@ -157,8 +162,9 @@ export const formatQuery = ({
}): string => {
if (exceptions.length > 0) {
const or = getLanguageBooleanOperator({ language, value: 'or' });
const and = getLanguageBooleanOperator({ language, value: 'and' });
const formattedExceptions = exceptions.map((exception) => {
return `(${query}${exception})`;
return `(${query} ${and} ${exception})`;
});
return formattedExceptions.join(` ${or} `);
@ -167,23 +173,22 @@ export const formatQuery = ({
}
};
export const buildExceptions = ({
query,
export const buildExceptionItemEntries = ({
lists,
language,
}: {
query: string;
lists: List[];
lists: EntriesArray;
language: Language;
}): string[] => {
return lists.reduce<string[]>((accum, listItem) => {
const { and, ...exceptionDetails } = { ...listItem };
const andExceptionsSegments = and ? buildExceptions({ query, lists: and, language }) : [];
const exceptionSegment = evaluateValues({ list: exceptionDetails, language });
const exception = [...exceptionSegment, ...andExceptionsSegments];
}): string => {
const and = getLanguageBooleanOperator({ language, value: 'and' });
const exceptionItem = lists
.filter((t) => !entriesList.is(t))
.reduce<string[]>((accum, listItem) => {
const exceptionSegment = evaluateValues({ item: listItem, language });
return [...accum, exceptionSegment];
}, []);
return [...accum, exception.join('')];
}, []);
return exceptionItem.join(` ${and} `);
};
export const buildQueryExceptions = ({
@ -193,12 +198,13 @@ export const buildQueryExceptions = ({
}: {
query: Query;
language: Language;
lists: ListAndOrUndefined;
lists: ExceptionListItemSchema[] | undefined;
}): DataQuery[] => {
if (lists && lists !== null) {
const exceptions = buildExceptions({ lists, language, query });
const exceptions = lists.map((exceptionItem) =>
buildExceptionItemEntries({ lists: exceptionItem.entries, language })
);
const formattedQuery = formatQuery({ exceptions, language, query });
return [
{
query: formattedQuery,

100
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts
View file

@ -7,6 +7,7 @@
import { buildRule } from './build_rule';
import { sampleRuleAlertParams, sampleRuleGuid } from './__mocks__/es_results';
import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock';
describe('buildRule', () => {
beforeEach(() => {
@ -80,38 +81,7 @@ describe('buildRule', () => {
query: 'host.name: Braden',
},
],
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptions_list: getListArrayMock(),
version: 1,
};
expect(rule).toEqual(expected);
@ -164,38 +134,7 @@ describe('buildRule', () => {
updated_at: rule.updated_at,
created_at: rule.created_at,
throttle: 'no_actions',
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptions_list: getListArrayMock(),
};
expect(rule).toEqual(expected);
});
@ -247,38 +186,7 @@ describe('buildRule', () => {
updated_at: rule.updated_at,
created_at: rule.created_at,
throttle: 'no_actions',
exceptions_list: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
],
exceptions_list: getListArrayMock(),
};
expect(rule).toEqual(expected);
});

171
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts
View file

@ -8,6 +8,7 @@ import uuid from 'uuid';
import { filterEventsAgainstList } from './filter_events_with_list';
import { mockLogger, repeatedSearchResultsWithSortId } from './__mocks__/es_results';
import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
import { getListItemResponseMock } from '../../../../../lists/common/schemas/response/list_item_schema.mock';
import { listMock } from '../../../../../lists/server/mocks';
@ -36,92 +37,42 @@ describe('filterEventsAgainstList', () => {
expect(res.hits.hits.length).toEqual(4);
});
it('should throw an error if malformed exception list present', async () => {
let message = '';
try {
await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'excluded',
values_type: 'list',
values: undefined,
},
],
eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'7.7.7.7',
]),
});
} catch (exc) {
message = exc.message;
}
expect(message).toEqual(
'Failed to query lists index. Reason: Malformed exception list provided'
);
});
it('should throw an error if unsupported exception type', async () => {
let message = '';
try {
await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'excluded',
values_type: 'list',
values: [
{
id: 'ci-badguys.txt',
name: 'unsupportedListPluginType',
},
],
},
],
eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'7.7.7.7',
]),
});
} catch (exc) {
message = exc.message;
}
expect(message).toEqual(
'Failed to query lists index. Reason: Unsupported list type used, please use one of ip,keyword'
);
});
describe('operator_type is includes', () => {
describe('operator_type is included', () => {
it('should respond with same list if no items match value list', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'list',
values: [
{
id: 'ci-badguys.txt',
name: 'ip',
},
],
},
],
exceptionsList: [exceptionItem],
eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)),
});
expect(res.hits.hits.length).toEqual(4);
});
it('should respond with less items in the list if some values match', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
listClient.getListItemByValues = jest.fn(({ value }) =>
Promise.resolve(
value.slice(0, 2).map((item) => ({
@ -133,19 +84,7 @@ describe('filterEventsAgainstList', () => {
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'list',
values: [
{
id: 'ci-badguys.txt',
name: 'ip',
},
],
},
],
exceptionsList: [exceptionItem],
eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [
'1.1.1.1',
'2.2.2.2',
@ -162,27 +101,39 @@ describe('filterEventsAgainstList', () => {
});
describe('operator type is excluded', () => {
it('should respond with empty list if no items match value list', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'excluded',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'excluded',
values_type: 'list',
values: [
{
id: 'ci-badguys.txt',
name: 'ip',
},
],
},
],
exceptionsList: [exceptionItem],
eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)),
});
expect(res.hits.hits.length).toEqual(0);
});
it('should respond with less items in the list if some values match', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'excluded',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
listClient.getListItemByValues = jest.fn(({ value }) =>
Promise.resolve(
value.slice(0, 2).map((item) => ({
@ -194,19 +145,7 @@ describe('filterEventsAgainstList', () => {
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'excluded',
values_type: 'list',
values: [
{
id: 'ci-badguys.txt',
name: 'ip',
},
],
},
],
exceptionsList: [exceptionItem],
eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [
'1.1.1.1',
'2.2.2.2',

118
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts
View file

@ -6,15 +6,17 @@
import { get } from 'lodash/fp';
import { Logger } from 'src/core/server';
import { ListAndOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas';
import { List } from '../../../../common/detection_engine/schemas/types/lists_default_array';
import { type } from '../../../../../lists/common/schemas/common';
import { ListClient } from '../../../../../lists/server';
import { SignalSearchResponse, SearchTypes } from './types';
import {
entriesList,
EntryList,
ExceptionListItemSchema,
} from '../../../../../lists/common/schemas';
interface FilterEventsAgainstList {
listClient: ListClient;
exceptionsList: ListAndOrUndefined;
exceptionsList: ExceptionListItemSchema[];
logger: Logger;
eventSearchResult: SignalSearchResponse;
}
@ -34,63 +36,63 @@ export const filterEventsAgainstList = async ({
const isStringableType = (val: SearchTypes) =>
['string', 'number', 'boolean'].includes(typeof val);
// grab the signals with values found in the given exception lists.
const filteredHitsPromises = exceptionsList
.filter((exceptionItem: List) => exceptionItem.values_type === 'list')
.map(async (exceptionItem: List) => {
if (exceptionItem.values == null || exceptionItem.values.length === 0) {
throw new Error('Malformed exception list provided');
}
if (!type.is(exceptionItem.values[0].name)) {
throw new Error(
`Unsupported list type used, please use one of ${Object.keys(type.keys).join()}`
);
}
if (!exceptionItem.values[0].id) {
throw new Error(`Missing list id for exception on field ${exceptionItem.field}`);
}
// acquire the list values we are checking for.
const valuesOfGivenType = eventSearchResult.hits.hits.reduce((acc, searchResultItem) => {
const valueField = get(exceptionItem.field, searchResultItem._source);
if (valueField != null && isStringableType(valueField)) {
acc.add(valueField.toString());
}
return acc;
}, new Set<string>());
const filteredHitsPromises = exceptionsList.map(
async (exceptionItem: ExceptionListItemSchema) => {
const { entries } = exceptionItem;
// matched will contain any list items that matched with the
// values passed in from the Set.
const matchedListItems = await listClient.getListItemByValues({
listId: exceptionItem.values[0].id,
type: exceptionItem.values[0].name,
value: [...valuesOfGivenType],
});
const filteredHitsEntries = entries
.filter((t): t is EntryList => entriesList.is(t))
.map(async (entry) => {
// acquire the list values we are checking for.
const valuesOfGivenType = eventSearchResult.hits.hits.reduce(
(acc, searchResultItem) => {
const valueField = get(entry.field, searchResultItem._source);
if (valueField != null && isStringableType(valueField)) {
acc.add(valueField.toString());
}
return acc;
},
new Set<string>()
);
// create a set of list values that were a hit - easier to work with
const matchedListItemsSet = new Set<SearchTypes>(
matchedListItems.map((item) => item.value)
);
// matched will contain any list items that matched with the
// values passed in from the Set.
const matchedListItems = await listClient.getListItemByValues({
listId: entry.list.id,
type: entry.list.type,
value: [...valuesOfGivenType],
});
// do a single search after with these values.
// painless script to do nested query in elasticsearch
// filter out the search results that match with the values found in the list.
const operator = exceptionItem.values_operator;
const filteredEvents = eventSearchResult.hits.hits.filter((item) => {
const eventItem = get(exceptionItem.field, item._source);
if (operator === 'included') {
if (eventItem != null) {
return !matchedListItemsSet.has(eventItem);
}
} else if (operator === 'excluded') {
if (eventItem != null) {
return matchedListItemsSet.has(eventItem);
}
}
return false;
});
const diff = eventSearchResult.hits.hits.length - filteredEvents.length;
logger.debug(`Lists filtered out ${diff} events`);
return filteredEvents;
});
// create a set of list values that were a hit - easier to work with
const matchedListItemsSet = new Set<SearchTypes>(
matchedListItems.map((item) => item.value)
);
// do a single search after with these values.
// painless script to do nested query in elasticsearch
// filter out the search results that match with the values found in the list.
const operator = entry.operator;
const filteredEvents = eventSearchResult.hits.hits.filter((item) => {
const eventItem = get(entry.field, item._source);
if (operator === 'included') {
if (eventItem != null) {
return !matchedListItemsSet.has(eventItem);
}
} else if (operator === 'excluded') {
if (eventItem != null) {
return matchedListItemsSet.has(eventItem);
}
}
return false;
});
const diff = eventSearchResult.hits.hits.length - filteredEvents.length;
logger.debug(`Lists filtered out ${diff} events`);
return filteredEvents;
});
return (await Promise.all(filteredHitsEntries)).flat();
}
);
const filteredHits = await Promise.all(filteredHitsPromises);
const toReturn: SignalSearchResponse = {

117
x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts
View file

@ -7,6 +7,7 @@
import { getQueryFilter, getFilter } from './get_filter';
import { PartialFilter } from '../types';
import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks';
import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
describe('get_filter', () => {
let servicesMock: AlertServicesMock;
@ -381,18 +382,7 @@ describe('get_filter', () => {
'kuery',
[],
['auditbeat-*'],
[
{
field: 'event.module',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'suricata',
},
],
},
]
[getExceptionListItemSchemaMock()]
);
expect(esQuery).toEqual({
bool: {
@ -414,11 +404,39 @@ describe('get_filter', () => {
},
{
bool: {
minimum_should_match: 1,
should: [
filter: [
{
match: {
'event.module': 'suricata',
nested: {
path: 'some.parentField',
query: {
bool: {
minimum_should_match: 1,
should: [
{
match: {
'some.parentField.nested.field': 'some value',
},
},
],
},
},
score_mode: 'none',
},
},
{
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [
{
match: {
'some.not.nested.field': 'some value',
},
},
],
},
},
},
},
],
@ -450,7 +468,7 @@ describe('get_filter', () => {
});
test('it should work when lists has value undefined', () => {
const esQuery = getQueryFilter('host.name: linux', 'kuery', [], ['auditbeat-*'], undefined);
const esQuery = getQueryFilter('host.name: linux', 'kuery', [], ['auditbeat-*'], []);
expect(esQuery).toEqual({
bool: {
filter: [
@ -529,7 +547,7 @@ describe('get_filter', () => {
savedId: undefined,
services: servicesMock,
index: ['auditbeat-*'],
lists: undefined,
lists: [],
});
expect(filter).toEqual({
bool: {
@ -564,7 +582,7 @@ describe('get_filter', () => {
savedId: undefined,
services: servicesMock,
index: ['auditbeat-*'],
lists: undefined,
lists: [],
})
).rejects.toThrow('query, filters, and index parameter should be defined');
});
@ -579,7 +597,7 @@ describe('get_filter', () => {
savedId: undefined,
services: servicesMock,
index: ['auditbeat-*'],
lists: undefined,
lists: [],
})
).rejects.toThrow('query, filters, and index parameter should be defined');
});
@ -594,7 +612,7 @@ describe('get_filter', () => {
savedId: undefined,
services: servicesMock,
index: undefined,
lists: undefined,
lists: [],
})
).rejects.toThrow('query, filters, and index parameter should be defined');
});
@ -608,7 +626,7 @@ describe('get_filter', () => {
savedId: 'some-id',
services: servicesMock,
index: ['auditbeat-*'],
lists: undefined,
lists: [],
});
expect(filter).toEqual({
bool: {
@ -632,7 +650,7 @@ describe('get_filter', () => {
savedId: undefined,
services: servicesMock,
index: ['auditbeat-*'],
lists: undefined,
lists: [],
})
).rejects.toThrow('savedId parameter should be defined');
});
@ -647,7 +665,7 @@ describe('get_filter', () => {
savedId: 'some-id',
services: servicesMock,
index: undefined,
lists: undefined,
lists: [],
})
).rejects.toThrow('savedId parameter should be defined');
});
@ -662,7 +680,7 @@ describe('get_filter', () => {
savedId: 'some-id',
services: servicesMock,
index: undefined,
lists: undefined,
lists: [],
})
).rejects.toThrow('Unsupported Rule of type "machine_learning" supplied to getFilter');
});
@ -812,18 +830,7 @@ describe('get_filter', () => {
savedId: undefined,
services: servicesMock,
index: ['auditbeat-*'],
lists: [
{
field: 'event.module',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'suricata',
},
],
},
],
lists: [getExceptionListItemSchemaMock()],
});
expect(filter).toEqual({
bool: {
@ -845,11 +852,39 @@ describe('get_filter', () => {
},
{
bool: {
minimum_should_match: 1,
should: [
filter: [
{
match: {
'event.module': 'suricata',
nested: {
path: 'some.parentField',
query: {
bool: {
minimum_should_match: 1,
should: [
{
match: {
'some.parentField.nested.field': 'some value',
},
},
],
},
},
score_mode: 'none',
},
},
{
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [
{
match: {
'some.not.nested.field': 'some value',
},
},
],
},
},
},
},
],

6
x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts
View file

@ -10,11 +10,11 @@ import {
Type,
SavedIdOrUndefined,
IndexOrUndefined,
ListAndOrUndefined,
Language,
Index,
Query,
} from '../../../../common/detection_engine/schemas/common/schemas';
import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
import { AlertServices } from '../../../../../alerts/server';
import { assertUnreachable } from '../../../utils/build_query';
import {
@ -33,7 +33,7 @@ export const getQueryFilter = (
language: Language,
filters: PartialFilter[],
index: Index,
lists: ListAndOrUndefined
lists: ExceptionListItemSchema[]
) => {
const indexPattern = {
fields: [],
@ -64,7 +64,7 @@ interface GetFilterArgs {
savedId: SavedIdOrUndefined;
services: AlertServices;
index: IndexOrUndefined;
lists: ListAndOrUndefined;
lists: ExceptionListItemSchema[];
}
interface QueryAttributes {

134
x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts
View file

@ -17,6 +17,7 @@ import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mock
import uuid from 'uuid';
import { getListItemResponseMock } from '../../../../../lists/common/schemas/response/list_item_schema.mock';
import { listMock } from '../../../../../lists/server/mocks';
import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
describe('searchAfterAndBulkCreate', () => {
let mockService: AlertServicesMock;
@ -94,22 +95,23 @@ describe('searchAfterAndBulkCreate', () => {
},
],
});
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'list',
values: [
{
id: 'ci-badguys.txt',
name: 'ip',
},
],
},
],
exceptionsList: [exceptionItem],
services: mockService,
logger: mockLogger,
id: sampleRuleGuid,
@ -168,22 +170,22 @@ describe('searchAfterAndBulkCreate', () => {
},
],
});
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'list',
values: [
{
id: 'ci-badguys.txt',
name: 'ip',
},
],
},
],
exceptionsList: [exceptionItem],
services: mockService,
logger: mockLogger,
id: sampleRuleGuid,
@ -254,7 +256,7 @@ describe('searchAfterAndBulkCreate', () => {
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleParams: sampleParams,
listClient,
exceptionsList: undefined,
exceptionsList: [],
services: mockService,
logger: mockLogger,
id: sampleRuleGuid,
@ -281,25 +283,25 @@ describe('searchAfterAndBulkCreate', () => {
});
test('if unsuccessful first bulk create', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
const sampleParams = sampleRuleAlertParams(10);
mockService.callCluster
.mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)))
.mockRejectedValue(new Error('bulk failed')); // Added this recently
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'list',
values: [
{
id: 'ci-badguys.txt',
name: 'ip',
},
],
},
],
exceptionsList: [exceptionItem],
ruleParams: sampleParams,
services: mockService,
logger: mockLogger,
@ -327,6 +329,18 @@ describe('searchAfterAndBulkCreate', () => {
});
test('should return success with 0 total hits', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
const sampleParams = sampleRuleAlertParams();
mockService.callCluster.mockResolvedValueOnce(sampleEmptyDocSearchResults());
listClient.getListItemByValues = jest.fn(({ value }) =>
@ -339,19 +353,7 @@ describe('searchAfterAndBulkCreate', () => {
);
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'list',
values: [
{
id: 'ci-badguys.txt',
name: 'ip',
},
],
},
],
exceptionsList: [exceptionItem],
ruleParams: sampleParams,
services: mockService,
logger: mockLogger,
@ -405,21 +407,21 @@ describe('searchAfterAndBulkCreate', () => {
}))
)
);
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
listClient,
exceptionsList: [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'list',
values: [
{
id: 'ci-badguys.txt',
name: 'ip',
},
],
},
],
exceptionsList: [exceptionItem],
ruleParams: sampleParams,
services: mockService,
logger: mockLogger,

4
x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts
View file

@ -4,7 +4,6 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { ListAndOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas';
import { AlertServices } from '../../../../../alerts/server';
import { ListClient } from '../../../../../lists/server';
import { RuleAlertAction } from '../../../../common/detection_engine/types';
@ -14,12 +13,13 @@ import { singleSearchAfter } from './single_search_after';
import { singleBulkCreate } from './single_bulk_create';
import { SignalSearchResponse } from './types';
import { filterEventsAgainstList } from './filter_events_with_list';
import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
interface SearchAfterAndBulkCreateParams {
ruleParams: RuleTypeParams;
services: AlertServices;
listClient: ListClient | undefined; // TODO: undefined is for temporary development, remove before merged
exceptionsList: ListAndOrUndefined;
exceptionsList: ExceptionListItemSchema[];
logger: Logger;
id: string;
inputIndexPattern: string[];

14
x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts
View file

@ -10,7 +10,7 @@ import { getResult, getMlResult } from '../routes/__mocks__/request_responses';
import { signalRulesAlertType } from './signal_rule_alert_type';
import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks';
import { ruleStatusServiceFactory } from './rule_status_service';
import { getGapBetweenRuns } from './utils';
import { getGapBetweenRuns, getListsClient, getExceptions, sortExceptionItems } from './utils';
import { RuleExecutorOptions } from './types';
import { searchAfterAndBulkCreate } from './search_after_bulk_create';
import { scheduleNotificationActions } from '../notifications/schedule_notification_actions';
@ -18,6 +18,9 @@ import { RuleAlertType } from '../rules/types';
import { findMlSignals } from './find_ml_signals';
import { bulkCreateMlSignals } from './bulk_create_ml_signals';
import { listMock } from '../../../../../lists/server/mocks';
import { getListClientMock } from '../../../../../lists/server/services/lists/list_client.mock';
import { getExceptionListClientMock } from '../../../../../lists/server/services/exception_lists/exception_list_client.mock';
import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
jest.mock('./rule_status_saved_objects_client');
jest.mock('./rule_status_service');
@ -84,6 +87,15 @@ describe('rules_notification_alert_type', () => {
};
(ruleStatusServiceFactory as jest.Mock).mockReturnValue(ruleStatusService);
(getGapBetweenRuns as jest.Mock).mockReturnValue(moment.duration(0));
(getListsClient as jest.Mock).mockReturnValue({
listClient: getListClientMock(),
exceptionsClient: getExceptionListClientMock(),
});
(getExceptions as jest.Mock).mockReturnValue([getExceptionListItemSchemaMock()]);
(sortExceptionItems as jest.Mock).mockReturnValue({
exceptionsWithoutValueLists: [getExceptionListItemSchemaMock()],
exceptionsWithValueLists: [],
});
(searchAfterAndBulkCreate as jest.Mock).mockClear();
(searchAfterAndBulkCreate as jest.Mock).mockResolvedValue({
success: true,

35
x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
View file

@ -15,9 +15,6 @@ import {
} from '../../../../common/constants';
import { isJobStarted, isMlRule } from '../../../../common/machine_learning/helpers';
import { SetupPlugins } from '../../../plugin';
import { ListClient } from '../../../../../lists/server';
import { getInputIndex } from './get_input_output_index';
import {
searchAfterAndBulkCreate,
@ -25,7 +22,7 @@ import {
} from './search_after_bulk_create';
import { getFilter } from './get_filter';
import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types';
import { getGapBetweenRuns, parseScheduleDates } from './utils';
import { getGapBetweenRuns, parseScheduleDates, getListsClient, getExceptions } from './utils';
import { signalParamsSchema } from './signal_params_schema';
import { siemRuleActionGroups } from './siem_rule_action_groups';
import { findMlSignals } from './find_ml_signals';
@ -38,7 +35,6 @@ import { ruleStatusServiceFactory } from './rule_status_service';
import { buildRuleMessageFactory } from './rule_messages';
import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client';
import { getNotificationResultsLink } from '../notifications/utils';
import { hasListsFeature } from '../feature_flags';
export const signalRulesAlertType = ({
logger,
@ -140,6 +136,18 @@ export const signalRulesAlertType = ({
await ruleStatusService.error(gapMessage, { gap: gapString });
}
try {
const { listClient, exceptionsClient } = await getListsClient({
services,
updatedByUser,
spaceId,
lists,
savedObjectClient: services.savedObjectsClient,
});
const exceptionItems = await getExceptions({
client: exceptionsClient,
lists: exceptionsList,
});
if (isMlRule(type)) {
if (ml == null) {
throw new Error('ML plugin unavailable during rule execution');
@ -214,18 +222,6 @@ export const signalRulesAlertType = ({
result.bulkCreateTimes.push(bulkCreateDuration);
}
} else {
let listClient: ListClient | undefined;
if (hasListsFeature()) {
if (lists == null) {
throw new Error('lists plugin unavailable during rule execution');
}
listClient = await lists.getListClient(
services.callCluster,
spaceId,
updatedByUser ?? 'elastic'
);
}
const inputIndex = await getInputIndex(services, version, index);
const esFilter = await getFilter({
type,
@ -235,13 +231,12 @@ export const signalRulesAlertType = ({
savedId,
services,
index: inputIndex,
// temporary filter out list type
lists: exceptionsList?.filter((item) => item.values_type !== 'list'),
lists: exceptionItems ?? [],
});
result = await searchAfterAndBulkCreate({
listClient,
exceptionsList,
exceptionsList: exceptionItems ?? [],
ruleParams: params,
services,
logger,

113
x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts
View file

@ -7,6 +7,12 @@
import moment from 'moment';
import sinon from 'sinon';
import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks';
import { listMock } from '../../../../../lists/server/mocks';
import { EntriesArray } from '../../../../common/detection_engine/lists_common_deps';
import * as featureFlags from '../feature_flags';
import {
generateId,
parseInterval,
@ -14,10 +20,10 @@ import {
getDriftTolerance,
getGapBetweenRuns,
errorAggregator,
getListsClient,
hasLargeValueList,
} from './utils';
import { BulkResponseErrorAggregation } from './types';
import {
sampleBulkResponse,
sampleEmptyBulkResponse,
@ -529,4 +535,107 @@ describe('utils', () => {
expect(aggregated).toEqual(expected);
});
});
describe('#getListsClient', () => {
let alertServices: AlertServicesMock;
beforeEach(() => {
alertServices = alertsMock.createAlertServices();
});
afterEach(() => {
jest.clearAllMocks();
});
test('it successfully returns list and exceptions list client', async () => {
jest.spyOn(featureFlags, 'hasListsFeature').mockReturnValue(true);
const { listClient, exceptionsClient } = await getListsClient({
services: alertServices,
savedObjectClient: alertServices.savedObjectsClient,
updatedByUser: 'some_user',
spaceId: '',
lists: listMock.createSetup(),
});
expect(listClient).toBeDefined();
expect(exceptionsClient).toBeDefined();
});
test('it returns list and exceptions client of "undefined" if lists feature flag is off', async () => {
jest.spyOn(featureFlags, 'hasListsFeature').mockReturnValue(false);
const listsClient = await getListsClient({
services: alertServices,
savedObjectClient: alertServices.savedObjectsClient,
updatedByUser: 'some_user',
spaceId: '',
lists: listMock.createSetup(),
});
expect(listsClient).toEqual({ listClient: undefined, exceptionsClient: undefined });
});
test('it throws if "lists" is undefined', async () => {
jest.spyOn(featureFlags, 'hasListsFeature').mockReturnValue(true);
await expect(() =>
getListsClient({
services: alertServices,
savedObjectClient: alertServices.savedObjectsClient,
updatedByUser: 'some_user',
spaceId: '',
lists: undefined,
})
).rejects.toThrowError('lists plugin unavailable during rule execution');
});
});
describe('#hasLargeValueList', () => {
test('it returns false if empty array', () => {
const hasLists = hasLargeValueList([]);
expect(hasLists).toBeFalsy();
});
test('it returns true if item of type EntryList exists', () => {
const entries: EntriesArray = [
{
field: 'actingProcess.file.signer',
type: 'list',
operator: 'included',
list: { id: 'some id', type: 'ip' },
},
{
field: 'file.signature.signer',
type: 'match',
operator: 'excluded',
value: 'Global Signer',
},
];
const hasLists = hasLargeValueList(entries);
expect(hasLists).toBeTruthy();
});
test('it returns false if item of type EntryList does not exist', () => {
const entries: EntriesArray = [
{
field: 'actingProcess.file.signer',
type: 'match',
operator: 'included',
value: 'Elastic, N.V.',
},
{
field: 'file.signature.signer',
type: 'match',
operator: 'excluded',
value: 'Global Signer',
},
];
const hasLists = hasLargeValueList(entries);
expect(hasLists).toBeFalsy();
});
});
});

118
x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts
View file

@ -7,9 +7,125 @@ import { createHash } from 'crypto';
import moment from 'moment';
import dateMath from '@elastic/datemath';
import { parseDuration } from '../../../../../alerts/server';
import { SavedObjectsClientContract } from '../../../../../../../src/core/server';
import { AlertServices, parseDuration } from '../../../../../alerts/server';
import { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server';
import { EntriesArray, ExceptionListItemSchema } from '../../../../../lists/common/schemas';
import { ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types/lists';
import { hasListsFeature } from '../feature_flags';
import { BulkResponse, BulkResponseErrorAggregation } from './types';
interface SortExceptionsReturn {
exceptionsWithValueLists: ExceptionListItemSchema[];
exceptionsWithoutValueLists: ExceptionListItemSchema[];
}
export const getListsClient = async ({
lists,
spaceId,
updatedByUser,
services,
savedObjectClient,
}: {
lists: ListPluginSetup | undefined;
spaceId: string;
updatedByUser: string | null;
services: AlertServices;
savedObjectClient: SavedObjectsClientContract;
}): Promise<{
listClient: ListClient | undefined;
exceptionsClient: ExceptionListClient | undefined;
}> => {
// TODO Remove check once feature is no longer behind flag
if (hasListsFeature()) {
if (lists == null) {
throw new Error('lists plugin unavailable during rule execution');
}
const listClient = await lists.getListClient(
services.callCluster,
spaceId,
updatedByUser ?? 'elastic'
);
const exceptionsClient = await lists.getExceptionListClient(
savedObjectClient,
updatedByUser ?? 'elastic'
);
return { listClient, exceptionsClient };
} else {
return { listClient: undefined, exceptionsClient: undefined };
}
};
export const hasLargeValueList = (entries: EntriesArray): boolean => {
const found = entries.filter(({ type }) => type === 'list');
return found.length > 0;
};
export const getExceptions = async ({
client,
lists,
}: {
client: ExceptionListClient | undefined;
lists: ListArrayOrUndefined;
}): Promise<ExceptionListItemSchema[] | undefined> => {
// TODO Remove check once feature is no longer behind flag
if (hasListsFeature()) {
if (client == null) {
throw new Error('lists plugin unavailable during rule execution');
}
if (lists != null) {
try {
// Gather all exception items of all exception lists linked to rule
const exceptions = await Promise.all(
lists
.map(async (list) => {
const { id, namespace_type: namespaceType } = list;
const items = await client.findExceptionListItem({
listId: id,
namespaceType,
page: 1,
perPage: 5000,
filter: undefined,
sortOrder: undefined,
sortField: undefined,
});
return items != null ? items.data : [];
})
.flat()
);
return exceptions.flat();
} catch {
return [];
}
}
}
};
export const sortExceptionItems = (exceptions: ExceptionListItemSchema[]): SortExceptionsReturn => {
return exceptions.reduce<SortExceptionsReturn>(
(acc, exception) => {
const { entries } = exception;
const { exceptionsWithValueLists, exceptionsWithoutValueLists } = acc;
if (hasLargeValueList(entries)) {
return {
exceptionsWithValueLists: [...exceptionsWithValueLists, { ...exception }],
exceptionsWithoutValueLists,
};
} else {
return {
exceptionsWithValueLists,
exceptionsWithoutValueLists: [...exceptionsWithoutValueLists, { ...exception }],
};
}
},
{ exceptionsWithValueLists: [], exceptionsWithoutValueLists: [] }
);
};
export const generateId = (
docIndex: string,
docId: string,

4
x-pack/plugins/security_solution/server/lib/detection_engine/types.ts
View file

@ -28,11 +28,11 @@ import {
Version,
MetaOrUndefined,
RuleId,
ListAndOrUndefined,
} from '../../../common/detection_engine/schemas/common/schemas';
import { CallAPIOptions } from '../../../../../../src/core/server';
import { Filter } from '../../../../../../src/plugins/data/server';
import { RuleType } from '../../../common/detection_engine/types';
import { ListArrayOrUndefined } from '../../../common/detection_engine/schemas/types';
export type PartialFilter = Partial<Filter>;
@ -62,7 +62,7 @@ export interface RuleTypeParams {
type: RuleType;
references: References;
version: Version;
exceptionsList: ListAndOrUndefined;
exceptionsList: ListArrayOrUndefined;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any