github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[SIEM] [Detections] Fixes filtering with large value lists to use "ands" between lists (#72304)

Browse source 
* wip - comment and sample json for exceptions

* promise.all for OR-ing exception items and quick-start script

* logging, added/updated json sample scripts, fixed  missing await on filter with lists

* WIP

* bug fix where two lists when 'anded' together were not filtering down result set

* undo changes from testing

* fix changes to example json and fixes missed conflict with master

* update log message and fix type errors

* change log statement and add unit test for when exception items without a value list are passed in to the filter function

* fix failing test

* update expect on one test and adds a new test to ensure anding of value lists when appearing in different exception items

* update test after rebasing with master

* properly ands exception item entries together with proper test cases

* fix test (log statement tests - need to come up with a better way to cover these)

* cleans up json examples

* rename test and use 'every' in lieu of 'some' when determining if the filter logic should execute
This commit is contained in:
Devin W. Hurley 2020-07-22 12:39:29 -04:00 • committed by GitHub
parent ba55ca9e86
commit f9cbc99a93
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 465 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item.json
View file

@ -8,7 +8,7 @@
"name": "Sample Endpoint Exception List",
"entries": [
{
"field": "host.ip",
"field": "actingProcess.file.signer",
"operator": "excluded",
"type": "exists"
},

24
x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item_with_bad_ip_list.json Normal file
View file

@ -0,0 +1,24 @@
{
"list_id": "endpoint_list",
"item_id": "endpoint_list_item_good_rock01",
"_tags": ["endpoint", "process", "malware", "os:windows"],
"tags": ["user added string for a tag", "malware"],
"type": "simple",
"description": "Don't signal when agent.name is rock01 and source.ip is in the goodguys.txt list",
"name": "Filter out good guys ip and agent.name rock01",
"comments": [],
"entries": [
{
"field": "agent.name",
"operator": "excluded",
"type": "match",
"value": ["rock01"]
},
{
"field": "source.ip",
"operator": "excluded",
"type": "list",
"list": { "id": "goodguys.txt", "type": "ip" }
}
]
}

4
x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json Normal file
View file

@ -0,0 +1,4 @@
{
"id": "hand_inserted_item_id",
"value": "127.0.0.1"
}

4
x-pack/plugins/lists/server/scripts/lists/new/list_keyword_item.json Normal file
View file

@ -0,0 +1,4 @@
{
"list_id": "keyword_list",
"value": "sh"
}

5
x-pack/plugins/lists/server/scripts/quick_start.sh Executable file
View file

@ -0,0 +1,5 @@
./hard_reset.sh && \
./post_list.sh lists/new/lists/keyword.json && \
./post_list_item.sh lists/new/list_keyword_item.json && \
./post_exception_list.sh && \
./post_exception_list_item.sh ./exception_lists/new/exception_list_item_with_list.json

15
x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
View file

@ -69,7 +69,8 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig
export const sampleDocWithSortId = (
someUuid: string = sampleIdGuid,
ip?: string
ip?: string,
destIp?: string
): SignalSourceHit => ({
_index: 'myFakeSignalIndex',
_type: 'doc',
@ -82,6 +83,9 @@ export const sampleDocWithSortId = (
source: {
ip: ip ?? '127.0.0.1',
},
destination: {
ip: destIp ?? '127.0.0.1',
},
},
sort: ['1234567891111'],
});
@ -307,7 +311,8 @@ export const repeatedSearchResultsWithSortId = (
total: number,
pageSize: number,
guids: string[],
ips?: string[]
ips?: string[],
destIps?: string[]
) => ({
took: 10,
timed_out: false,
@ -321,7 +326,11 @@ export const repeatedSearchResultsWithSortId = (
total,
max_score: 100,
hits: Array.from({ length: pageSize }).map((x, index) => ({
...sampleDocWithSortId(guids[index], ips ? ips[index] : '127.0.0.1'),
...sampleDocWithSortId(
guids[index],
ips ? ips[index] : '127.0.0.1',
destIps ? destIps[index] : '127.0.0.1'
),
})),
},
});

293
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts
View file

@ -44,6 +44,25 @@ describe('filterEventsAgainstList', () => {
expect(res.hits.hits.length).toEqual(4);
});
it('should respond with eventSearchResult if exceptionList does not contain value list exceptions', async () => {
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [getExceptionListItemSchemaMock()],
eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'7.7.7.7',
]),
buildRuleMessage,
});
expect(res.hits.hits.length).toEqual(4);
expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[0][0]).toContain(
'no exception items of type list found - returning original search result'
);
});
describe('operator_type is included', () => {
it('should respond with same list if no items match value list', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
@ -106,6 +125,280 @@ describe('filterEventsAgainstList', () => {
'ci-badguys.txt'
);
expect(res.hits.hits.length).toEqual(2);
// @ts-ignore
const ipVals = res.hits.hits.map((item) => item._source.source.ip);
expect(['3.3.3.3', '7.7.7.7']).toEqual(ipVals);
});
it('should respond with less items in the list given two exception items with entries of type list if some values match', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
const exceptionItemAgain = getExceptionListItemSchemaMock();
exceptionItemAgain.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys-again.txt',
type: 'ip',
},
},
];
// this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4']
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
{ ...getListItemResponseMock(), value: '2.2.2.2' },
{ ...getListItemResponseMock(), value: '4.4.4.4' },
]);
// this call represents an exception list with a value list containing ['6.6.6.6']
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
{ ...getListItemResponseMock(), value: '6.6.6.6' },
]);
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [exceptionItem, exceptionItemAgain],
eventSearchResult: repeatedSearchResultsWithSortId(9, 9, someGuids.slice(0, 9), [
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'4.4.4.4',
'5.5.5.5',
'6.6.6.6',
'7.7.7.7',
'8.8.8.8',
'9.9.9.9',
]),
buildRuleMessage,
});
expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
expect(res.hits.hits.length).toEqual(6);
// @ts-ignore
const ipVals = res.hits.hits.map((item) => item._source.source.ip);
expect(['1.1.1.1', '3.3.3.3', '5.5.5.5', '7.7.7.7', '8.8.8.8', '9.9.9.9']).toEqual(ipVals);
});
it('should respond with less items in the list given two exception items, each with one entry of type list if some values match', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
];
const exceptionItemAgain = getExceptionListItemSchemaMock();
exceptionItemAgain.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys-again.txt',
type: 'ip',
},
},
];
// this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4']
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
{ ...getListItemResponseMock(), value: '2.2.2.2' },
]);
// this call represents an exception list with a value list containing ['6.6.6.6']
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
{ ...getListItemResponseMock(), value: '6.6.6.6' },
]);
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [exceptionItem, exceptionItemAgain],
eventSearchResult: repeatedSearchResultsWithSortId(9, 9, someGuids.slice(0, 9), [
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'4.4.4.4',
'5.5.5.5',
'6.6.6.6',
'7.7.7.7',
'8.8.8.8',
'9.9.9.9',
]),
buildRuleMessage,
});
expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
// @ts-ignore
const ipVals = res.hits.hits.map((item) => item._source.source.ip);
expect(res.hits.hits.length).toEqual(7);
expect(['1.1.1.1', '3.3.3.3', '4.4.4.4', '5.5.5.5', '7.7.7.7', '8.8.8.8', '9.9.9.9']).toEqual(
ipVals
);
});
it('should respond with less items in the list given one exception item with two entries of type list only if source.ip and destination.ip are in the events', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
{
field: 'destination.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys-again.txt',
type: 'ip',
},
},
];
// this call represents an exception list with a value list containing ['2.2.2.2']
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
{ ...getListItemResponseMock(), value: '2.2.2.2' },
]);
// this call represents an exception list with a value list containing ['4.4.4.4']
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
{ ...getListItemResponseMock(), value: '4.4.4.4' },
]);
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [exceptionItem],
eventSearchResult: repeatedSearchResultsWithSortId(
9,
9,
someGuids.slice(0, 9),
[
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'4.4.4.4',
'5.5.5.5',
'6.6.6.6',
'2.2.2.2',
'8.8.8.8',
'9.9.9.9',
],
[
'2.2.2.2',
'2.2.2.2',
'2.2.2.2',
'2.2.2.2',
'2.2.2.2',
'2.2.2.2',
'4.4.4.4',
'2.2.2.2',
'2.2.2.2',
]
),
buildRuleMessage,
});
expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
expect(res.hits.hits.length).toEqual(8);
// @ts-ignore
const ipVals = res.hits.hits.map((item) => item._source.source.ip);
expect([
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'4.4.4.4',
'5.5.5.5',
'6.6.6.6',
'8.8.8.8',
'9.9.9.9',
]).toEqual(ipVals);
});
it('should respond with the same items in the list given one exception item with two entries of type list where the entries are included and excluded', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
{
field: 'source.ip',
operator: 'excluded',
type: 'list',
list: {
id: 'ci-badguys-again.txt',
type: 'ip',
},
},
];
// this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4']
(listClient.getListItemByValues as jest.Mock).mockResolvedValue([
{ ...getListItemResponseMock(), value: '2.2.2.2' },
]);
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [exceptionItem],
eventSearchResult: repeatedSearchResultsWithSortId(9, 9, someGuids.slice(0, 9), [
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'4.4.4.4',
'5.5.5.5',
'6.6.6.6',
'7.7.7.7',
'8.8.8.8',
'9.9.9.9',
]),
buildRuleMessage,
});
expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
expect(res.hits.hits.length).toEqual(9);
// @ts-ignore
const ipVals = res.hits.hits.map((item) => item._source.source.ip);
expect([
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'4.4.4.4',
'5.5.5.5',
'6.6.6.6',
'7.7.7.7',
'8.8.8.8',
'9.9.9.9',
]).toEqual(ipVals);
});
});
describe('operator type is excluded', () => {

177
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts
View file

@ -10,9 +10,10 @@ import { ListClient } from '../../../../../lists/server';
import { SignalSearchResponse, SearchTypes } from './types';
import { BuildRuleMessage } from './rule_messages';
import {
entriesList,
EntryList,
ExceptionListItemSchema,
entriesList,
Type,
} from '../../../../../lists/common/schemas';
import { hasLargeValueList } from '../../../../common/detection_engine/utils';
@ -24,6 +25,51 @@ interface FilterEventsAgainstList {
buildRuleMessage: BuildRuleMessage;
}
export const createSetToFilterAgainst = async ({
events,
field,
listId,
listType,
listClient,
logger,
buildRuleMessage,
}: {
events: SignalSearchResponse['hits']['hits'];
field: string;
listId: string;
listType: Type;
listClient: ListClient;
logger: Logger;
buildRuleMessage: BuildRuleMessage;
}): Promise<Set<SearchTypes>> => {
// narrow unioned type to be single
const isStringableType = (val: SearchTypes) =>
['string', 'number', 'boolean'].includes(typeof val);
const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => {
const valueField = get(field, searchResultItem._source);
if (valueField != null && isStringableType(valueField)) {
acc.add(valueField.toString());
}
return acc;
}, new Set<string>());
logger.debug(
`number of distinct values from ${field}: ${[...valuesFromSearchResultField].length}`
);
// matched will contain any list items that matched with the
// values passed in from the Set.
const matchedListItems = await listClient.getListItemByValues({
listId,
type: listType,
value: [...valuesFromSearchResultField],
});
logger.debug(`number of matched items from list with id ${listId}: ${matchedListItems.length}`);
// create a set of list values that were a hit - easier to work with
const matchedListItemsSet = new Set<SearchTypes>(matchedListItems.map((item) => item.value));
return matchedListItemsSet;
};
export const filterEventsAgainstList = async ({
listClient,
exceptionsList,
@ -32,7 +78,6 @@ export const filterEventsAgainstList = async ({
buildRuleMessage,
}: FilterEventsAgainstList): Promise<SignalSearchResponse> => {
try {
logger.debug(buildRuleMessage(`exceptionsList: ${JSON.stringify(exceptionsList, null, 2)}`));
if (exceptionsList == null || exceptionsList.length === 0) {
logger.debug(buildRuleMessage('about to return original search result'));
return eventSearchResult;
@ -51,87 +96,97 @@ export const filterEventsAgainstList = async ({
);
if (exceptionItemsWithLargeValueLists.length === 0) {
logger.debug(buildRuleMessage('about to return original search result'));
logger.debug(
buildRuleMessage('no exception items of type list found - returning original search result')
);
return eventSearchResult;
}
// narrow unioned type to be single
const isStringableType = (val: SearchTypes) =>
['string', 'number', 'boolean'].includes(typeof val);
// grab the signals with values found in the given exception lists.
const filteredHitsPromises = exceptionItemsWithLargeValueLists.map(
async (exceptionItem: ExceptionListItemSchema) => {
const { entries } = exceptionItem;
const valueListExceptionItems = exceptionsList.filter((listItem: ExceptionListItemSchema) => {
return listItem.entries.every((entry) => entriesList.is(entry));
});
const filteredHitsEntries = entries
.filter((t): t is EntryList => entriesList.is(t))
.map(async (entry) => {
// now that we have all the exception items which are value lists (whether single entry or have multiple entries)
const res = await valueListExceptionItems.reduce<Promise<SignalSearchResponse['hits']['hits']>>(
async (
filteredAccum: Promise<SignalSearchResponse['hits']['hits']>,
exceptionItem: ExceptionListItemSchema
) => {
// 1. acquire the values from the specified fields to check
// e.g. if the value list is checking against source.ip, gather
// all the values for source.ip from the search response events.
// 2. search against the value list with the values found in the search result
// and see if there are any matches. For every match, add that value to a set
// that represents the "matched" values
// 3. filter the search result against the set from step 2 using the
// given operator (included vs excluded).
// acquire the list values we are checking for in the field.
const filtered = await filteredAccum;
const typedEntries = exceptionItem.entries.filter((entry): entry is EntryList =>
entriesList.is(entry)
);
const fieldAndSetTuples = await Promise.all(
typedEntries.map(async (entry) => {
const { list, field, operator } = entry;
const { id, type } = list;
// acquire the list values we are checking for.
const valuesOfGivenType = eventSearchResult.hits.hits.reduce(
(acc, searchResultItem) => {
const valueField = get(field, searchResultItem._source);
if (valueField != null && isStringableType(valueField)) {
acc.add(valueField.toString());
}
return acc;
},
new Set<string>()
);
// matched will contain any list items that matched with the
// values passed in from the Set.
const matchedListItems = await listClient.getListItemByValues({
const matchedSet = await createSetToFilterAgainst({
events: filtered,
field,
listId: id,
type,
value: [...valuesOfGivenType],
listType: type,
listClient,
logger,
buildRuleMessage,
});
// create a set of list values that were a hit - easier to work with
const matchedListItemsSet = new Set<SearchTypes>(
matchedListItems.map((item) => item.value)
);
return Promise.resolve({ field, operator, matchedSet });
})
);
// do a single search after with these values.
// painless script to do nested query in elasticsearch
// filter out the search results that match with the values found in the list.
const filteredEvents = eventSearchResult.hits.hits.filter((item) => {
const eventItem = get(entry.field, item._source);
if (operator === 'included') {
if (eventItem != null) {
return !matchedListItemsSet.has(eventItem);
}
} else if (operator === 'excluded') {
if (eventItem != null) {
return matchedListItemsSet.has(eventItem);
}
// check if for each tuple, the entry is not in both for when two value list entries exist.
// need to re-write this as a reduce.
const filteredEvents = filtered.filter((item) => {
const vals = fieldAndSetTuples.map((tuple) => {
const eventItem = get(tuple.field, item._source);
if (tuple.operator === 'included') {
// only create a signal if the event is not in the value list
if (eventItem != null) {
return !tuple.matchedSet.has(eventItem);
}
return false;
});
const diff = eventSearchResult.hits.hits.length - filteredEvents.length;
logger.debug(buildRuleMessage(`Lists filtered out ${diff} events`));
return filteredEvents;
return true;
} else if (tuple.operator === 'excluded') {
// only create a signal if the event is in the value list
if (eventItem != null) {
return tuple.matchedSet.has(eventItem);
}
return true;
}
return false;
});
return (await Promise.all(filteredHitsEntries)).flat();
}
return vals.some((value) => value);
});
const diff = eventSearchResult.hits.hits.length - filteredEvents.length;
logger.debug(
buildRuleMessage(`Exception with id ${exceptionItem.id} filtered out ${diff} events`)
);
const toReturn = filteredEvents;
return toReturn;
},
Promise.resolve<SignalSearchResponse['hits']['hits']>(eventSearchResult.hits.hits)
);
const filteredHits = await Promise.all(filteredHitsPromises);
const toReturn: SignalSearchResponse = {
took: eventSearchResult.took,
timed_out: eventSearchResult.timed_out,
_shards: eventSearchResult._shards,
hits: {
total: filteredHits.length,
total: res.length,
max_score: eventSearchResult.hits.max_score,
hits: filteredHits.flat(),
hits: res,
},
};
return toReturn;
} catch (exc) {
throw new Error(`Failed to query lists index. Reason: ${exc.message}`);

4
x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts
View file

@ -475,7 +475,7 @@ describe('searchAfterAndBulkCreate', () => {
expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
// I don't like testing log statements since logs change but this is the best
// way I can think of to ensure this section is getting hit with this test case.
expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[7][0]).toContain(
expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[8][0]).toContain(
'sortIds was empty on searchResult'
);
});
@ -558,7 +558,7 @@ describe('searchAfterAndBulkCreate', () => {
expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
// I don't like testing log statements since logs change but this is the best
// way I can think of to ensure this section is getting hit with this test case.
expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[12][0]).toContain(
expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[15][0]).toContain(
'sortIds was empty on filteredEvents'
);
});

5
x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts
View file

@ -83,6 +83,7 @@ export const singleBulkCreate = async ({
throttle,
}: SingleBulkCreateParams): Promise<SingleBulkCreateResponse> => {
filteredEvents.hits.hits = filterDuplicateRules(id, filteredEvents);
logger.debug(`about to bulk create ${filteredEvents.hits.hits.length} events`);
if (filteredEvents.hits.hits.length === 0) {
logger.debug(`all events were duplicates`);
return { success: true, createdItemsCount: 0 };
@ -135,6 +136,8 @@ export const singleBulkCreate = async ({
logger.debug(`took property says bulk took: ${response.took} milliseconds`);
if (response.errors) {
const duplicateSignalsCount = countBy(response.items, 'create.status')['409'];
logger.debug(`ignored ${duplicateSignalsCount} duplicate signals`);
const errorCountByMessage = errorAggregator(response, [409]);
if (!isEmpty(errorCountByMessage)) {
logger.error(
@ -144,6 +147,6 @@ export const singleBulkCreate = async ({
}
const createdItemsCount = countBy(response.items, 'create.status')['201'] ?? 0;
logger.debug(`bulk created ${createdItemsCount} signals`);
return { success: true, bulkCreateDuration: makeFloatString(end - start), createdItemsCount };
};