github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add events-first (reverse) search for IM rule (#127428)

Browse source 
* WIP

* Add tets and refactoring

* Add abstraction to run threat im rule

* Add per page to search threat indicators

* Fix tests and linting

* fix tests

* Add integrations tests

* Fix IP

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Khristinin Nikita 2022-03-24 09:45:19 +01:00 • committed by GitHub
parent 0695df6497
commit 8f6322596c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 1238 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

66
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts
View file

@ -36,6 +36,7 @@ describe('build_threat_mapping_filter', () => {
threatMapping,
threatList,
chunkSize: 1025,
entryKey: 'value',
})
).toThrow('chunk sizes cannot exceed 1024 in size');
});
@ -44,28 +45,28 @@ describe('build_threat_mapping_filter', () => {
const threatMapping = getThreatMappingMock();
const threatList = getThreatListSearchResponseMock().hits.hits;
expect(() =>
buildThreatMappingFilter({ threatMapping, threatList, chunkSize: 1023 })
buildThreatMappingFilter({ threatMapping, threatList, chunkSize: 1023, entryKey: 'value' })
).not.toThrow();
});
test('it should create the correct entries when using the default mocks', () => {
const threatMapping = getThreatMappingMock();
const threatList = getThreatListSearchResponseMock().hits.hits;
const filter = buildThreatMappingFilter({ threatMapping, threatList });
const filter = buildThreatMappingFilter({ threatMapping, threatList, entryKey: 'value' });
expect(filter).toEqual(getThreatMappingFilterMock());
});
test('it should not mutate the original threatMapping', () => {
const threatMapping = getThreatMappingMock();
const threatList = getThreatListSearchResponseMock().hits.hits;
buildThreatMappingFilter({ threatMapping, threatList });
buildThreatMappingFilter({ threatMapping, threatList, entryKey: 'value' });
expect(threatMapping).toEqual(getThreatMappingMock());
});
test('it should not mutate the original threatListItem', () => {
const threatMapping = getThreatMappingMock();
const threatList = getThreatListSearchResponseMock().hits.hits;
buildThreatMappingFilter({ threatMapping, threatList });
buildThreatMappingFilter({ threatMapping, threatList, entryKey: 'value' });
expect(threatList).toEqual(getThreatListSearchResponseMock().hits.hits);
});
});
@ -75,7 +76,7 @@ describe('build_threat_mapping_filter', () => {
const threatMapping = getThreatMappingMock();
const threatListItem = getThreatListSearchResponseMock().hits.hits[0];
const item = filterThreatMapping({ threatMapping, threatListItem });
const item = filterThreatMapping({ threatMapping, threatListItem, entryKey: 'value' });
const expected = getFilterThreatMapping();
expect(item).toEqual(expected);
});
@ -84,7 +85,11 @@ describe('build_threat_mapping_filter', () => {
const [firstElement] = getThreatMappingMock(); // get only the first element
const threatListItem = getThreatListSearchResponseMock().hits.hits[0];
const item = filterThreatMapping({ threatMapping: [firstElement], threatListItem });
const item = filterThreatMapping({
threatMapping: [firstElement],
threatListItem,
entryKey: 'value',
});
const [firstElementFilter] = getFilterThreatMapping(); // get only the first element to compare
expect(item).toEqual([firstElementFilter]);
});
@ -96,6 +101,7 @@ describe('build_threat_mapping_filter', () => {
filterThreatMapping({
threatMapping,
threatListItem,
entryKey: 'value',
});
expect(threatMapping).toEqual(getThreatMappingMock());
});
@ -107,6 +113,7 @@ describe('build_threat_mapping_filter', () => {
filterThreatMapping({
threatMapping,
threatListItem,
entryKey: 'value',
});
expect(threatListItem).toEqual(getThreatListSearchResponseMock().hits.hits[0]);
});
@ -142,6 +149,7 @@ describe('build_threat_mapping_filter', () => {
'host.name': ['host-1'],
},
}),
entryKey: 'value',
});
expect(item).toEqual([]);
});
@ -185,6 +193,7 @@ describe('build_threat_mapping_filter', () => {
'host.name': ['host-1'],
},
}),
entryKey: 'value',
});
expect(item).toEqual([
{
@ -204,7 +213,11 @@ describe('build_threat_mapping_filter', () => {
test('it should return two clauses given a single entry', () => {
const [{ entries: threatMappingEntries }] = getThreatMappingMock(); // get the first element
const threatListItem = getThreatListSearchResponseMock().hits.hits[0];
const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem });
const innerClause = createInnerAndClauses({
threatMappingEntries,
threatListItem,
entryKey: 'value',
});
const {
bool: {
should: [
@ -219,7 +232,11 @@ describe('build_threat_mapping_filter', () => {
test('it should return an empty array given an empty array', () => {
const threatListItem = getThreatListItemMock();
const innerClause = createInnerAndClauses({ threatMappingEntries: [], threatListItem });
const innerClause = createInnerAndClauses({
threatMappingEntries: [],
threatListItem,
entryKey: 'value',
});
expect(innerClause).toEqual([]);
});
@ -234,7 +251,11 @@ describe('build_threat_mapping_filter', () => {
},
];
const threatListItem = getThreatListSearchResponseMock().hits.hits[0];
const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem });
const innerClause = createInnerAndClauses({
threatMappingEntries,
threatListItem,
entryKey: 'value',
});
const {
bool: {
should: [
@ -263,7 +284,11 @@ describe('build_threat_mapping_filter', () => {
},
];
const threatListItem = getThreatListSearchResponseMock().hits.hits[0];
const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem });
const innerClause = createInnerAndClauses({
threatMappingEntries,
threatListItem,
entryKey: 'value',
});
const {
bool: {
should: [
@ -290,7 +315,11 @@ describe('build_threat_mapping_filter', () => {
},
];
const threatListItem = getThreatListSearchResponseMock().hits.hits[0];
const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem });
const innerClause = createInnerAndClauses({
threatMappingEntries,
threatListItem,
entryKey: 'value',
});
expect(innerClause).toEqual([]);
});
});
@ -299,7 +328,7 @@ describe('build_threat_mapping_filter', () => {
test('it should return all clauses given the entries', () => {
const threatMapping = getThreatMappingMock();
const threatListItem = getThreatListSearchResponseMock().hits.hits[0];
const innerClause = createAndOrClauses({ threatMapping, threatListItem });
const innerClause = createAndOrClauses({ threatMapping, threatListItem, entryKey: 'value' });
expect(innerClause).toEqual(getThreatMappingFilterShouldMock());
});
@ -310,13 +339,17 @@ describe('build_threat_mapping_filter', () => {
...getThreatListSearchResponseMock().hits.hits[0]._source,
foo: 'bar',
};
const innerClause = createAndOrClauses({ threatMapping, threatListItem });
const innerClause = createAndOrClauses({ threatMapping, threatListItem, entryKey: 'value' });
expect(innerClause).toEqual(getThreatMappingFilterShouldMock());
});
test('it should return an empty boolean given an empty array', () => {
const threatListItem = getThreatListSearchResponseMock().hits.hits[0];
const innerClause = createAndOrClauses({ threatMapping: [], threatListItem });
const innerClause = createAndOrClauses({
threatMapping: [],
threatListItem,
entryKey: 'value',
});
expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } });
});
@ -325,6 +358,7 @@ describe('build_threat_mapping_filter', () => {
const innerClause = createAndOrClauses({
threatMapping,
threatListItem: getThreatListItemMock({ _source: {}, fields: {} }),
entryKey: 'value',
});
expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } });
});
@ -338,6 +372,7 @@ describe('build_threat_mapping_filter', () => {
threatMapping,
threatList,
chunkSize: 1024,
entryKey: 'value',
});
const expected: BooleanFilter = {
bool: { should: [getThreatMappingFilterShouldMock()], minimum_should_match: 1 },
@ -352,6 +387,7 @@ describe('build_threat_mapping_filter', () => {
threatMapping,
threatList,
chunkSize: 1024,
entryKey: 'value',
});
const expected: BooleanFilter = {
bool: { should: [], minimum_should_match: 1 },
@ -365,6 +401,7 @@ describe('build_threat_mapping_filter', () => {
threatMapping: [],
threatList,
chunkSize: 1024,
entryKey: 'value',
});
const expected: BooleanFilter = {
bool: { should: [], minimum_should_match: 1 },
@ -399,6 +436,7 @@ describe('build_threat_mapping_filter', () => {
threatMapping,
threatList,
chunkSize: 1024,
entryKey: 'value',
});
const expected: BooleanFilter = {
bool: { should: [getThreatMappingFilterShouldMock()], minimum_should_match: 1 },

15
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts
View file

@ -25,6 +25,7 @@ export const buildThreatMappingFilter = ({
threatMapping,
threatList,
chunkSize,
entryKey = 'value',
}: BuildThreatMappingFilterOptions): Filter => {
const computedChunkSize = chunkSize ?? MAX_CHUNK_SIZE;
if (computedChunkSize > 1024) {
@ -34,6 +35,7 @@ export const buildThreatMappingFilter = ({
threatMapping,
threatList,
chunkSize: computedChunkSize,
entryKey,
});
const filterChunk: Filter = {
meta: {
@ -52,11 +54,12 @@ export const buildThreatMappingFilter = ({
export const filterThreatMapping = ({
threatMapping,
threatListItem,
entryKey,
}: FilterThreatMappingOptions): ThreatMapping =>
threatMapping
.map((threatMap) => {
const atLeastOneItemMissingInThreatList = threatMap.entries.some((entry) => {
const itemValue = get(entry.value, threatListItem.fields);
const itemValue = get(entry[entryKey], threatListItem.fields);
return itemValue == null || itemValue.length !== 1;
});
if (atLeastOneItemMissingInThreatList) {
@ -70,9 +73,10 @@ export const filterThreatMapping = ({
export const createInnerAndClauses = ({
threatMappingEntries,
threatListItem,
entryKey,
}: CreateInnerAndClausesOptions): BooleanFilter[] => {
return threatMappingEntries.reduce<BooleanFilter[]>((accum, threatMappingEntry) => {
const value = get(threatMappingEntry.value, threatListItem.fields);
const value = get(threatMappingEntry[entryKey], threatListItem.fields);
if (value != null && value.length === 1) {
// These values could be potentially 10k+ large so mutating the array intentionally
accum.push({
@ -80,7 +84,7 @@ export const createInnerAndClauses = ({
should: [
{
match: {
[threatMappingEntry.field]: {
[threatMappingEntry[entryKey === 'field' ? 'value' : 'field']]: {
query: value[0],
_name: encodeThreatMatchNamedQuery({
id: threatListItem._id,
@ -103,11 +107,13 @@ export const createInnerAndClauses = ({
export const createAndOrClauses = ({
threatMapping,
threatListItem,
entryKey,
}: CreateAndOrClausesOptions): BooleanFilter => {
const should = threatMapping.reduce<unknown[]>((accum, threatMap) => {
const innerAndClauses = createInnerAndClauses({
threatMappingEntries: threatMap.entries,
threatListItem,
entryKey,
});
if (innerAndClauses.length !== 0) {
// These values could be potentially 10k+ large so mutating the array intentionally
@ -124,15 +130,18 @@ export const buildEntriesMappingFilter = ({
threatMapping,
threatList,
chunkSize,
entryKey,
}: BuildEntriesMappingFilterOptions): BooleanFilter => {
const combinedShould = threatList.reduce<BooleanFilter[]>((accum, threatListSearchItem) => {
const filteredEntries = filterThreatMapping({
threatMapping,
threatListItem: threatListSearchItem,
entryKey,
});
const queryWithAndOrClause = createAndOrClauses({
threatMapping: filteredEntries,
threatListItem: threatListSearchItem,
entryKey,
});
if (queryWithAndOrClause.bool.should.length !== 0) {
// These values can be 10k+ large, so using a push here for performance

155
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts Normal file
View file

@ -0,0 +1,155 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { buildThreatMappingFilter } from './build_threat_mapping_filter';
import { getFilter } from '../get_filter';
import { searchAfterAndBulkCreate } from '../search_after_bulk_create';
import { buildReasonMessageForThreatMatchAlert } from '../reason_formatters';
import { CreateEventSignalOptions } from './types';
import { SearchAfterAndBulkCreateReturnType, SignalSearchResponse } from '../types';
import { getAllThreatListHits } from './get_threat_list';
import {
enrichSignalThreatMatches,
getSignalMatchesFromThreatList,
} from './enrich_signal_threat_matches';
export const createEventSignal = async ({
alertId,
buildRuleMessage,
bulkCreate,
completeRule,
currentResult,
currentEventList,
eventsTelemetry,
exceptionItems,
filters,
inputIndex,
language,
listClient,
logger,
outputIndex,
query,
savedId,
searchAfterSize,
services,
threatMapping,
tuple,
type,
wrapHits,
threatQuery,
threatFilters,
threatLanguage,
threatIndex,
threatListConfig,
threatIndicatorPath,
perPage,
}: CreateEventSignalOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
const threatFilter = buildThreatMappingFilter({
threatMapping,
threatList: currentEventList,
entryKey: 'field',
});
if (!threatFilter.query || threatFilter.query?.bool.should.length === 0) {
// empty event list and we do not want to return everything as being
// a hit so opt to return the existing result.
logger.debug(
buildRuleMessage(
'Indicator items are empty after filtering for missing data, returning without attempting a match'
)
);
return currentResult;
} else {
const threatListHits = await getAllThreatListHits({
esClient: services.scopedClusterClient.asCurrentUser,
exceptionItems,
threatFilters: [...threatFilters, threatFilter],
query: threatQuery,
language: threatLanguage,
index: threatIndex,
logger,
buildRuleMessage,
threatListConfig: {
_source: [`${threatIndicatorPath}.*`, 'threat.feed.*'],
fields: undefined,
},
perPage,
});
const signalMatches = getSignalMatchesFromThreatList(threatListHits);
const ids = signalMatches.map((item) => item.signalId);
const indexFilter = {
query: {
bool: {
filter: {
ids: { values: ids },
},
},
},
};
const esFilter = await getFilter({
type,
filters: [...filters, indexFilter],
language,
query,
savedId,
services,
index: inputIndex,
lists: exceptionItems,
});
logger.debug(
buildRuleMessage(
`${ids?.length} matched signals found from ${threatListHits.length} indicators`
)
);
const threatEnrichment = (signals: SignalSearchResponse): Promise<SignalSearchResponse> =>
enrichSignalThreatMatches(
signals,
() => Promise.resolve(threatListHits),
threatIndicatorPath,
signalMatches
);
const result = await searchAfterAndBulkCreate({
buildReasonMessage: buildReasonMessageForThreatMatchAlert,
buildRuleMessage,
bulkCreate,
completeRule,
enrichment: threatEnrichment,
eventsTelemetry,
exceptionsList: exceptionItems,
filter: esFilter,
id: alertId,
inputIndexPattern: inputIndex,
listClient,
logger,
pageSize: searchAfterSize,
services,
signalsIndex: outputIndex,
sortOrder: 'desc',
trackTotalHits: false,
tuple,
wrapHits,
});
logger.debug(
buildRuleMessage(
`${
threatFilter.query?.bool.should.length
} items have completed match checks and the total times to search were ${
result.searchAfterTimes.length !== 0 ? result.searchAfterTimes : '(unknown) '
}ms`
)
);
return result;
}
};

1
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts
View file

@ -41,6 +41,7 @@ export const createThreatSignal = async ({
const threatFilter = buildThreatMappingFilter({
threatMapping,
threatList: currentThreatList,
entryKey: 'value',
});
if (!threatFilter.query || threatFilter.query?.bool.should.length === 0) {

186
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts
View file

@ -7,13 +7,17 @@
import chunk from 'lodash/fp/chunk';
import { getThreatList, getThreatListCount } from './get_threat_list';
import { CreateThreatSignalsOptions } from './types';
import {
CreateThreatSignalsOptions,
CreateSignalInterface,
GetDocumentListInterface,
} from './types';
import { createThreatSignal } from './create_threat_signal';
import { createEventSignal } from './create_event_signal';
import { SearchAfterAndBulkCreateReturnType } from '../types';
import { buildExecutionIntervalValidator, combineConcurrentResults } from './utils';
import { buildThreatEnrichment } from './build_threat_enrichment';
import { getEventCount } from './get_event_count';
import { getEventCount, getEventList } from './get_event_count';
import { getMappingFilters } from './get_mapping_filters';
export const createThreatSignals = async ({
@ -85,7 +89,7 @@ export const createThreatSignals = async ({
return results;
}
let threatListCount = await getThreatListCount({
const threatListCount = await getThreatListCount({
esClient: services.scopedClusterClient.asCurrentUser,
exceptionItems,
threatFilters: allThreatFilters,
@ -101,20 +105,6 @@ export const createThreatSignals = async ({
_source: false,
};
let threatList = await getThreatList({
esClient: services.scopedClusterClient.asCurrentUser,
exceptionItems,
threatFilters: allThreatFilters,
query: threatQuery,
language: threatLanguage,
index: threatIndex,
searchAfter: undefined,
logger,
buildRuleMessage,
perPage,
threatListConfig,
});
const threatEnrichment = buildThreatEnrichment({
buildRuleMessage,
exceptionItems,
@ -127,12 +117,124 @@ export const createThreatSignals = async ({
threatQuery,
});
while (threatList.hits.hits.length !== 0) {
verifyExecutionCanProceed();
const chunks = chunk(itemsPerSearch, threatList.hits.hits);
logger.debug(buildRuleMessage(`${chunks.length} concurrent indicator searches are starting.`));
const concurrentSearchesPerformed = chunks.map<Promise<SearchAfterAndBulkCreateReturnType>>(
(slicedChunk) =>
const createSignals = async ({
getDocumentList,
createSignal,
totalDocumentCount,
}: {
getDocumentList: GetDocumentListInterface;
createSignal: CreateSignalInterface;
totalDocumentCount: number;
}) => {
let list = await getDocumentList({ searchAfter: undefined });
let documentCount = totalDocumentCount;
while (list.hits.hits.length !== 0) {
verifyExecutionCanProceed();
const chunks = chunk(itemsPerSearch, list.hits.hits);
logger.debug(
buildRuleMessage(`${chunks.length} concurrent indicator searches are starting.`)
);
const concurrentSearchesPerformed =
chunks.map<Promise<SearchAfterAndBulkCreateReturnType>>(createSignal);
const searchesPerformed = await Promise.all(concurrentSearchesPerformed);
results = combineConcurrentResults(results, searchesPerformed);
documentCount -= list.hits.hits.length;
logger.debug(
buildRuleMessage(
`Concurrent indicator match searches completed with ${results.createdSignalsCount} signals found`,
`search times of ${results.searchAfterTimes}ms,`,
`bulk create times ${results.bulkCreateTimes}ms,`,
`all successes are ${results.success}`
)
);
if (results.createdSignalsCount >= params.maxSignals) {
logger.debug(
buildRuleMessage(
`Indicator match has reached its max signals count ${params.maxSignals}. Additional documents not checked are ${documentCount}`
)
);
break;
}
logger.debug(buildRuleMessage(`Documents items left to check are ${documentCount}`));
list = await getDocumentList({
searchAfter: list.hits.hits[list.hits.hits.length - 1].sort,
});
}
};
if (eventCount < threatListCount) {
await createSignals({
totalDocumentCount: eventCount,
getDocumentList: async ({ searchAfter }) =>
getEventList({
services,
exceptionItems,
filters: allEventFilters,
query,
language,
index: inputIndex,
searchAfter,
logger,
buildRuleMessage,
perPage,
tuple,
}),
createSignal: (slicedChunk) =>
createEventSignal({
alertId,
buildRuleMessage,
bulkCreate,
completeRule,
currentResult: results,
currentEventList: slicedChunk,
eventsTelemetry,
exceptionItems,
filters: allEventFilters,
inputIndex,
language,
listClient,
logger,
outputIndex,
query,
savedId,
searchAfterSize,
services,
threatEnrichment,
threatMapping,
tuple,
type,
wrapHits,
threatQuery,
threatFilters: allThreatFilters,
threatLanguage,
threatIndex,
threatListConfig,
threatIndicatorPath,
perPage,
}),
});
} else {
await createSignals({
totalDocumentCount: threatListCount,
getDocumentList: async ({ searchAfter }) =>
getThreatList({
esClient: services.scopedClusterClient.asCurrentUser,
exceptionItems,
threatFilters: allThreatFilters,
query: threatQuery,
language: threatLanguage,
index: threatIndex,
searchAfter,
logger,
buildRuleMessage,
perPage,
threatListConfig,
}),
createSignal: (slicedChunk) =>
createThreatSignal({
alertId,
buildRuleMessage,
@ -157,41 +259,7 @@ export const createThreatSignals = async ({
tuple,
type,
wrapHits,
})
);
const searchesPerformed = await Promise.all(concurrentSearchesPerformed);
results = combineConcurrentResults(results, searchesPerformed);
threatListCount -= threatList.hits.hits.length;
logger.debug(
buildRuleMessage(
`Concurrent indicator match searches completed with ${results.createdSignalsCount} signals found`,
`search times of ${results.searchAfterTimes}ms,`,
`bulk create times ${results.bulkCreateTimes}ms,`,
`all successes are ${results.success}`
)
);
if (results.createdSignalsCount >= params.maxSignals) {
logger.debug(
buildRuleMessage(
`Indicator match has reached its max signals count ${params.maxSignals}. Additional indicator items not checked are ${threatListCount}`
)
);
break;
}
logger.debug(buildRuleMessage(`Indicator items left to check are ${threatListCount}`));
threatList = await getThreatList({
esClient: services.scopedClusterClient.asCurrentUser,
exceptionItems,
query: threatQuery,
language: threatLanguage,
threatFilters: allThreatFilters,
index: threatIndex,
searchAfter: threatList.hits.hits[threatList.hits.hits.length - 1].sort,
buildRuleMessage,
logger,
perPage,
threatListConfig,
}),
});
}

105
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts
View file

@ -14,6 +14,7 @@ import {
buildEnrichments,
enrichSignalThreatMatches,
groupAndMergeSignalMatches,
getSignalMatchesFromThreatList,
} from './enrich_signal_threat_matches';
import {
getNamedQueryMock,
@ -793,3 +794,107 @@ describe('enrichSignalThreatMatches', () => {
]);
});
});
describe('getSignalMatchesFromThreatList', () => {
it('return empty array if there no threat indicators', () => {
const signalMatches = getSignalMatchesFromThreatList();
expect(signalMatches).toEqual([]);
});
it("return empty array if threat indicators doesn't have matched query", () => {
const signalMatches = getSignalMatchesFromThreatList([getThreatListItemMock()]);
expect(signalMatches).toEqual([]);
});
it('return signal mathces from threat indicators', () => {
const signalMatches = getSignalMatchesFromThreatList([
getThreatListItemMock({
_id: 'threatId',
matched_queries: [
encodeThreatMatchNamedQuery(
getNamedQueryMock({
id: 'signalId1',
index: 'source_index',
value: 'threat.indicator.domain',
field: 'event.domain',
})
),
encodeThreatMatchNamedQuery(
getNamedQueryMock({
id: 'signalId2',
index: 'source_index',
value: 'threat.indicator.domain',
field: 'event.domain',
})
),
],
}),
]);
const queries = [
{
field: 'event.domain',
value: 'threat.indicator.domain',
index: 'threat_index',
id: 'threatId',
},
];
expect(signalMatches).toEqual([
{
signalId: 'signalId1',
queries,
},
{
signalId: 'signalId2',
queries,
},
]);
});
it('merge signal mathces if different threat indicators matched the same signal', () => {
const matchedQuery = [
encodeThreatMatchNamedQuery(
getNamedQueryMock({
id: 'signalId',
index: 'source_index',
value: 'threat.indicator.domain',
field: 'event.domain',
})
),
];
const signalMatches = getSignalMatchesFromThreatList([
getThreatListItemMock({
_id: 'threatId1',
matched_queries: matchedQuery,
}),
getThreatListItemMock({
_id: 'threatId2',
matched_queries: matchedQuery,
}),
]);
const query = {
field: 'event.domain',
value: 'threat.indicator.domain',
index: 'threat_index',
id: 'threatId',
};
expect(signalMatches).toEqual([
{
signalId: 'signalId',
queries: [
{
...query,
id: 'threatId1',
},
{
...query,
id: 'threatId2',
},
],
},
]);
});
});

59
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts
View file

@ -14,9 +14,43 @@ import type {
ThreatEnrichment,
ThreatListItem,
ThreatMatchNamedQuery,
SignalMatch,
} from './types';
import { extractNamedQueries } from './utils';
export const getSignalMatchesFromThreatList = (
threatList: ThreatListItem[] = []
): SignalMatch[] => {
const signalMap: { [key: string]: ThreatMatchNamedQuery[] } = {};
threatList.forEach((threatHit) =>
extractNamedQueries(threatHit).forEach((item) => {
const signalId = item.id;
if (!signalId) {
return;
}
if (!signalMap[signalId]) {
signalMap[signalId] = [];
}
signalMap[signalId].push({
id: threatHit._id,
index: threatHit._index,
field: item.field,
value: item.value,
});
})
);
const signalMatches = Object.entries(signalMap).map(([key, value]) => ({
signalId: key,
queries: value,
}));
return signalMatches;
};
const getSignalId = (signal: SignalSourceHit): string => signal._id;
export const groupAndMergeSignalMatches = (signalHits: SignalSourceHit[]): SignalSourceHit[] => {
@ -77,7 +111,8 @@ export const buildEnrichments = ({
export const enrichSignalThreatMatches = async (
signals: SignalSearchResponse,
getMatchedThreats: GetMatchedThreats,
indicatorPath: string
indicatorPath: string,
signalMatchesArg?: SignalMatch[]
): Promise<SignalSearchResponse> => {
const signalHits = signals.hits.hits;
if (signalHits.length === 0) {
@ -85,13 +120,27 @@ export const enrichSignalThreatMatches = async (
}
const uniqueHits = groupAndMergeSignalMatches(signalHits);
const signalMatches = uniqueHits.map((signalHit) => extractNamedQueries(signalHit));
const matchedThreatIds = [...new Set(signalMatches.flat().map(({ id }) => id))];
const signalMatches: SignalMatch[] = signalMatchesArg
? signalMatchesArg
: uniqueHits.map((signalHit) => ({
signalId: signalHit._id,
queries: extractNamedQueries(signalHit),
}));
const matchedThreatIds = [
...new Set(
signalMatches
.map((signalMatch) => signalMatch.queries)
.flat()
.map(({ id }) => id)
),
];
const matchedThreats = await getMatchedThreats(matchedThreatIds);
const enrichmentsWithoutAtomic = signalMatches.map((queries) =>
const enrichmentsWithoutAtomic = signalMatches.map((signalMatch) =>
buildEnrichments({
indicatorPath,
queries,
queries: signalMatch.queries,
threats: matchedThreats,
})
);

54
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_event_count.ts
View file

@ -5,10 +5,62 @@
* 2.0.
*/
import { EventCountOptions } from './types';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { EventCountOptions, EventsOptions, EventDoc } from './types';
import { getQueryFilter } from '../../../../../common/detection_engine/get_query_filter';
import { singleSearchAfter } from '../../signals/single_search_after';
import { buildEventsSearchQuery } from '../build_events_query';
export const MAX_PER_PAGE = 9000;
export const getEventList = async ({
services,
query,
language,
index,
perPage,
searchAfter,
exceptionItems,
filters,
buildRuleMessage,
logger,
tuple,
timestampOverride,
}: EventsOptions): Promise<estypes.SearchResponse<EventDoc>> => {
const calculatedPerPage = perPage ?? MAX_PER_PAGE;
if (calculatedPerPage > 10000) {
throw new TypeError('perPage cannot exceed the size of 10000');
}
logger.debug(
buildRuleMessage(
`Querying the events items from the index: "${index}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items`
)
);
const filter = getQueryFilter(query, language ?? 'kuery', filters, index, exceptionItems);
const { searchResult } = await singleSearchAfter({
buildRuleMessage,
searchAfterSortIds: searchAfter,
index,
from: tuple.from.toISOString(),
to: tuple.to.toISOString(),
services,
logger,
filter,
pageSize: Math.ceil(Math.min(tuple.maxSignals, calculatedPerPage)),
timestampOverride,
sortOrder: 'desc',
trackTotalHits: false,
});
logger.debug(
buildRuleMessage(`Retrieved events items of size: ${searchResult.hits.hits.length}`)
);
return searchResult;
};
export const getEventCount = async ({
esClient,
query,

26
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts
View file

@ -7,7 +7,12 @@
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { getQueryFilter } from '../../../../../common/detection_engine/get_query_filter';
import { GetThreatListOptions, ThreatListCountOptions, ThreatListDoc } from './types';
import {
GetThreatListOptions,
ThreatListCountOptions,
ThreatListDoc,
ThreatListItem,
} from './types';
/**
* This should not exceed 10000 (10k)
@ -89,3 +94,22 @@ export const getThreatListCount = async ({
});
return response.count;
};
export const getAllThreatListHits = async (
params: Omit<GetThreatListOptions, 'searchAfter'>
): Promise<ThreatListItem[]> => {
let allThreatListHits: ThreatListItem[] = [];
let threatList = await getThreatList({ ...params, searchAfter: undefined });
allThreatListHits = allThreatListHits.concat(threatList.hits.hits);
while (threatList.hits.hits.length !== 0) {
threatList = await getThreatList({
...params,
searchAfter: threatList.hits.hits[threatList.hits.hits.length - 1].sort,
});
allThreatListHits = allThreatListHits.concat(threatList.hits.hits);
}
return allThreatListHits;
};

72
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts
View file

@ -94,31 +94,70 @@ export interface CreateThreatSignalOptions {
wrapHits: WrapHits;
}
export interface CreateEventSignalOptions {
alertId: string;
buildRuleMessage: BuildRuleMessage;
bulkCreate: BulkCreate;
completeRule: CompleteRule<ThreatRuleParams>;
currentResult: SearchAfterAndBulkCreateReturnType;
currentEventList: EventItem[];
eventsTelemetry: ITelemetryEventsSender | undefined;
exceptionItems: ExceptionListItemSchema[];
filters: unknown[];
inputIndex: string[];
language: LanguageOrUndefined;
listClient: ListClient;
logger: Logger;
outputIndex: string;
query: string;
savedId: string | undefined;
searchAfterSize: number;
services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
threatEnrichment: SignalsEnrichment;
tuple: RuleRangeTuple;
type: Type;
wrapHits: WrapHits;
threatFilters: unknown[];
threatIndex: ThreatIndex;
threatIndicatorPath: ThreatIndicatorPath;
threatLanguage: ThreatLanguageOrUndefined;
threatMapping: ThreatMapping;
threatQuery: ThreatQuery;
threatListConfig: ThreatListConfig;
perPage?: number;
}
type EntryKey = 'field' | 'value';
export interface BuildThreatMappingFilterOptions {
chunkSize?: number;
threatList: ThreatListItem[];
threatMapping: ThreatMapping;
entryKey: EntryKey;
}
export interface FilterThreatMappingOptions {
threatListItem: ThreatListItem;
threatMapping: ThreatMapping;
entryKey: EntryKey;
}
export interface CreateInnerAndClausesOptions {
threatListItem: ThreatListItem;
threatMappingEntries: ThreatMappingEntries;
entryKey: EntryKey;
}
export interface CreateAndOrClausesOptions {
threatListItem: ThreatListItem;
threatMapping: ThreatMapping;
entryKey: EntryKey;
}
export interface BuildEntriesMappingFilterOptions {
chunkSize: number;
threatList: ThreatListItem[];
threatMapping: ThreatMapping;
entryKey: EntryKey;
}
export interface SplitShouldClausesOptions {
@ -199,6 +238,26 @@ export interface BuildThreatEnrichmentOptions {
threatQuery: ThreatQuery;
}
export interface EventsOptions {
services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
query: string;
buildRuleMessage: BuildRuleMessage;
language: ThreatLanguageOrUndefined;
exceptionItems: ExceptionListItemSchema[];
index: string[];
searchAfter: estypes.SortResults | undefined;
perPage?: number;
logger: Logger;
filters: unknown[];
timestampOverride?: string;
tuple: RuleRangeTuple;
}
export interface EventDoc {
[key: string]: unknown;
}
export type EventItem = estypes.SearchHit<EventDoc>;
export interface EventCountOptions {
esClient: ElasticsearchClient;
exceptionItems: ExceptionListItemSchema[];
@ -209,3 +268,16 @@ export interface EventCountOptions {
tuple: RuleRangeTuple;
timestampOverride?: string;
}
export interface SignalMatch {
signalId: string;
queries: ThreatMatchNamedQuery[];
}
export type GetDocumentListInterface = (params: {
searchAfter: estypes.SortResults | undefined;
}) => Promise<estypes.SearchResponse<EventDoc | ThreatListDoc>>;
export type CreateSignalInterface = (
params: EventItem[] | ThreatListItem[]
) => Promise<SearchAfterAndBulkCreateReturnType>;

6
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts
View file

@ -9,7 +9,7 @@ import moment from 'moment';
import { SearchAfterAndBulkCreateReturnType, SignalSourceHit } from '../types';
import { parseInterval } from '../utils';
import { ThreatMatchNamedQuery } from './types';
import { ThreatMatchNamedQuery, ThreatListItem } from './types';
/**
* Given two timers this will take the max of each and add them to each other and return that addition.
@ -147,7 +147,9 @@ export const decodeThreatMatchNamedQuery = (encoded: string): ThreatMatchNamedQu
return query;
};
export const extractNamedQueries = (hit: SignalSourceHit): ThreatMatchNamedQuery[] =>
export const extractNamedQueries = (
hit: SignalSourceHit | ThreatListItem
): ThreatMatchNamedQuery[] =>
hit.matched_queries?.map((match) => decodeThreatMatchNamedQuery(match)) ?? [];
export const buildExecutionIntervalValidator: (interval: string) => () => void = (interval) => {

439
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts
View file

@ -493,7 +493,7 @@ export default ({ getService }: FtrProviderContext) => {
});
});
describe('indicator enrichment', () => {
describe('indicator enrichment: threat-first search', () => {
before(async () => {
await esArchiver.load('x-pack/test/functional/es_archives/filebeat/threat_intel');
});
@ -513,7 +513,440 @@ export default ({ getService }: FtrProviderContext) => {
language: 'kuery',
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: '*:*',
query: '*:*', // narrow events down to 2 with a destination.ip
threat_indicator_path: 'threat.indicator',
threat_query: 'threat.indicator.domain: 159.89.119.67', // narrow things down to indicators with a domain
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
{
entries: [
{
value: 'threat.indicator.domain',
field: 'destination.ip',
type: 'mapping',
},
],
},
],
threat_filters: [],
};
const { id } = await createRule(supertest, log, rule);
await waitForRuleSuccessOrStatus(supertest, log, id);
await waitForSignalsToBePresent(supertest, log, 2, [id]);
const signalsOpen = await getSignalsByIds(supertest, log, [id]);
expect(signalsOpen.hits.hits.length).equal(2);
const { hits } = signalsOpen.hits;
const threats = hits.map((hit) => hit._source?.threat);
expect(threats).to.eql([
{
enrichments: [
{
feed: {},
indicator: {
description: "domain should match the auditbeat hosts' data's source.ip",
domain: '159.89.119.67',
first_seen: '2021-01-26T11:09:04.000Z',
provider: 'geenensp',
url: {
full: 'http://159.89.119.67:59600/bin.sh',
scheme: 'http',
},
type: 'url',
},
matched: {
atomic: '159.89.119.67',
id: '978783',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'destination.ip',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
],
},
{
enrichments: [
{
feed: {},
indicator: {
description: "domain should match the auditbeat hosts' data's source.ip",
domain: '159.89.119.67',
first_seen: '2021-01-26T11:09:04.000Z',
provider: 'geenensp',
url: {
full: 'http://159.89.119.67:59600/bin.sh',
scheme: 'http',
},
type: 'url',
},
matched: {
atomic: '159.89.119.67',
id: '978783',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'destination.ip',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
],
},
]);
});
it('enriches signals with multiple indicators if several matched', async () => {
const rule: CreateRulesSchema = {
description: 'Detecting root and admin users',
name: 'Query with a rule id',
severity: 'high',
index: ['auditbeat-*'],
type: 'threat_match',
risk_score: 55,
language: 'kuery',
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: 'NOT source.port:35326', // specify query to have signals more than treat indicators, but only 1 will match
threat_indicator_path: 'threat.indicator',
threat_query: 'threat.indicator.ip: *',
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
{
entries: [
{
value: 'threat.indicator.ip',
field: 'source.ip',
type: 'mapping',
},
],
},
],
threat_filters: [],
};
const { id } = await createRule(supertest, log, rule);
await waitForRuleSuccessOrStatus(supertest, log, id);
await waitForSignalsToBePresent(supertest, log, 1, [id]);
const signalsOpen = await getSignalsByIds(supertest, log, [id]);
expect(signalsOpen.hits.hits.length).equal(1);
const { hits } = signalsOpen.hits;
const [threat] = hits.map((hit) => hit._source?.threat) as Array<{
enrichments: unknown[];
}>;
assertContains(threat.enrichments, [
{
feed: {},
indicator: {
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
port: 57324,
provider: 'geenensp',
type: 'url',
},
matched: {
atomic: '45.115.45.3',
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
{
feed: {},
indicator: {
description: 'this should match auditbeat/hosts on ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
provider: 'other_provider',
type: 'ip',
},
matched: {
atomic: '45.115.45.3',
id: '978787',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
]);
});
it('adds a single indicator that matched multiple fields', async () => {
const rule: CreateRulesSchema = {
description: 'Detecting root and admin users',
name: 'Query with a rule id',
severity: 'high',
index: ['auditbeat-*'],
type: 'threat_match',
risk_score: 55,
language: 'kuery',
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: 'NOT source.port:35326', // specify query to have signals more than treat indicators, but only 1 will match
threat_indicator_path: 'threat.indicator',
threat_query: 'threat.indicator.port: 57324 or threat.indicator.ip:45.115.45.3', // narrow our query to a single indicator
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
{
entries: [
{
value: 'threat.indicator.port',
field: 'source.port',
type: 'mapping',
},
],
},
{
entries: [
{
value: 'threat.indicator.ip',
field: 'source.ip',
type: 'mapping',
},
],
},
],
threat_filters: [],
};
const { id } = await createRule(supertest, log, rule);
await waitForRuleSuccessOrStatus(supertest, log, id);
await waitForSignalsToBePresent(supertest, log, 1, [id]);
const signalsOpen = await getSignalsByIds(supertest, log, [id]);
expect(signalsOpen.hits.hits.length).equal(1);
const { hits } = signalsOpen.hits;
const [threat] = hits.map((hit) => hit._source?.threat) as Array<{
enrichments: unknown[];
}>;
assertContains(threat.enrichments, [
{
feed: {},
indicator: {
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
port: 57324,
provider: 'geenensp',
type: 'url',
},
matched: {
atomic: '45.115.45.3',
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
// We do not merge matched indicators during enrichment, so in
// certain circumstances a given indicator document could appear
// multiple times in an enriched alert (albeit with different
// threat.indicator.matched data). That's the case with the
// first and third indicators matched, here.
{
feed: {},
indicator: {
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
port: 57324,
provider: 'geenensp',
type: 'url',
},
matched: {
atomic: 57324,
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.port',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
{
feed: {},
indicator: {
description: 'this should match auditbeat/hosts on ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
provider: 'other_provider',
type: 'ip',
},
matched: {
atomic: '45.115.45.3',
id: '978787',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
]);
});
it('generates multiple signals with multiple matches', async () => {
const rule: CreateRulesSchema = {
description: 'Detecting root and admin users',
name: 'Query with a rule id',
severity: 'high',
index: ['auditbeat-*'],
type: 'threat_match',
risk_score: 55,
language: 'kuery',
threat_language: 'kuery',
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: '*:*', // narrow our query to a single record that matches two indicators
threat_indicator_path: 'threat.indicator',
threat_query: '*:*',
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
threat_mapping: [
{
entries: [
{
value: 'threat.indicator.port',
field: 'source.port',
type: 'mapping',
},
{
value: 'threat.indicator.ip',
field: 'source.ip',
type: 'mapping',
},
],
},
{
entries: [
{
value: 'threat.indicator.domain',
field: 'destination.ip',
type: 'mapping',
},
],
},
],
threat_filters: [],
};
const { id } = await createRule(supertest, log, rule);
await waitForRuleSuccessOrStatus(supertest, log, id);
await waitForSignalsToBePresent(supertest, log, 2, [id]);
const signalsOpen = await getSignalsByIds(supertest, log, [id]);
expect(signalsOpen.hits.hits.length).equal(2);
const { hits } = signalsOpen.hits;
const threats = hits.map((hit) => hit._source?.threat) as Array<{
enrichments: unknown[];
}>;
assertContains(threats[0].enrichments, [
{
feed: {},
indicator: {
description: "domain should match the auditbeat hosts' data's source.ip",
domain: '159.89.119.67',
first_seen: '2021-01-26T11:09:04.000Z',
provider: 'geenensp',
type: 'url',
url: {
full: 'http://159.89.119.67:59600/bin.sh',
scheme: 'http',
},
},
matched: {
atomic: '159.89.119.67',
id: '978783',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'destination.ip',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
{
feed: {},
indicator: {
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
port: 57324,
provider: 'geenensp',
type: 'url',
},
matched: {
atomic: '45.115.45.3',
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
{
feed: {},
indicator: {
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
port: 57324,
provider: 'geenensp',
type: 'url',
},
matched: {
atomic: 57324,
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.port',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
]);
assertContains(threats[1].enrichments, [
{
feed: {},
indicator: {
description: "domain should match the auditbeat hosts' data's source.ip",
domain: '159.89.119.67',
first_seen: '2021-01-26T11:09:04.000Z',
provider: 'geenensp',
type: 'url',
url: {
full: 'http://159.89.119.67:59600/bin.sh',
scheme: 'http',
},
},
matched: {
atomic: '159.89.119.67',
id: '978783',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'destination.ip',
type: ENRICHMENT_TYPES.IndicatorMatchRule,
},
},
]);
});
});
describe('indicator enrichment: event-first search', () => {
before(async () => {
await esArchiver.load('x-pack/test/functional/es_archives/filebeat/threat_intel');
});
after(async () => {
await esArchiver.unload('x-pack/test/functional/es_archives/filebeat/threat_intel');
});
it('enriches signals with the single indicator that matched', async () => {
const rule: CreateRulesSchema = {
description: 'Detecting root and admin users',
name: 'Query with a rule id',
severity: 'high',
index: ['auditbeat-*'],
type: 'threat_match',
risk_score: 55,
language: 'kuery',
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: 'destination.ip:159.89.119.67',
threat_indicator_path: 'threat.indicator',
threat_query: 'threat.indicator.domain: *', // narrow things down to indicators with a domain
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module
@ -797,7 +1230,7 @@ export default ({ getService }: FtrProviderContext) => {
threat_language: 'kuery',
rule_id: 'rule-1',
from: '1900-01-01T00:00:00.000Z',
query: '*:*', // narrow our query to a single record that matches two indicators
query: '(source.port:57324 and source.ip:45.115.45.3) or destination.ip:159.89.119.67', // narrow our query to a single record that matches two indicators
threat_indicator_path: 'threat.indicator',
threat_query: '*:*',
threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module

142
x-pack/test/functional/es_archives/filebeat/threat_intel/data.json
View file

@ -274,3 +274,145 @@
}
}
}
{
"type": "doc",
"value": {
"id": "978766",
"index": "filebeat-8.0.0-2021.01.26-000001",
"source": {
"@timestamp": "2021-01-26T11:09:05.529Z",
"agent": {
"ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0",
"id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51",
"name": "MacBook-Pro-de-Gloria.local",
"type": "filebeat",
"version": "8.0.0"
},
"ecs": {
"version": "1.6.0"
},
"event": {
"category": "threat",
"created": "2021-01-26T11:09:05.529Z",
"dataset": "ti_abusech.malware",
"ingested": "2021-01-26T11:09:06.595350Z",
"kind": "enrichment",
"module": "threatintel",
"reference": "https://urlhaus.abuse.ch/url/978783/",
"type": "indicator"
},
"fileset": {
"name": "abuseurl"
},
"input": {
"type": "httpjson"
},
"service": {
"type": "threatintel"
},
"tags": [
"threatintel-abuseurls",
"forwarded"
],
"threat": {
"indicator": {
"description": "domain should match the auditbeat hosts' data's source.ip",
"domain": "172.16.0.0",
"ip": "8.8.8.8",
"port": 777,
"first_seen": "2021-01-26T11:09:04.000Z",
"provider": "geenensp",
"type": "url",
"url": {
"full": "http://159.89.119.67:59600/bin.sh",
"scheme": "http"
}
}
},
"threatintel": {
"abuseurl": {
"blacklists": {
"spamhaus_dbl": "not listed",
"surbl": "not listed"
},
"larted": false,
"tags": null,
"threat": "malware_download",
"url_status": "online"
}
}
}
}
}
{
"type": "doc",
"value": {
"id": "978767",
"index": "filebeat-8.0.0-2021.01.26-000001",
"source": {
"@timestamp": "2021-01-26T11:09:05.529Z",
"agent": {
"ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0",
"id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51",
"name": "MacBook-Pro-de-Gloria.local",
"type": "filebeat",
"version": "8.0.0"
},
"ecs": {
"version": "1.6.0"
},
"event": {
"category": "threat",
"created": "2021-01-26T11:09:05.529Z",
"dataset": "ti_abusech.malware",
"ingested": "2021-01-26T11:09:06.595350Z",
"kind": "enrichment",
"module": "threatintel",
"reference": "https://urlhaus.abuse.ch/url/978783/",
"type": "indicator"
},
"fileset": {
"name": "abuseurl"
},
"input": {
"type": "httpjson"
},
"service": {
"type": "threatintel"
},
"tags": [
"threatintel-abuseurls",
"forwarded"
],
"threat": {
"indicator": {
"description": "domain should match the auditbeat hosts' data's source.ip",
"domain": "172.16.0.0",
"ip": "9.9.9.9",
"port": 123,
"first_seen": "2021-01-26T11:09:04.000Z",
"provider": "geenensp",
"type": "url",
"url": {
"full": "http://159.89.119.67:59600/bin.sh",
"scheme": "http"
}
}
},
"threatintel": {
"abuseurl": {
"blacklists": {
"spamhaus_dbl": "not listed",
"surbl": "not listed"
},
"larted": false,
"tags": null,
"threat": "malware_download",
"url_status": "online"
}
}
}
}
}