[SIEM] fix data provider query when the field is a timestamp (#37281)

* fix dataprovider query when the filed is a timestamp

* review II

* review III
This commit is contained in:
Xavier Mouligneau 2019-05-29 13:33:06 -04:00 committed by GitHub
parent 86c3ac4ff4
commit 3f78d29ee5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 433 additions and 24 deletions

View file

@ -339,6 +339,25 @@ exports[`DragDropContextWrapper rendering it renders against the snapshot 1`] =
},
},
},
"event": Object {
"fields": Object {
"event.end": Object {
"aggregatable": true,
"category": "event",
"description": "event.end contains the date when the event ended or when the activity was last observed.",
"example": null,
"indexes": Array [
"auditbeat-*",
"filebeat-*",
"packetbeat-*",
"winlogbeat-*",
],
"name": "event.end",
"searchable": true,
"type": "date",
},
},
},
"source": Object {
"fields": Object {
"source.ip": Object {

View file

@ -339,6 +339,25 @@ exports[`DraggableWrapper rendering it renders against the snapshot 1`] = `
},
},
},
"event": Object {
"fields": Object {
"event.end": Object {
"aggregatable": true,
"category": "event",
"description": "event.end contains the date when the event ended or when the activity was last observed.",
"example": null,
"indexes": Array [
"auditbeat-*",
"filebeat-*",
"packetbeat-*",
"winlogbeat-*",
],
"name": "event.end",
"searchable": true,
"type": "date",
},
},
},
"source": Object {
"fields": Object {
"source.ip": Object {

View file

@ -339,6 +339,25 @@ exports[`DroppableWrapper rendering it renders against the snapshot 1`] = `
},
},
},
"event": Object {
"fields": Object {
"event.end": Object {
"aggregatable": true,
"category": "event",
"description": "event.end contains the date when the event ended or when the activity was last observed.",
"example": null,
"indexes": Array [
"auditbeat-*",
"filebeat-*",
"packetbeat-*",
"winlogbeat-*",
],
"name": "event.end",
"searchable": true,
"type": "date",
},
},
},
"source": Object {
"fields": Object {
"source.ip": Object {

View file

@ -335,6 +335,25 @@ exports[`EventDetails rendering should match snapshot 1`] = `
},
},
},
"event": Object {
"fields": Object {
"event.end": Object {
"aggregatable": true,
"category": "event",
"description": "event.end contains the date when the event ended or when the activity was last observed.",
"example": null,
"indexes": Array [
"auditbeat-*",
"filebeat-*",
"packetbeat-*",
"winlogbeat-*",
],
"name": "event.end",
"searchable": true,
"type": "date",
},
},
},
"source": Object {
"fields": Object {
"source.ip": Object {

View file

@ -334,6 +334,25 @@ exports[`Timeline rendering renders correctly against snapshot 1`] = `
},
},
},
"event": Object {
"fields": Object {
"event.end": Object {
"aggregatable": true,
"category": "event",
"description": "event.end contains the date when the event ended or when the activity was last observed.",
"example": null,
"indexes": Array [
"auditbeat-*",
"filebeat-*",
"packetbeat-*",
"winlogbeat-*",
],
"name": "event.end",
"searchable": true,
"type": "date",
},
},
},
"source": Object {
"fields": Object {
"source.ip": Object {

View file

@ -335,6 +335,25 @@ exports[`ColumnHeaders rendering renders correctly against snapshot 1`] = `
},
},
},
"event": Object {
"fields": Object {
"event.end": Object {
"aggregatable": true,
"category": "event",
"description": "event.end contains the date when the event ended or when the activity was last observed.",
"example": null,
"indexes": Array [
"auditbeat-*",
"filebeat-*",
"packetbeat-*",
"winlogbeat-*",
],
"name": "event.end",
"searchable": true,
"type": "date",
},
},
},
"source": Object {
"fields": Object {
"source.ip": Object {

View file

@ -334,6 +334,25 @@ exports[`SuricataDetails rendering it renders the default SuricataDetails 1`] =
},
},
},
"event": Object {
"fields": Object {
"event.end": Object {
"aggregatable": true,
"category": "event",
"description": "event.end contains the date when the event ended or when the activity was last observed.",
"example": null,
"indexes": Array [
"auditbeat-*",
"filebeat-*",
"packetbeat-*",
"winlogbeat-*",
],
"name": "event.end",
"searchable": true,
"type": "date",
},
},
},
"source": Object {
"fields": Object {
"source.ip": Object {

View file

@ -342,6 +342,25 @@ exports[`suricata_row_renderer renders correctly against snapshot 1`] = `
},
},
},
"event": Object {
"fields": Object {
"event.end": Object {
"aggregatable": true,
"category": "event",
"description": "event.end contains the date when the event ended or when the activity was last observed.",
"example": null,
"indexes": Array [
"auditbeat-*",
"filebeat-*",
"packetbeat-*",
"winlogbeat-*",
],
"name": "event.end",
"searchable": true,
"type": "date",
},
},
},
"source": Object {
"fields": Object {
"source.ip": Object {

View file

@ -334,6 +334,25 @@ exports[`ZeekDetails rendering it renders the default ZeekDetails 1`] = `
},
},
},
"event": Object {
"fields": Object {
"event.end": Object {
"aggregatable": true,
"category": "event",
"description": "event.end contains the date when the event ended or when the activity was last observed.",
"example": null,
"indexes": Array [
"auditbeat-*",
"filebeat-*",
"packetbeat-*",
"winlogbeat-*",
],
"name": "event.end",
"searchable": true,
"type": "date",
},
},
},
"source": Object {
"fields": Object {
"source.ip": Object {

View file

@ -342,6 +342,25 @@ exports[`zeek_row_renderer renders correctly against snapshot 1`] = `
},
},
},
"event": Object {
"fields": Object {
"event.end": Object {
"aggregatable": true,
"category": "event",
"description": "event.end contains the date when the event ended or when the activity was last observed.",
"example": null,
"indexes": Array [
"auditbeat-*",
"filebeat-*",
"packetbeat-*",
"winlogbeat-*",
],
"name": "event.end",
"searchable": true,
"type": "date",
},
},
},
"source": Object {
"fields": Object {
"source.ip": Object {

View file

@ -10,36 +10,105 @@ import { mockIndexPattern } from '../../mock';
import { mockDataProviders } from './data_providers/mock/mock_data_providers';
import { buildGlobalQuery, combineQueries } from './helpers';
import { mockBrowserFields } from '../../containers/source/mock';
const cleanUpKqlQuery = (str: string) => str.replace(/\n/g, '').replace(/\s\s+/g, ' ');
const startDate = new Date('2018-03-23T18:49:23.132Z').valueOf();
const endDate = new Date('2018-03-24T03:33:52.253Z').valueOf();
describe('Build KQL Query', () => {
test('Buld KQL query with one data provider', () => {
test('Build KQL query with one data provider', () => {
const dataProviders = mockDataProviders.slice(0, 1);
const kqlQuery = buildGlobalQuery(dataProviders);
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( name : Provider 1 )');
});
test('Buld KQL query with two data provider', () => {
test('Build KQL query with one data provider as timestamp (string input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].queryMatch.field = '@timestamp';
dataProviders[0].queryMatch.value = '2018-03-23T23:36:23.232Z';
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( @timestamp: 1521848183232 )');
});
test('Buld KQL query with one data provider as timestamp (numeric input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].queryMatch.field = '@timestamp';
dataProviders[0].queryMatch.value = 1521848183232;
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( @timestamp: 1521848183232 )');
});
test('Build KQL query with one data provider as date type (string input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].queryMatch.field = 'event.end';
dataProviders[0].queryMatch.value = '2018-03-23T23:36:23.232Z';
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( event.end: 1521848183232 )');
});
test('Buld KQL query with one data provider as date type (numeric input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].queryMatch.field = 'event.end';
dataProviders[0].queryMatch.value = 1521848183232;
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( event.end: 1521848183232 )');
});
test('Build KQL query with two data provider', () => {
const dataProviders = mockDataProviders.slice(0, 2);
const kqlQuery = buildGlobalQuery(dataProviders);
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( name : Provider 1 ) or ( name : Provider 2 )');
});
test('Buld KQL query with one data provider and one and', () => {
test('Build KQL query with one data provider and one and', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].and = mockDataProviders.slice(1, 2);
const kqlQuery = buildGlobalQuery(dataProviders);
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( name : Provider 1 and name : Provider 2)');
});
test('Buld KQL query with two data provider and mutiple and', () => {
test('Build KQL query with one data provider and one and as timestamp (string input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].and = cloneDeep(mockDataProviders.slice(1, 2));
dataProviders[0].and[0].queryMatch.field = '@timestamp';
dataProviders[0].and[0].queryMatch.value = '2018-03-23T23:36:23.232Z';
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( name : Provider 1 and @timestamp: 1521848183232)');
});
test('Build KQL query with one data provider and one and as timestamp (numeric input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].and = cloneDeep(mockDataProviders.slice(1, 2));
dataProviders[0].and[0].queryMatch.field = '@timestamp';
dataProviders[0].and[0].queryMatch.value = 1521848183232;
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( name : Provider 1 and @timestamp: 1521848183232)');
});
test('Build KQL query with one data provider and one and as date type (string input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].and = cloneDeep(mockDataProviders.slice(1, 2));
dataProviders[0].and[0].queryMatch.field = 'event.end';
dataProviders[0].and[0].queryMatch.value = '2018-03-23T23:36:23.232Z';
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( name : Provider 1 and event.end: 1521848183232)');
});
test('Build KQL query with one data provider and one and as date type (numeric input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].and = cloneDeep(mockDataProviders.slice(1, 2));
dataProviders[0].and[0].queryMatch.field = 'event.end';
dataProviders[0].and[0].queryMatch.value = 1521848183232;
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual('( name : Provider 1 and event.end: 1521848183232)');
});
test('Build KQL query with two data provider and multiple and', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 2));
dataProviders[0].and = mockDataProviders.slice(2, 4);
dataProviders[1].and = mockDataProviders.slice(4, 5);
const kqlQuery = buildGlobalQuery(dataProviders);
const kqlQuery = buildGlobalQuery(dataProviders, mockBrowserFields);
expect(cleanUpKqlQuery(kqlQuery)).toEqual(
'( name : Provider 1 and name : Provider 3 and name : Provider 4) or ( name : Provider 2 and name : Provider 5)'
);
@ -48,7 +117,9 @@ describe('Build KQL Query', () => {
describe('Combined Queries', () => {
test('No Data Provider & No kqlQuery', () => {
expect(combineQueries([], mockIndexPattern, '', 'search', startDate, endDate)).toBeNull();
expect(
combineQueries([], mockIndexPattern, mockBrowserFields, '', 'search', startDate, endDate)
).toBeNull();
});
test('Only Data Provider', () => {
@ -56,6 +127,7 @@ describe('Combined Queries', () => {
const { filterQuery } = combineQueries(
dataProviders,
mockIndexPattern,
mockBrowserFields,
'',
'search',
startDate,
@ -66,10 +138,83 @@ describe('Combined Queries', () => {
);
});
test('Only Data Provider with timestamp (string input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].queryMatch.field = '@timestamp';
dataProviders[0].queryMatch.value = '2018-03-23T23:36:23.232Z';
const { filterQuery } = combineQueries(
dataProviders,
mockIndexPattern,
mockBrowserFields,
'',
'search',
startDate,
endDate
)!;
expect(filterQuery).toEqual(
'{"bool":{"filter":[{"bool":{"should":[{"range":{"@timestamp":{"gte":1521848183232,"lte":1521848183232}}}],"minimum_should_match":1}},{"bool":{"filter":[{"bool":{"should":[{"range":{"@timestamp":{"gte":1521830963132}}}],"minimum_should_match":1}},{"bool":{"should":[{"range":{"@timestamp":{"lte":1521862432253}}}],"minimum_should_match":1}}]}}]}}'
);
});
test('Only Data Provider with timestamp (numeric input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].queryMatch.field = '@timestamp';
dataProviders[0].queryMatch.value = 1521848183232;
const { filterQuery } = combineQueries(
dataProviders,
mockIndexPattern,
mockBrowserFields,
'',
'search',
startDate,
endDate
)!;
expect(filterQuery).toEqual(
'{"bool":{"filter":[{"bool":{"should":[{"range":{"@timestamp":{"gte":1521848183232,"lte":1521848183232}}}],"minimum_should_match":1}},{"bool":{"filter":[{"bool":{"should":[{"range":{"@timestamp":{"gte":1521830963132}}}],"minimum_should_match":1}},{"bool":{"should":[{"range":{"@timestamp":{"lte":1521862432253}}}],"minimum_should_match":1}}]}}]}}'
);
});
test('Only Data Provider with a date type (string input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].queryMatch.field = 'event.end';
dataProviders[0].queryMatch.value = '2018-03-23T23:36:23.232Z';
const { filterQuery } = combineQueries(
dataProviders,
mockIndexPattern,
mockBrowserFields,
'',
'search',
startDate,
endDate
)!;
expect(filterQuery).toEqual(
'{"bool":{"filter":[{"bool":{"should":[{"match":{"event.end":1521848183232}}],"minimum_should_match":1}},{"bool":{"filter":[{"bool":{"should":[{"range":{"@timestamp":{"gte":1521830963132}}}],"minimum_should_match":1}},{"bool":{"should":[{"range":{"@timestamp":{"lte":1521862432253}}}],"minimum_should_match":1}}]}}]}}'
);
});
test('Only Data Provider with date type (numeric input)', () => {
const dataProviders = cloneDeep(mockDataProviders.slice(0, 1));
dataProviders[0].queryMatch.field = 'event.end';
dataProviders[0].queryMatch.value = 1521848183232;
const { filterQuery } = combineQueries(
dataProviders,
mockIndexPattern,
mockBrowserFields,
'',
'search',
startDate,
endDate
)!;
expect(filterQuery).toEqual(
'{"bool":{"filter":[{"bool":{"should":[{"match":{"event.end":1521848183232}}],"minimum_should_match":1}},{"bool":{"filter":[{"bool":{"should":[{"range":{"@timestamp":{"gte":1521830963132}}}],"minimum_should_match":1}},{"bool":{"should":[{"range":{"@timestamp":{"lte":1521862432253}}}],"minimum_should_match":1}}]}}]}}'
);
});
test('Only KQL search/filter query', () => {
const { filterQuery } = combineQueries(
[],
mockIndexPattern,
mockBrowserFields,
'host.name: "host-1"',
'search',
startDate,
@ -85,6 +230,7 @@ describe('Combined Queries', () => {
const { filterQuery } = combineQueries(
dataProviders,
mockIndexPattern,
mockBrowserFields,
'host.name: "host-1"',
'search',
startDate,
@ -100,6 +246,7 @@ describe('Combined Queries', () => {
const { filterQuery } = combineQueries(
dataProviders,
mockIndexPattern,
mockBrowserFields,
'host.name: "host-1"',
'filter',
startDate,

View file

@ -4,43 +4,86 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { isEmpty, isNumber } from 'lodash/fp';
import { isEmpty, isNumber, get } from 'lodash/fp';
import memoizeOne from 'memoize-one';
import { StaticIndexPattern } from 'ui/index_patterns';
import { convertKueryToElasticSearchQuery, escapeQueryValue } from '../../lib/keury';
import { DataProvider, DataProvidersAnd, EXISTS_OPERATOR } from './data_providers/data_provider';
import { BrowserFields } from '../../containers/source';
const buildQueryMatch = (dataProvider: DataProvider | DataProvidersAnd) =>
const convertDateFieldToQuery = (field: string, value: string | number) =>
`${field}: ${isNumber(value) ? value : new Date(value).valueOf()}`;
const getBaseFields = memoizeOne(
(browserFields: BrowserFields): string[] => {
const baseFields = get('base', browserFields);
if (baseFields != null && baseFields.fields != null) {
return Object.keys(baseFields.fields);
}
return [];
}
);
const getBrowserFieldPath = (field: string, browserFields: BrowserFields) => {
const splitFields = field.split('.');
const baseFields = getBaseFields(browserFields);
if (baseFields.includes(field)) {
return ['base', 'fields', field];
}
return [splitFields[0], 'fields', field];
};
const checkIfFieldTypeIsDate = (field: string, browserFields: BrowserFields) => {
const pathBrowserField = getBrowserFieldPath(field, browserFields);
const browserField = get(pathBrowserField, browserFields);
if (browserField != null && browserField.type === 'date') {
return true;
}
return false;
};
const buildQueryMatch = (
dataProvider: DataProvider | DataProvidersAnd,
browserFields: BrowserFields
) =>
`${dataProvider.excluded ? 'NOT ' : ''}${
dataProvider.queryMatch.operator !== EXISTS_OPERATOR
? `${dataProvider.queryMatch.field} : ${
isNumber(dataProvider.queryMatch.value)
? dataProvider.queryMatch.value
: escapeQueryValue(dataProvider.queryMatch.value)
}`
? checkIfFieldTypeIsDate(dataProvider.queryMatch.field, browserFields)
? convertDateFieldToQuery(dataProvider.queryMatch.field, dataProvider.queryMatch.value)
: `${dataProvider.queryMatch.field} : ${
isNumber(dataProvider.queryMatch.value)
? dataProvider.queryMatch.value
: escapeQueryValue(dataProvider.queryMatch.value)
}`
: `${dataProvider.queryMatch.field} ${EXISTS_OPERATOR}`
}`.trim();
const buildQueryForAndProvider = (dataAndProviders: DataProvidersAnd[]) =>
const buildQueryForAndProvider = (
dataAndProviders: DataProvidersAnd[],
browserFields: BrowserFields
) =>
dataAndProviders
.reduce((andQuery, andDataProvider) => {
const prepend = (q: string) => `${q !== '' ? `${q} and ` : ''}`;
return andDataProvider.enabled
? `${prepend(andQuery)} ${buildQueryMatch(andDataProvider)}`
? `${prepend(andQuery)} ${buildQueryMatch(andDataProvider, browserFields)}`
: andQuery;
}, '')
.trim();
export const buildGlobalQuery = (dataProviders: DataProvider[]) =>
export const buildGlobalQuery = (dataProviders: DataProvider[], browserFields: BrowserFields) =>
dataProviders
.reduce((query, dataProvider: DataProvider) => {
const prepend = (q: string) => `${q !== '' ? `${q} or ` : ''}`;
return dataProvider.enabled
? `${prepend(query)}(
${buildQueryMatch(dataProvider)}
${buildQueryMatch(dataProvider, browserFields)}
${
dataProvider.and.length > 0 ? ` and ${buildQueryForAndProvider(dataProvider.and)}` : ''
dataProvider.and.length > 0
? ` and ${buildQueryForAndProvider(dataProvider.and, browserFields)}`
: ''
})`.trim()
: query;
}, '')
@ -49,6 +92,7 @@ export const buildGlobalQuery = (dataProviders: DataProvider[]) =>
export const combineQueries = (
dataProviders: DataProvider[],
indexPattern: StaticIndexPattern,
browserFields: BrowserFields,
kqlQuery: string,
kqlMode: string,
start: number,
@ -67,7 +111,8 @@ export const combineQueries = (
return {
filterQuery: convertKueryToElasticSearchQuery(
`((${buildGlobalQuery(
dataProviders
dataProviders,
browserFields
)}) and @timestamp >= ${start} and @timestamp <= ${end})`,
indexPattern
),
@ -75,10 +120,9 @@ export const combineQueries = (
}
const operatorKqlQuery = kqlMode === 'filter' ? 'and' : 'or';
const postpend = (q: string) => `${!isEmpty(q) ? ` ${operatorKqlQuery} (${q})` : ''}`;
const globalQuery = `((${buildGlobalQuery(dataProviders)}${postpend(
const globalQuery = `((${buildGlobalQuery(dataProviders, browserFields)}${postpend(
kqlQuery
)}) and @timestamp >= ${start} and @timestamp <= ${end})`;
return {
filterQuery: convertKueryToElasticSearchQuery(globalQuery, indexPattern),
};

View file

@ -104,6 +104,7 @@ export const Timeline = pure<Props>(
const combinedQueries = combineQueries(
dataProviders,
indexPattern,
browserFields,
kqlQueryExpression,
kqlMode,
start,

View file

@ -301,6 +301,17 @@ export const mocksSource = [
searchable: true,
type: 'long',
},
{
aggregatable: true,
category: 'event',
description:
'event.end contains the date when the event ended or when the activity was last observed.',
example: null,
indexes: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
name: 'event.end',
searchable: true,
type: 'date',
},
],
},
},
@ -334,6 +345,7 @@ export const mockIndexFields = [
{ aggregatable: true, name: 'destination.port', searchable: true, type: 'long' },
{ aggregatable: true, name: 'source.ip', searchable: true, type: 'ip' },
{ aggregatable: true, name: 'source.port', searchable: true, type: 'long' },
{ aggregatable: true, name: 'event.end', searchable: true, type: 'date' },
];
export const mockBrowserFields: BrowserFields = {
@ -593,6 +605,21 @@ export const mockBrowserFields: BrowserFields = {
},
},
},
event: {
fields: {
'event.end': {
category: 'event',
description:
'event.end contains the date when the event ended or when the activity was last observed.',
example: null,
indexes: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
name: 'event.end',
searchable: true,
type: 'date',
aggregatable: true,
},
},
},
source: {
fields: {
'source.ip': {