github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution] [Timeline] Consolidate reduces, remove unneeded async/awaits, other small fixes (#197168)

Browse source 
## Summary

For most of 8.x, both anecdotally from users and in development,
timeline search strategy based apis would often seem slower than the
equivalent search in discover or elsewhere in kibana, and I have long
suspected that this came from how the timeline sever code formatted the
elasticsearch responses for use in the UI, and while working on
something else, noticed even higher than normal occurrences in logs of
"][http.server.Kibana] Event loop utilization for
/internal/search/timelineSearchStrategy exceeded threshold of..." and so
I tried to refactor all of the functions in place as much as possible,
keeping the apis similar, most of the unit tests, etc, but removing as
many as possible of the Promise.alls, reduce within reduce, etc. This
has lead to a substantial improvement in performance, as you can see
below, and with larger result sets, I think the difference would only be
more noticeable.

After fix:
~40 ms for formatTimelineData with ~1000 docs
<img width="1470" alt="image"
src="https://github.com/user-attachments/assets/c664f940-aa37-4335-9204-2a9300fbafa0">
Before fix:
~18000 ms for formatTimelineData with ~1000 docs
<img width="1464" alt="image"
src="https://github.com/user-attachments/assets/124fa327-13b9-41ef-9489-8d27f853590c">


[chrome_profile_timeline_slow.cpuprofile](https://github.com/user-attachments/files/17825602/chrome_profile_timeline_slow.cpuprofile)

[chrome_profile_timeline_fast.cpuprofile](https://github.com/user-attachments/files/17825606/chrome_profile_timeline_fast.cpuprofile)
I've attached the chrome devtools profiles for each, the time was
measured with the function:

```
async function measureAwait<T>(promise: Promise<T>, label: string): Promise<T> {
  const start = performance.now();
  try {
    const result = await promise;
    const duration = performance.now() - start;
    console.log(`${label} took ${duration}ms`);
    return result;
  } catch (error) {
    const duration = performance.now() - start;
    console.log(`${label} failed after ${duration}ms`);
    throw error;
  }
}
```

Wrapped around the call to formatTimelineData in
x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts


### Checklist

- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
This commit is contained in:
Kevin Qualters 2024-11-22 13:24:54 -05:00 • committed by GitHub
parent ad61b9d101
commit 30fb8dd5bb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 740 additions and 1060 deletions
Download patch file Download diff file Expand all files Collapse all files

46
packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts
View file

@ -291,6 +291,29 @@ export const eventDetailsFormattedFields = [
originalValue: [`{"lon":118.7778,"lat":32.0617}`],
values: [`{"lon":118.7778,"lat":32.0617}`],
},
{
category: 'threat',
field: 'threat.enrichments',
isObjectArray: true,
originalValue: [
'{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}',
'{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}',
'{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}',
'{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
],
values: [
'{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}',
'{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}',
'{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}',
'{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
],
},
{
category: 'threat',
field: 'threat.enrichments.matched.field',
@ -376,27 +399,4 @@ export const eventDetailsFormattedFields = [
originalValue: ['FFEtSYIBZ61VHL7LvV2j', 'E1EtSYIBZ61VHL7Ltl3m', 'CFErSYIBZ61VHL7LIV1N'],
values: ['FFEtSYIBZ61VHL7LvV2j', 'E1EtSYIBZ61VHL7Ltl3m', 'CFErSYIBZ61VHL7LIV1N'],
},
{
category: 'threat',
field: 'threat.enrichments',
isObjectArray: true,
originalValue: [
'{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}',
'{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}',
'{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}',
'{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
],
values: [
'{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}',
'{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}',
'{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}',
'{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
'{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}',
],
},
];

94
x-pack/plugins/security_solution/common/utils/field_formatters.test.ts
View file

@ -1,94 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { EventHit } from '../search_strategy';
import { getDataFromFieldsHits, getDataSafety } from './field_formatters';
import { eventDetailsFormattedFields, eventHit } from '@kbn/securitysolution-t-grid';
describe('Events Details Helpers', () => {
const fields: EventHit['fields'] = eventHit.fields;
const resultFields = eventDetailsFormattedFields;
describe('#getDataFromFieldsHits', () => {
it('happy path', () => {
const result = getDataFromFieldsHits(fields);
expect(result).toEqual(resultFields);
});
it('lets get weird', () => {
const whackFields = {
'crazy.pants': [
{
'matched.field': ['matched_field'],
first_seen: ['2021-02-22T17:29:25.195Z'],
provider: ['yourself'],
type: ['custom'],
'matched.atomic': ['matched_atomic'],
lazer: [
{
'great.field': ['grrrrr'],
lazer: [
{
lazer: [
{
cool: true,
lazer: [
{
lazer: [
{
lazer: [
{
lazer: [
{
whoa: false,
},
],
},
],
},
],
},
],
},
],
},
{
lazer: [
{
cool: false,
},
],
},
],
},
{
'great.field': ['grrrrr_2'],
},
],
},
],
};
const whackResultFields = [
{
category: 'crazy',
field: 'crazy.pants',
values: [
'{"matched.field":["matched_field"],"first_seen":["2021-02-22T17:29:25.195Z"],"provider":["yourself"],"type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"],"lazer":[{"lazer":[{"cool":true,"lazer":[{"lazer":[{"lazer":[{"lazer":[{"whoa":false}]}]}]}]}]},{"lazer":[{"cool":false}]}]},{"great.field":["grrrrr_2"]}]}',
],
originalValue: [
'{"matched.field":["matched_field"],"first_seen":["2021-02-22T17:29:25.195Z"],"provider":["yourself"],"type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"],"lazer":[{"lazer":[{"cool":true,"lazer":[{"lazer":[{"lazer":[{"lazer":[{"whoa":false}]}]}]}]}]},{"lazer":[{"cool":false}]}]},{"great.field":["grrrrr_2"]}]}',
],
isObjectArray: true,
},
];
const result = getDataFromFieldsHits(whackFields);
expect(result).toEqual(whackResultFields);
});
});
it('#getDataSafety', async () => {
const result = await getDataSafety(getDataFromFieldsHits, fields);
expect(result).toEqual(resultFields);
});
});

148
x-pack/plugins/security_solution/common/utils/field_formatters.ts
View file

@ -1,148 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map';
import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils';
import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map';
import { isEmpty } from 'lodash/fp';
import { ENRICHMENT_DESTINATION_PATH } from '../constants';
import type { Fields, TimelineEventsDetailsItem } from '../search_strategy';
import { toObjectArrayOfStrings, toStringArray } from './to_array';
export const baseCategoryFields = ['@timestamp', 'labels', 'message', 'tags'];
export const getFieldCategory = (field: string): string => {
const fieldCategory = field.split('.')[0];
if (!isEmpty(fieldCategory) && baseCategoryFields.includes(fieldCategory)) {
return 'base';
}
return fieldCategory;
};
export const formatGeoLocation = (item: unknown[]) => {
const itemGeo = item.length > 0 ? (item[0] as { coordinates: number[] }) : null;
if (itemGeo != null && !isEmpty(itemGeo.coordinates)) {
try {
return toStringArray({
lon: itemGeo.coordinates[0],
lat: itemGeo.coordinates[1],
});
} catch {
return toStringArray(item);
}
}
return toStringArray(item);
};
export const isGeoField = (field: string) =>
field.includes('geo.location') || field.includes('geoip.location');
export const isThreatEnrichmentFieldOrSubfield = (field: string, prependField?: string) =>
prependField?.includes(ENRICHMENT_DESTINATION_PATH) || field === ENRICHMENT_DESTINATION_PATH;
export const getDataFromFieldsHits = (
fields: Fields,
prependField?: string,
prependFieldCategory?: string
): TimelineEventsDetailsItem[] =>
Object.keys(fields).reduce<TimelineEventsDetailsItem[]>((accumulator, field) => {
const item: unknown[] = fields[field];
const fieldCategory =
prependFieldCategory != null ? prependFieldCategory : getFieldCategory(field);
if (isGeoField(field)) {
return [
...accumulator,
{
category: fieldCategory,
field,
values: formatGeoLocation(item),
originalValue: formatGeoLocation(item),
isObjectArray: true, // important for UI
},
];
}
const objArrStr = toObjectArrayOfStrings(item);
const strArr = objArrStr.map(({ str }) => str);
const isObjectArray = objArrStr.some((o) => o.isObjectArray);
const dotField = prependField ? `${prependField}.${field}` : field;
// return simple field value (non-esc object, non-array)
if (
!isObjectArray ||
Object.keys({ ...ecsFieldMap, ...technicalRuleFieldMap, ...legacyExperimentalFieldMap }).find(
(ecsField) => ecsField === field
) === undefined
) {
return [
...accumulator,
{
category: fieldCategory,
field: dotField,
values: strArr,
originalValue: strArr,
isObjectArray,
},
];
}
const threatEnrichmentObject = isThreatEnrichmentFieldOrSubfield(field, prependField)
? [
{
category: fieldCategory,
field: dotField,
values: strArr,
originalValue: strArr,
isObjectArray,
},
]
: [];
// format nested fields
const nestedFields = Array.isArray(item)
? item
.reduce<TimelineEventsDetailsItem[][]>((acc, curr) => {
acc.push(getDataFromFieldsHits(curr as Fields, dotField, fieldCategory));
return acc;
}, [])
.flat()
: getDataFromFieldsHits(item, prependField, fieldCategory);
// combine duplicate fields
const flat: Record<string, TimelineEventsDetailsItem> = [
...accumulator,
...nestedFields,
...threatEnrichmentObject,
].reduce(
(acc, f) => ({
...acc,
// acc/flat is hashmap to determine if we already have the field or not without an array iteration
// its converted back to array in return with Object.values
...(acc[f.field] != null
? {
[f.field]: {
...f,
originalValue: acc[f.field].originalValue.includes(f.originalValue[0])
? acc[f.field].originalValue
: [...acc[f.field].originalValue, ...f.originalValue],
values: acc[f.field].values?.includes(f.values?.[0] || '')
? acc[f.field].values
: [...(acc[f.field].values || []), ...(f.values || [])],
},
}
: { [f.field]: f }),
}),
{} as Record<string, TimelineEventsDetailsItem>
);
return Object.values(flat);
}, []);
export const getDataSafety = <A, T>(fn: (args: A) => T, args: A): Promise<T> =>
new Promise((resolve) => setTimeout(() => resolve(fn(args))));

89
x-pack/plugins/security_solution/common/utils/to_array.ts
View file

@ -1,89 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export const toArray = <T = string>(value: T | T[] | null | undefined): T[] =>
Array.isArray(value) ? value : value == null ? [] : [value];
export const toStringArray = <T = string>(value: T | T[] | null): string[] => {
if (Array.isArray(value)) {
return value.reduce<string[]>((acc, v) => {
if (v != null) {
switch (typeof v) {
case 'number':
case 'boolean':
return [...acc, v.toString()];
case 'object':
try {
return [...acc, JSON.stringify(v)];
} catch {
return [...acc, 'Invalid Object'];
}
case 'string':
return [...acc, v];
default:
return [...acc, `${v}`];
}
}
return acc;
}, []);
} else if (value == null) {
return [];
} else if (!Array.isArray(value) && typeof value === 'object') {
try {
return [JSON.stringify(value)];
} catch {
return ['Invalid Object'];
}
} else {
return [`${value}`];
}
};
export const toObjectArrayOfStrings = <T = string>(
value: T | T[] | null
): Array<{
str: string;
isObjectArray?: boolean;
}> => {
if (Array.isArray(value)) {
return value.reduce<
Array<{
str: string;
isObjectArray?: boolean;
}>
>((acc, v) => {
if (v != null) {
switch (typeof v) {
case 'number':
case 'boolean':
return [...acc, { str: v.toString() }];
case 'object':
try {
return [...acc, { str: JSON.stringify(v), isObjectArray: true }]; // need to track when string is not a simple value
} catch {
return [...acc, { str: 'Invalid Object' }];
}
case 'string':
return [...acc, { str: v }];
default:
return [...acc, { str: `${v}` }];
}
}
return acc;
}, []);
} else if (value == null) {
return [];
} else if (!Array.isArray(value) && typeof value === 'object') {
try {
return [{ str: JSON.stringify(value), isObjectArray: true }];
} catch {
return [{ str: 'Invalid Object' }];
}
} else {
return [{ str: `${value}` }];
}
};

2
x-pack/plugins/security_solution/public/flyout/document_details/shared/utils/threat_intelligence.tsx
View file

@ -7,11 +7,11 @@
import { groupBy, isObject } from 'lodash';
import type { TimelineEventsDetailsItem } from '@kbn/timelines-plugin/common';
import { getDataFromFieldsHits } from '@kbn/timelines-plugin/common';
import { i18n } from '@kbn/i18n';
import type { ThreatDetailsRow } from '../../left/components/threat_details_view_enrichment_accordion';
import type { CtiEnrichment, EventFields } from '../../../../../common/search_strategy';
import { isValidEventField } from '../../../../../common/search_strategy';
import { getDataFromFieldsHits } from '../../../../../common/utils/field_formatters';
import {
DEFAULT_INDICATOR_SOURCE_PATH,
ENRICHMENT_DESTINATION_PATH,

4
x-pack/plugins/security_solution/server/search_strategy/helpers/format_response_object_values.ts
View file

@ -7,9 +7,7 @@
import { mapValues, isObject, isArray } from 'lodash/fp';
import { set } from '@kbn/safer-lodash-set';
import { toArray } from '../../../common/utils/to_array';
import { isGeoField } from '../../../common/utils/field_formatters';
import { toArray, isGeoField } from '@kbn/timelines-plugin/common';
export const mapObjectValuesToStringArray = (object: object): object =>
mapValues((o) => {

2
x-pack/plugins/security_solution/server/search_strategy/helpers/get_flattened_fields.ts
View file

@ -6,7 +6,7 @@
*/
import { set } from '@kbn/safer-lodash-set';
import { get, isEmpty } from 'lodash/fp';
import { toObjectArrayOfStrings } from '../../../common/utils/to_array';
import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common';
export function getFlattenedFields<T>(
fields: string[],

2
x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts
View file

@ -8,12 +8,12 @@
import { set } from '@kbn/safer-lodash-set/fp';
import { get, has } from 'lodash/fp';
import { hostFieldsMap } from '@kbn/securitysolution-ecs';
import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common';
import type {
HostAggEsItem,
HostsEdges,
HostValue,
} from '../../../../../../common/search_strategy/security_solution/hosts';
import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array';
export const HOSTS_FIELDS: readonly string[] = [
'_id',

2
x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts
View file

@ -13,6 +13,7 @@ import type {
SavedObjectsClientContract,
} from '@kbn/core/server';
import { hostFieldsMap } from '@kbn/securitysolution-ecs';
import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common';
import { Direction } from '../../../../../../common/search_strategy/common';
import type {
AggregationRequest,
@ -22,7 +23,6 @@ import type {
HostItem,
HostValue,
} from '../../../../../../common/search_strategy/security_solution/hosts';
import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array';
import type { EndpointAppContext } from '../../../../../endpoint/types';
import { getPendingActionsSummary } from '../../../../../endpoint/services';

2
x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts
View file

@ -8,7 +8,7 @@
import { get, getOr, isEmpty } from 'lodash/fp';
import { set } from '@kbn/safer-lodash-set/fp';
import { sourceFieldsMap, hostFieldsMap } from '@kbn/securitysolution-ecs';
import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array';
import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common';
import type {
AuthenticationsEdges,
AuthenticationHit,

2
x-pack/plugins/timelines/common/index.ts
View file

@ -67,3 +67,5 @@ export type {
} from './search_strategy';
export { Direction, EntityType, EMPTY_BROWSER_FIELDS } from './search_strategy';
export { getDataFromFieldsHits, toArray, isGeoField, toObjectArrayOfStrings } from './utils';

35
x-pack/plugins/timelines/common/utils/field_formatters.test.ts
View file

@ -7,7 +7,7 @@
import { eventDetailsFormattedFields, eventHit } from '@kbn/securitysolution-t-grid';
import { EventHit } from '../search_strategy';
import { getDataFromFieldsHits, getDataSafety } from './field_formatters';
import { getDataFromFieldsHits } from './field_formatters';
describe('Events Details Helpers', () => {
const fields: EventHit['fields'] = eventHit.fields;
@ -84,7 +84,7 @@ describe('Events Details Helpers', () => {
},
];
const result = getDataFromFieldsHits(whackFields);
expect(result).toEqual(whackResultFields);
expect(result).toMatchObject(whackResultFields);
});
it('flattens alert parameters', () => {
const ruleParameterFields = {
@ -191,7 +191,7 @@ describe('Events Details Helpers', () => {
];
const result = getDataFromFieldsHits(ruleParameterFields);
expect(result).toEqual(ruleParametersResultFields);
expect(result).toMatchObject(ruleParametersResultFields);
});
it('get data from threat enrichments', () => {
@ -546,6 +546,17 @@ describe('Events Details Helpers', () => {
originalValue: ['495ad7a7-316e-4544-8a0f-9c098daee76e'],
values: ['495ad7a7-316e-4544-8a0f-9c098daee76e'],
},
{
category: 'threat',
field: 'threat.enrichments',
isObjectArray: true,
originalValue: [
'{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}',
],
values: [
'{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}',
],
},
{
category: 'threat',
field: 'threat.enrichments.matched.field',
@ -581,25 +592,9 @@ describe('Events Details Helpers', () => {
originalValue: ['a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3'],
values: ['a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3'],
},
{
category: 'threat',
field: 'threat.enrichments',
isObjectArray: true,
originalValue: [
'{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}',
],
values: [
'{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}',
],
},
];
const result = getDataFromFieldsHits(data);
expect(result).toEqual(ruleParametersResultFields);
expect(result).toMatchObject(ruleParametersResultFields);
});
});
it('#getDataSafety', async () => {
const result = await getDataSafety(getDataFromFieldsHits, fields);
expect(result).toEqual(resultFields);
});
});

228
x-pack/plugins/timelines/common/utils/field_formatters.ts
View file

@ -8,9 +8,15 @@
import { isEmpty } from 'lodash/fp';
import { ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils';
import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map';
import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map';
import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils';
import {
ecsFieldMap,
EcsFieldMap,
} from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map';
import {
technicalRuleFieldMap,
TechnicalRuleFieldMap,
} from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map';
import { legacyExperimentalFieldMap, ExperimentalRuleFieldMap } from '@kbn/alerts-as-data-utils';
import { Fields, TimelineEventsDetailsItem } from '../search_strategy';
import { toObjectArrayOfStrings, toStringArray } from './to_array';
import { ENRICHMENT_DESTINATION_PATH } from '../constants';
@ -51,117 +57,141 @@ export const isRuleParametersFieldOrSubfield = (field: string, prependField?: st
export const isThreatEnrichmentFieldOrSubfield = (field: string, prependField?: string) =>
prependField?.includes(ENRICHMENT_DESTINATION_PATH) || field === ENRICHMENT_DESTINATION_PATH;
// Helper functions
const createFieldItem = (
fieldCategory: string,
field: string,
values: string[],
isObjectArray: boolean
): TimelineEventsDetailsItem => ({
category: fieldCategory,
field,
values,
originalValue: values,
isObjectArray,
});
const processGeoField = (
field: string,
item: unknown[],
fieldCategory: string
): TimelineEventsDetailsItem => {
const formattedLocation = formatGeoLocation(item);
return createFieldItem(fieldCategory, field, formattedLocation, true);
};
const processSimpleField = (
dotField: string,
strArr: string[],
isObjectArray: boolean,
fieldCategory: string
): TimelineEventsDetailsItem => createFieldItem(fieldCategory, dotField, strArr, isObjectArray);
const processNestedFields = (
item: unknown,
dotField: string,
fieldCategory: string,
prependDotField: boolean
): TimelineEventsDetailsItem[] => {
if (Array.isArray(item)) {
return item.flatMap((curr) =>
getDataFromFieldsHits(curr as Fields, prependDotField ? dotField : undefined, fieldCategory)
);
}
return getDataFromFieldsHits(
item as Fields,
prependDotField ? dotField : undefined,
fieldCategory
);
};
type DisjointFieldNames = 'ecs.version' | 'event.action' | 'event.kind' | 'event.original';
// Memoized field maps
const fieldMaps: EcsFieldMap &
Omit<TechnicalRuleFieldMap, DisjointFieldNames> &
ExperimentalRuleFieldMap = {
...technicalRuleFieldMap,
...ecsFieldMap,
...legacyExperimentalFieldMap,
};
export const getDataFromFieldsHits = (
fields: Fields,
prependField?: string,
prependFieldCategory?: string
): TimelineEventsDetailsItem[] =>
Object.keys(fields).reduce<TimelineEventsDetailsItem[]>((accumulator, field) => {
): TimelineEventsDetailsItem[] => {
const resultMap = new Map<string, TimelineEventsDetailsItem>();
const fieldNames = Object.keys(fields);
for (let i = 0; i < fieldNames.length; i++) {
const field = fieldNames[i];
const item: unknown[] = fields[field];
const fieldCategory =
prependFieldCategory != null ? prependFieldCategory : getFieldCategory(field);
const fieldCategory = prependFieldCategory ?? getFieldCategory(field);
const dotField = prependField ? `${prependField}.${field}` : field;
// Handle geo fields
if (isGeoField(field)) {
return [
...accumulator,
{
category: fieldCategory,
field,
values: formatGeoLocation(item),
originalValue: formatGeoLocation(item),
isObjectArray: true, // important for UI
},
];
const geoItem = processGeoField(field, item, fieldCategory);
resultMap.set(field, geoItem);
// eslint-disable-next-line no-continue
continue;
}
const objArrStr = toObjectArrayOfStrings(item);
const strArr = objArrStr.map(({ str }) => str);
const isObjectArray = objArrStr.some((o) => o.isObjectArray);
const dotField = prependField ? `${prependField}.${field}` : field;
// return simple field value (non-ecs object, non-array)
if (
!isObjectArray ||
(Object.keys({
...ecsFieldMap,
...technicalRuleFieldMap,
...legacyExperimentalFieldMap,
}).find((ecsField) => ecsField === field) === undefined &&
!isRuleParametersFieldOrSubfield(field, prependField))
) {
return [
...accumulator,
{
category: fieldCategory,
field: dotField,
values: strArr,
originalValue: strArr,
isObjectArray,
},
];
const isEcsField = fieldMaps[field as keyof typeof fieldMaps] !== undefined;
const isRuleParameters = isRuleParametersFieldOrSubfield(field, prependField);
// Handle simple fields
if (!isObjectArray || (!isEcsField && !isRuleParameters)) {
const simpleItem = processSimpleField(dotField, strArr, isObjectArray, fieldCategory);
resultMap.set(dotField, simpleItem);
// eslint-disable-next-line no-continue
continue;
}
const threatEnrichmentObject = isThreatEnrichmentFieldOrSubfield(field, prependField)
? [
{
category: fieldCategory,
field: dotField,
values: strArr,
originalValue: strArr,
isObjectArray,
},
]
: [];
// format nested fields
let nestedFields: TimelineEventsDetailsItem[] = [];
if (isRuleParametersFieldOrSubfield(field, prependField)) {
nestedFields = Array.isArray(item)
? item
.reduce<TimelineEventsDetailsItem[][]>((acc, curr) => {
acc.push(getDataFromFieldsHits(curr as Fields, dotField, fieldCategory));
return acc;
}, [])
.flat()
: getDataFromFieldsHits(item, dotField, fieldCategory);
} else {
nestedFields = Array.isArray(item)
? item
.reduce<TimelineEventsDetailsItem[][]>((acc, curr) => {
acc.push(getDataFromFieldsHits(curr as Fields, dotField, fieldCategory));
return acc;
}, [])
.flat()
: getDataFromFieldsHits(item, prependField, fieldCategory);
// Handle threat enrichment
if (isThreatEnrichmentFieldOrSubfield(field, prependField)) {
const enrichmentItem = createFieldItem(fieldCategory, dotField, strArr, isObjectArray);
resultMap.set(dotField, enrichmentItem);
}
// combine duplicate fields
const flat: Record<string, TimelineEventsDetailsItem> = [
...accumulator,
...nestedFields,
...threatEnrichmentObject,
].reduce(
(acc, f) => ({
...acc,
// acc/flat is hashmap to determine if we already have the field or not without an array iteration
// its converted back to array in return with Object.values
...(acc[f.field] != null
? {
[f.field]: {
...f,
originalValue: acc[f.field].originalValue.includes(f.originalValue[0])
? acc[f.field].originalValue
: [...acc[f.field].originalValue, ...f.originalValue],
values: acc[f.field].values?.includes(f.values?.[0] || '')
? acc[f.field].values
: [...(acc[f.field].values || []), ...(f.values || [])],
},
}
: { [f.field]: f }),
}),
{} as Record<string, TimelineEventsDetailsItem>
// Process nested fields
const nestedFields = processNestedFields(
item,
dotField,
fieldCategory,
isRuleParameters || isThreatEnrichmentFieldOrSubfield(field, prependField)
);
// Merge results
for (const nestedItem of nestedFields) {
const existing = resultMap.get(nestedItem.field);
return Object.values(flat);
}, []);
if (!existing) {
resultMap.set(nestedItem.field, nestedItem);
// eslint-disable-next-line no-continue
continue;
}
export const getDataSafety = <A, T>(fn: (args: A) => T, args: A): Promise<T> =>
new Promise((resolve) => setTimeout(() => resolve(fn(args))));
// Merge values and originalValue arrays
const mergedValues = existing.values?.includes(nestedItem.values?.[0] || '')
? existing.values
: [...(existing.values || []), ...(nestedItem.values || [])];
const mergedOriginal = existing.originalValue.includes(nestedItem.originalValue[0])
? existing.originalValue
: [...existing.originalValue, ...nestedItem.originalValue];
resultMap.set(nestedItem.field, {
...nestedItem,
values: mergedValues,
originalValue: mergedOriginal,
});
}
}
return Array.from(resultMap.values());
};

9
x-pack/plugins/timelines/common/utils/index.ts Normal file
View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { getDataFromFieldsHits, isGeoField } from './field_formatters';
export { toArray, toObjectArrayOfStrings } from './to_array';

106
x-pack/plugins/timelines/common/utils/to_array.ts
View file

@ -5,83 +5,55 @@
* 2.0.
*/
export const toArray = <T = string>(value: T | T[] | null): T[] =>
Array.isArray(value) ? value : value == null ? [] : [value];
export const toStringArray = <T = string>(value: T | T[] | null): string[] => {
if (Array.isArray(value)) {
return value.reduce<string[]>((acc, v) => {
if (v != null) {
switch (typeof v) {
case 'number':
case 'boolean':
return [...acc, v.toString()];
case 'object':
try {
return [...acc, JSON.stringify(v)];
} catch {
return [...acc, 'Invalid Object'];
}
case 'string':
return [...acc, v];
default:
return [...acc, `${v}`];
}
export const toArray = <T>(value: T | T[] | null | undefined): T[] =>
value == null ? [] : Array.isArray(value) ? value : [value];
export const toStringArray = <T>(value: T | T[] | null): string[] => {
if (value == null) return [];
const arr = Array.isArray(value) ? value : [value];
return arr.reduce<string[]>((acc, v) => {
if (v == null) return acc;
if (typeof v === 'object') {
try {
acc.push(JSON.stringify(v));
} catch {
acc.push('Invalid Object');
}
return acc;
}, []);
} else if (value == null) {
return [];
} else if (!Array.isArray(value) && typeof value === 'object') {
try {
return [JSON.stringify(value)];
} catch {
return ['Invalid Object'];
}
} else {
return [`${value}`];
}
acc.push(String(v));
return acc;
}, []);
};
export const toObjectArrayOfStrings = <T = string>(
export const toObjectArrayOfStrings = <T>(
value: T | T[] | null
): Array<{
str: string;
isObjectArray?: boolean;
}> => {
if (Array.isArray(value)) {
return value.reduce<
Array<{
str: string;
isObjectArray?: boolean;
}>
>((acc, v) => {
if (v != null) {
switch (typeof v) {
case 'number':
case 'boolean':
return [...acc, { str: v.toString() }];
case 'object':
try {
return [...acc, { str: JSON.stringify(v), isObjectArray: true }]; // need to track when string is not a simple value
} catch {
return [...acc, { str: 'Invalid Object' }];
}
case 'string':
return [...acc, { str: v }];
default:
return [...acc, { str: `${v}` }];
}
if (value == null) return [];
const arr = Array.isArray(value) ? value : [value];
return arr.reduce<Array<{ str: string; isObjectArray?: boolean }>>((acc, v) => {
if (v == null) return acc;
if (typeof v === 'object') {
try {
acc.push({
str: JSON.stringify(v),
isObjectArray: true,
});
} catch {
acc.push({ str: 'Invalid Object' });
}
return acc;
}, []);
} else if (value == null) {
return [];
} else if (!Array.isArray(value) && typeof value === 'object') {
try {
return [{ str: JSON.stringify(value), isObjectArray: true }];
} catch {
return [{ str: 'Invalid Object' }];
}
} else {
return [{ str: `${value}` }];
}
acc.push({ str: String(v) });
return acc;
}, []);
};

69
x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts
View file

@ -17,7 +17,6 @@ import {
} from '../../../../common/search_strategy';
import { TimelineEqlResponse } from '../../../../common/search_strategy/timeline/events/eql';
import { inspectStringifyObject } from '../../../utils/build_query';
import { TIMELINE_EVENTS_FIELDS } from '../factory/helpers/constants';
import { formatTimelineData } from '../factory/helpers/format_timeline_data';
export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record<string, unknown> => {
@ -68,38 +67,38 @@ export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record<string,
},
};
};
const parseSequences = async (sequences: Array<EqlSequence<unknown>>, fieldRequested: string[]) =>
sequences.reduce<Promise<TimelineEdges[]>>(async (acc, sequence, sequenceIndex) => {
const parseSequences = async (sequences: Array<EqlSequence<unknown>>, fieldRequested: string[]) => {
let result: TimelineEdges[] = [];
for (const [sequenceIndex, sequence] of sequences.entries()) {
const sequenceParentId = sequence.events[0]?._id ?? null;
const data = await acc;
const allData = await Promise.all(
sequence.events.map(async (event, eventIndex) => {
const item = await formatTimelineData(
fieldRequested,
TIMELINE_EVENTS_FIELDS,
event as EventHit
);
return Promise.resolve({
...item,
node: {
...item.node,
ecs: {
...item.node.ecs,
...(sequenceParentId != null
? {
eql: {
parentId: sequenceParentId,
sequenceNumber: `${sequenceIndex}-${eventIndex}`,
},
}
: {}),
},
},
});
})
const formattedEvents = await formatTimelineData(
sequence.events as EventHit[],
fieldRequested,
false
);
return Promise.resolve([...data, ...allData]);
}, Promise.resolve([]));
const eventsWithEql = formattedEvents.map((item, eventIndex) => ({
...item,
node: {
...item.node,
ecs: {
...item.node.ecs,
...(sequenceParentId && {
eql: {
parentId: sequenceParentId,
sequenceNumber: `${sequenceIndex}-${eventIndex}`,
},
}),
},
},
}));
result = result.concat(eventsWithEql);
}
return result;
};
export const parseEqlResponse = async (
options: TimelineEqlRequestOptions,
@ -116,10 +115,10 @@ export const parseEqlResponse = async (
if (response.rawResponse.hits.sequences !== undefined) {
edges = await parseSequences(response.rawResponse.hits.sequences, options.fieldRequested);
} else if (response.rawResponse.hits.events !== undefined) {
edges = await Promise.all(
response.rawResponse.hits.events.map(async (event) =>
formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, event as EventHit)
)
edges = await formatTimelineData(
response.rawResponse.hits.events as EventHit[],
options.fieldRequested,
false
);
}

14
x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts
View file

@ -14,13 +14,11 @@ import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants
import {
EventHit,
TimelineEventsAllStrategyResponse,
TimelineEdges,
} from '../../../../../../common/search_strategy';
import { TimelineFactory } from '../../types';
import { buildTimelineEventsAllQuery } from './query.events_all.dsl';
import { inspectStringifyObject } from '../../../../../utils/build_query';
import { formatTimelineData } from '../../helpers/format_timeline_data';
import { TIMELINE_EVENTS_FIELDS } from '../../helpers/constants';
export const timelineEventsAll: TimelineFactory<TimelineEventsQueries.all> = {
buildDsl: ({ authFilter, ...options }) => {
@ -54,14 +52,10 @@ export const timelineEventsAll: TimelineFactory<TimelineEventsQueries.all> = {
fieldRequested = [...new Set(fieldsReturned)];
}
const edges: TimelineEdges[] = await Promise.all(
hits.map((hit) =>
formatTimelineData(
fieldRequested,
options.excludeEcsData ? [] : TIMELINE_EVENTS_FIELDS,
hit as EventHit
)
)
const edges = await formatTimelineData(
hits as EventHit[],
fieldRequested,
options.excludeEcsData ?? false
);
const consumers = producerBuckets.reduce(

11
x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/index.ts
View file

@ -12,15 +12,11 @@ import { TimelineEventsQueries } from '../../../../../../common/api/search_strat
import {
EventHit,
TimelineEventsDetailsStrategyResponse,
TimelineEventsDetailsItem,
} from '../../../../../../common/search_strategy';
import { inspectStringifyObject } from '../../../../../utils/build_query';
import { TimelineFactory } from '../../types';
import { buildTimelineDetailsQuery } from './query.events_details.dsl';
import {
getDataFromFieldsHits,
getDataSafety,
} from '../../../../../../common/utils/field_formatters';
import { getDataFromFieldsHits } from '../../../../../../common/utils/field_formatters';
import { buildEcsObjects } from '../../helpers/build_ecs_objects';
export const timelineEventsDetails: TimelineFactory<TimelineEventsQueries.details> = {
@ -57,10 +53,7 @@ export const timelineEventsDetails: TimelineFactory<TimelineEventsQueries.detail
};
}
const fieldsData = await getDataSafety<EventHit['fields'], TimelineEventsDetailsItem[]>(
getDataFromFieldsHits,
merge(fields, hitsData)
);
const fieldsData = getDataFromFieldsHits(merge(fields, hitsData));
const rawEventData = response.rawResponse.hits.hits[0];
const ecs = buildEcsObjects(rawEventData as EventHit);

729
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.test.ts
View file

@ -7,12 +7,12 @@
import { eventHit } from '@kbn/securitysolution-t-grid';
import { EventHit } from '../../../../../common/search_strategy';
import { TIMELINE_EVENTS_FIELDS } from './constants';
import { formatTimelineData } from './format_timeline_data';
describe('formatTimelineData', () => {
it('should properly format the timeline data', async () => {
const res = await formatTimelineData(
[eventHit],
[
'@timestamp',
'host.name',
@ -21,187 +21,188 @@ describe('formatTimelineData', () => {
'source.geo.location',
'threat.enrichments.matched.field',
],
TIMELINE_EVENTS_FIELDS,
eventHit
false
);
expect(res).toEqual({
cursor: {
tiebreaker: 'beats-ci-immutable-ubuntu-1804-1605624279743236239',
value: '1605624488922',
},
node: {
_id: 'tkCt1nUBaEgqnrVSZ8R_',
_index: 'auditbeat-7.8.0-2020.11.05-000003',
data: [
{
field: '@timestamp',
value: ['2020-11-17T14:48:08.922Z'],
},
{
field: 'host.name',
value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
},
{
field: 'threat.enrichments.matched.field',
value: [
'matched_field',
'other_matched_field',
'matched_field_2',
'host.name',
'host.hostname',
'host.architecture',
],
},
{
field: 'source.geo.location',
value: [`{"lon":118.7778,"lat":32.0617}`],
},
],
ecs: {
'@timestamp': ['2020-11-17T14:48:08.922Z'],
expect(res).toEqual([
{
cursor: {
tiebreaker: 'beats-ci-immutable-ubuntu-1804-1605624279743236239',
value: '1605624488922',
},
node: {
_id: 'tkCt1nUBaEgqnrVSZ8R_',
_index: 'auditbeat-7.8.0-2020.11.05-000003',
agent: {
type: ['auditbeat'],
},
event: {
action: ['process_started'],
category: ['process'],
dataset: ['process'],
kind: ['event'],
module: ['system'],
type: ['start'],
},
host: {
id: ['e59991e835905c65ed3e455b33e13bd6'],
ip: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'],
name: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
os: {
family: ['debian'],
data: [
{
field: '@timestamp',
value: ['2020-11-17T14:48:08.922Z'],
},
},
message: ['Process go (PID: 4313) by user jenkins STARTED'],
process: {
args: ['go', 'vet', './...'],
entity_id: ['Z59cIkAAIw8ZoK0H'],
executable: [
'/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go',
],
hash: {
sha1: ['1eac22336a41e0660fb302add9d97daa2bcc7040'],
{
field: 'host.name',
value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
},
name: ['go'],
pid: ['4313'],
ppid: ['3977'],
working_directory: [
'/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat',
],
},
timestamp: '2020-11-17T14:48:08.922Z',
user: {
name: ['jenkins'],
},
threat: {
enrichments: [
{
feed: { name: [] },
indicator: {
provider: ['yourself'],
reference: [],
},
matched: {
atomic: ['matched_atomic'],
field: ['matched_field', 'other_matched_field'],
type: [],
},
{
field: 'threat.enrichments.matched.field',
value: [
'matched_field',
'other_matched_field',
'matched_field_2',
'host.name',
'host.hostname',
'host.architecture',
],
},
{
field: 'source.geo.location',
value: [`{"lon":118.7778,"lat":32.0617}`],
},
],
ecs: {
'@timestamp': ['2020-11-17T14:48:08.922Z'],
_id: 'tkCt1nUBaEgqnrVSZ8R_',
_index: 'auditbeat-7.8.0-2020.11.05-000003',
agent: {
type: ['auditbeat'],
},
event: {
action: ['process_started'],
category: ['process'],
dataset: ['process'],
kind: ['event'],
module: ['system'],
type: ['start'],
},
host: {
id: ['e59991e835905c65ed3e455b33e13bd6'],
ip: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'],
name: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
os: {
family: ['debian'],
},
{
feed: { name: [] },
indicator: {
provider: ['other_you'],
reference: [],
},
matched: {
atomic: ['matched_atomic_2'],
field: ['matched_field_2'],
type: [],
},
},
message: ['Process go (PID: 4313) by user jenkins STARTED'],
process: {
args: ['go', 'vet', './...'],
entity_id: ['Z59cIkAAIw8ZoK0H'],
executable: [
'/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go',
],
hash: {
sha1: ['1eac22336a41e0660fb302add9d97daa2bcc7040'],
},
{
feed: {
name: [],
name: ['go'],
pid: ['4313'],
ppid: ['3977'],
working_directory: [
'/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat',
],
},
timestamp: '2020-11-17T14:48:08.922Z',
user: {
name: ['jenkins'],
},
threat: {
enrichments: [
{
feed: { name: [] },
indicator: {
provider: ['yourself'],
reference: [],
},
matched: {
atomic: ['matched_atomic'],
field: ['matched_field', 'other_matched_field'],
type: [],
},
},
indicator: {
provider: [],
reference: [],
{
feed: { name: [] },
indicator: {
provider: ['other_you'],
reference: [],
},
matched: {
atomic: ['matched_atomic_2'],
field: ['matched_field_2'],
type: [],
},
},
matched: {
atomic: ['MacBook-Pro-de-Gloria.local'],
field: ['host.name'],
type: ['indicator_match_rule'],
{
feed: {
name: [],
},
indicator: {
provider: [],
reference: [],
},
matched: {
atomic: ['MacBook-Pro-de-Gloria.local'],
field: ['host.name'],
type: ['indicator_match_rule'],
},
},
},
{
feed: {
name: [],
{
feed: {
name: [],
},
indicator: {
provider: [],
reference: [],
},
matched: {
atomic: ['MacBook-Pro-de-Gloria.local'],
field: ['host.hostname'],
type: ['indicator_match_rule'],
},
},
indicator: {
provider: [],
reference: [],
{
feed: {
name: [],
},
indicator: {
provider: [],
reference: [],
},
matched: {
atomic: ['x86_64'],
field: ['host.architecture'],
type: ['indicator_match_rule'],
},
},
matched: {
atomic: ['MacBook-Pro-de-Gloria.local'],
field: ['host.hostname'],
type: ['indicator_match_rule'],
{
feed: {
name: [],
},
indicator: {
provider: [],
reference: [],
},
matched: {
atomic: ['MacBook-Pro-de-Gloria.local'],
field: ['host.name'],
type: ['indicator_match_rule'],
},
},
},
{
feed: {
name: [],
{
feed: {
name: [],
},
indicator: {
provider: [],
reference: [],
},
matched: {
atomic: ['MacBook-Pro-de-Gloria.local'],
field: ['host.hostname'],
type: ['indicator_match_rule'],
},
},
indicator: {
provider: [],
reference: [],
},
matched: {
atomic: ['x86_64'],
field: ['host.architecture'],
type: ['indicator_match_rule'],
},
},
{
feed: {
name: [],
},
indicator: {
provider: [],
reference: [],
},
matched: {
atomic: ['MacBook-Pro-de-Gloria.local'],
field: ['host.name'],
type: ['indicator_match_rule'],
},
},
{
feed: {
name: [],
},
indicator: {
provider: [],
reference: [],
},
matched: {
atomic: ['MacBook-Pro-de-Gloria.local'],
field: ['host.hostname'],
type: ['indicator_match_rule'],
},
},
],
],
},
},
},
},
});
]);
});
it('should properly format the rule signal results', async () => {
@ -240,57 +241,61 @@ describe('formatTimelineData', () => {
expect(
await formatTimelineData(
[response],
['@timestamp', 'host.name', 'destination.ip', 'source.ip'],
TIMELINE_EVENTS_FIELDS,
response
false
)
).toEqual({
cursor: {
tiebreaker: null,
value: '',
},
node: {
_id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562',
_index: '.siem-signals-patrykkopycinski-default-000007',
data: [
{
field: '@timestamp',
value: ['2021-01-09T13:41:40.517Z'],
},
],
ecs: {
'@timestamp': ['2021-01-09T13:41:40.517Z'],
timestamp: '2021-01-09T13:41:40.517Z',
).toEqual([
{
cursor: {
tiebreaker: null,
value: '',
},
node: {
_id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562',
_index: '.siem-signals-patrykkopycinski-default-000007',
event: {
kind: ['signal'],
},
kibana: {
alert: {
original_time: ['2021-01-09T13:39:32.595Z'],
workflow_status: ['open'],
threshold_result: ['{"count":10000,"value":"2a990c11-f61b-4c8e-b210-da2574e9f9db"}'],
severity: ['low'],
risk_score: ['21'],
rule: {
building_block_type: [],
exceptions_list: [],
from: ['now-360s'],
uuid: ['696c24e0-526d-11eb-836c-e1620268b945'],
name: ['Threshold test'],
to: ['now'],
type: ['threshold'],
version: ['1'],
timeline_id: [],
timeline_title: [],
note: [],
data: [
{
field: '@timestamp',
value: ['2021-01-09T13:41:40.517Z'],
},
],
ecs: {
'@timestamp': ['2021-01-09T13:41:40.517Z'],
timestamp: '2021-01-09T13:41:40.517Z',
_id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562',
_index: '.siem-signals-patrykkopycinski-default-000007',
event: {
kind: ['signal'],
},
kibana: {
alert: {
original_time: ['2021-01-09T13:39:32.595Z'],
workflow_status: ['open'],
threshold_result: [
'{"count":10000,"value":"2a990c11-f61b-4c8e-b210-da2574e9f9db"}',
],
severity: ['low'],
risk_score: ['21'],
rule: {
building_block_type: [],
exceptions_list: [],
from: ['now-360s'],
uuid: ['696c24e0-526d-11eb-836c-e1620268b945'],
name: ['Threshold test'],
to: ['now'],
type: ['threshold'],
version: ['1'],
timeline_id: [],
timeline_title: [],
note: [],
},
},
},
},
},
},
});
]);
});
it('should properly format the inventory rule signal results', async () => {
@ -347,6 +352,7 @@ describe('formatTimelineData', () => {
expect(
await formatTimelineData(
[response],
[
'kibana.alert.status',
'@timestamp',
@ -376,168 +382,169 @@ describe('formatTimelineData', () => {
'event.kind',
'kibana.alert.rule.parameters',
],
TIMELINE_EVENTS_FIELDS,
response
false
)
).toEqual({
cursor: {
tiebreaker: null,
value: '',
},
node: {
_id: '3fef4a4c-3d96-4e79-b4e5-158a0461d577',
_index: '.internal.alerts-observability.metrics.alerts-default-000001',
data: [
{
field: 'kibana.alert.rule.consumer',
value: ['infrastructure'],
},
{
field: '@timestamp',
value: ['2022-07-21T22:38:57.888Z'],
},
{
field: 'kibana.alert.workflow_status',
value: ['open'],
},
{
field: 'kibana.alert.reason',
value: [
'CPU usage is 37.8% in the last 1 day for gke-edge-oblt-pool-1-9a60016d-7dvq. Alert when > 10%.',
],
},
{
field: 'kibana.alert.rule.name',
value: ['test 1212'],
},
{
field: 'kibana.alert.rule.uuid',
value: ['15d82f10-0926-11ed-bece-6b0c033d0075'],
},
{
field: 'kibana.alert.rule.parameters.sourceId',
value: ['default'],
},
{
field: 'kibana.alert.rule.parameters.nodeType',
value: ['host'],
},
{
field: 'kibana.alert.rule.parameters.criteria.comparator',
value: ['>'],
},
{
field: 'kibana.alert.rule.parameters.criteria.timeSize',
value: ['1'],
},
{
field: 'kibana.alert.rule.parameters.criteria.metric',
value: ['cpu'],
},
{
field: 'kibana.alert.rule.parameters.criteria.threshold',
value: ['10'],
},
{
field: 'kibana.alert.rule.parameters.criteria.customMetric.aggregation',
value: ['avg'],
},
{
field: 'kibana.alert.rule.parameters.criteria.customMetric.id',
value: ['alert-custom-metric'],
},
{
field: 'kibana.alert.rule.parameters.criteria.customMetric.field',
value: [''],
},
{
field: 'kibana.alert.rule.parameters.criteria.customMetric.type',
value: ['custom'],
},
{
field: 'kibana.alert.rule.parameters.criteria.timeUnit',
value: ['d'],
},
{
field: 'event.action',
value: ['active'],
},
{
field: 'event.kind',
value: ['signal'],
},
{
field: 'kibana.alert.status',
value: ['active'],
},
{
field: 'kibana.alert.duration.us',
value: ['9502040000'],
},
{
field: 'kibana.alert.rule.category',
value: ['Inventory'],
},
{
field: 'kibana.alert.uuid',
value: ['3fef4a4c-3d96-4e79-b4e5-158a0461d577'],
},
{
field: 'kibana.alert.start',
value: ['2022-07-21T20:00:35.848Z'],
},
{
field: 'kibana.alert.rule.producer',
value: ['infrastructure'],
},
{
field: 'kibana.alert.rule.rule_type_id',
value: ['metrics.alert.inventory.threshold'],
},
{
field: 'kibana.alert.instance.id',
value: ['gke-edge-oblt-pool-1-9a60016d-7dvq'],
},
{
field: 'kibana.alert.rule.execution.uuid',
value: ['37498c42-0190-4a83-adfa-c7e5f817f977'],
},
{
field: 'kibana.space_ids',
value: ['default'],
},
{
field: 'kibana.version',
value: ['8.4.0'],
},
],
ecs: {
'@timestamp': ['2022-07-21T22:38:57.888Z'],
).toEqual([
{
cursor: {
tiebreaker: null,
value: '',
},
node: {
_id: '3fef4a4c-3d96-4e79-b4e5-158a0461d577',
_index: '.internal.alerts-observability.metrics.alerts-default-000001',
event: {
action: ['active'],
kind: ['signal'],
},
kibana: {
alert: {
reason: [
data: [
{
field: 'kibana.alert.rule.consumer',
value: ['infrastructure'],
},
{
field: '@timestamp',
value: ['2022-07-21T22:38:57.888Z'],
},
{
field: 'kibana.alert.workflow_status',
value: ['open'],
},
{
field: 'kibana.alert.reason',
value: [
'CPU usage is 37.8% in the last 1 day for gke-edge-oblt-pool-1-9a60016d-7dvq. Alert when > 10%.',
],
rule: {
consumer: ['infrastructure'],
name: ['test 1212'],
uuid: ['15d82f10-0926-11ed-bece-6b0c033d0075'],
parameters: [
'{"sourceId":"default","nodeType":"host","criteria":[{"comparator":">","timeSize":1,"metric":"cpu","threshold":[10],"customMetric":{"aggregation":"avg","id":"alert-custom-metric","field":"","type":"custom"},"timeUnit":"d"}]}',
],
},
workflow_status: ['open'],
},
{
field: 'kibana.alert.rule.name',
value: ['test 1212'],
},
{
field: 'kibana.alert.rule.uuid',
value: ['15d82f10-0926-11ed-bece-6b0c033d0075'],
},
{
field: 'kibana.alert.rule.parameters.sourceId',
value: ['default'],
},
{
field: 'kibana.alert.rule.parameters.nodeType',
value: ['host'],
},
{
field: 'kibana.alert.rule.parameters.criteria.comparator',
value: ['>'],
},
{
field: 'kibana.alert.rule.parameters.criteria.timeSize',
value: ['1'],
},
{
field: 'kibana.alert.rule.parameters.criteria.metric',
value: ['cpu'],
},
{
field: 'kibana.alert.rule.parameters.criteria.threshold',
value: ['10'],
},
{
field: 'kibana.alert.rule.parameters.criteria.customMetric.aggregation',
value: ['avg'],
},
{
field: 'kibana.alert.rule.parameters.criteria.customMetric.id',
value: ['alert-custom-metric'],
},
{
field: 'kibana.alert.rule.parameters.criteria.customMetric.field',
value: [''],
},
{
field: 'kibana.alert.rule.parameters.criteria.customMetric.type',
value: ['custom'],
},
{
field: 'kibana.alert.rule.parameters.criteria.timeUnit',
value: ['d'],
},
{
field: 'event.action',
value: ['active'],
},
{
field: 'event.kind',
value: ['signal'],
},
{
field: 'kibana.alert.status',
value: ['active'],
},
{
field: 'kibana.alert.duration.us',
value: ['9502040000'],
},
{
field: 'kibana.alert.rule.category',
value: ['Inventory'],
},
{
field: 'kibana.alert.uuid',
value: ['3fef4a4c-3d96-4e79-b4e5-158a0461d577'],
},
{
field: 'kibana.alert.start',
value: ['2022-07-21T20:00:35.848Z'],
},
{
field: 'kibana.alert.rule.producer',
value: ['infrastructure'],
},
{
field: 'kibana.alert.rule.rule_type_id',
value: ['metrics.alert.inventory.threshold'],
},
{
field: 'kibana.alert.instance.id',
value: ['gke-edge-oblt-pool-1-9a60016d-7dvq'],
},
{
field: 'kibana.alert.rule.execution.uuid',
value: ['37498c42-0190-4a83-adfa-c7e5f817f977'],
},
{
field: 'kibana.space_ids',
value: ['default'],
},
{
field: 'kibana.version',
value: ['8.4.0'],
},
],
ecs: {
'@timestamp': ['2022-07-21T22:38:57.888Z'],
_id: '3fef4a4c-3d96-4e79-b4e5-158a0461d577',
_index: '.internal.alerts-observability.metrics.alerts-default-000001',
event: {
action: ['active'],
kind: ['signal'],
},
kibana: {
alert: {
reason: [
'CPU usage is 37.8% in the last 1 day for gke-edge-oblt-pool-1-9a60016d-7dvq. Alert when > 10%.',
],
rule: {
consumer: ['infrastructure'],
name: ['test 1212'],
uuid: ['15d82f10-0926-11ed-bece-6b0c033d0075'],
parameters: [
'{"sourceId":"default","nodeType":"host","criteria":[{"comparator":">","timeSize":1,"metric":"cpu","threshold":[10],"customMetric":{"aggregation":"avg","id":"alert-custom-metric","field":"","type":"custom"},"timeUnit":"d"}]}',
],
},
workflow_status: ['open'],
},
},
timestamp: '2022-07-21T22:38:57.888Z',
},
timestamp: '2022-07-21T22:38:57.888Z',
},
},
});
]);
});
});

206
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.ts
View file

@ -5,118 +5,130 @@
* 2.0.
*/
import { get, has, merge, uniq } from 'lodash/fp';
import { EventHit, TimelineEdges, TimelineNonEcsData } from '../../../../../common/search_strategy';
import { get, has } from 'lodash/fp';
import {
EventHit,
TimelineEdges,
TimelineNonEcsData,
EventSource,
} from '../../../../../common/search_strategy';
import { toStringArray } from '../../../../../common/utils/to_array';
import { getDataFromFieldsHits, getDataSafety } from '../../../../../common/utils/field_formatters';
import { getDataFromFieldsHits } from '../../../../../common/utils/field_formatters';
import { getTimestamp } from './get_timestamp';
import { getNestedParentPath } from './get_nested_parent_path';
import { buildObjectRecursive } from './build_object_recursive';
import { ECS_METADATA_FIELDS } from './constants';
import { ECS_METADATA_FIELDS, TIMELINE_EVENTS_FIELDS } from './constants';
export const formatTimelineData = async (
dataFields: readonly string[],
ecsFields: readonly string[],
hit: EventHit
) =>
uniq([...ecsFields, ...dataFields]).reduce<Promise<TimelineEdges>>(
async (acc, fieldName) => {
const flattenedFields: TimelineEdges = await acc;
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
flattenedFields.node._id = hit._id!;
flattenedFields.node._index = hit._index;
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
flattenedFields.node.ecs._id = hit._id!;
flattenedFields.node.ecs.timestamp = getTimestamp(hit);
flattenedFields.node.ecs._index = hit._index;
if (hit.sort && hit.sort.length > 1) {
flattenedFields.cursor.value = hit.sort[0];
flattenedFields.cursor.tiebreaker = hit.sort[1];
}
const waitForIt = await mergeTimelineFieldsWithHit(
fieldName,
flattenedFields,
hit,
dataFields,
ecsFields
);
return Promise.resolve(waitForIt);
},
Promise.resolve({
node: { ecs: { _id: '' }, data: [], _id: '', _index: '' },
cursor: {
value: '',
tiebreaker: null,
},
})
);
const createBaseTimelineEdges = (): TimelineEdges => ({
node: {
ecs: { _id: '' },
data: [],
_id: '',
_index: '',
},
cursor: {
value: '',
tiebreaker: null,
},
});
const getValuesFromFields = async (
function deepMerge(target: EventSource, source: EventSource) {
for (const key in source) {
if (source[key] instanceof Object && key in target) {
deepMerge(target[key], source[key]);
} else {
target[key] = source[key];
}
}
return target;
}
const processMetadataField = (fieldName: string, hit: EventHit): TimelineNonEcsData[] => [
{
field: fieldName,
value: toStringArray(get(fieldName, hit)),
},
];
const processFieldData = (
fieldName: string,
hit: EventHit,
nestedParentFieldName?: string
): Promise<TimelineNonEcsData[]> => {
if (ECS_METADATA_FIELDS.includes(fieldName)) {
return [{ field: fieldName, value: toStringArray(get(fieldName, hit)) }];
}
): TimelineNonEcsData[] => {
const fieldToEval = nestedParentFieldName
? { [nestedParentFieldName]: hit.fields[nestedParentFieldName] }
: { [fieldName]: hit.fields[fieldName] };
let fieldToEval;
if (nestedParentFieldName == null) {
fieldToEval = {
[fieldName]: hit.fields[fieldName],
};
} else {
fieldToEval = {
[nestedParentFieldName]: hit.fields[nestedParentFieldName],
};
}
const formattedData = await getDataSafety(getDataFromFieldsHits, fieldToEval);
return formattedData.reduce((acc: TimelineNonEcsData[], { field, values }) => {
// nested fields return all field values, pick only the one we asked for
const formattedData = getDataFromFieldsHits(fieldToEval);
const fieldsData: TimelineNonEcsData[] = [];
return formattedData.reduce((agg, { field, values }) => {
if (field.includes(fieldName)) {
acc.push({ field, value: values });
agg.push({
field,
value: values,
});
}
return acc;
}, []);
return agg;
}, fieldsData);
};
const mergeTimelineFieldsWithHit = async <T>(
fieldName: string,
flattenedFields: T,
hit: EventHit,
dataFields: readonly string[],
ecsFields: readonly string[]
) => {
if (fieldName != null) {
const nestedParentPath = getNestedParentPath(fieldName, hit.fields);
if (
nestedParentPath != null ||
has(fieldName, hit.fields) ||
ECS_METADATA_FIELDS.includes(fieldName)
) {
const objectWithProperty = {
node: {
...get('node', flattenedFields),
data: dataFields.includes(fieldName)
? [
...get('node.data', flattenedFields),
...(await getValuesFromFields(fieldName, hit, nestedParentPath)),
]
: get('node.data', flattenedFields),
ecs: ecsFields.includes(fieldName)
? {
...get('node.ecs', flattenedFields),
...buildObjectRecursive(fieldName, hit.fields),
}
: get('node.ecs', flattenedFields),
},
};
return merge(flattenedFields, objectWithProperty);
export const formatTimelineData = async (
hits: EventHit[],
fieldRequested: readonly string[],
excludeEcsData: boolean
): Promise<TimelineEdges[]> => {
const ecsFields = excludeEcsData ? [] : TIMELINE_EVENTS_FIELDS;
const uniqueFields = new Set([...ecsFields, ...fieldRequested]);
const dataFieldSet = new Set(fieldRequested);
const ecsFieldSet = new Set(ecsFields);
const results: TimelineEdges[] = new Array(hits.length);
for (let i = 0; i < hits.length; i++) {
const hit = hits[i];
if (hit._id) {
const result = createBaseTimelineEdges();
result.node._id = hit._id;
result.node._index = hit._index;
result.node.ecs._id = hit._id;
result.node.ecs.timestamp = getTimestamp(hit);
result.node.ecs._index = hit._index;
if (hit.sort?.length > 1) {
result.cursor.value = hit.sort[0];
result.cursor.tiebreaker = hit.sort[1];
}
result.node.data = [];
for (const fieldName of uniqueFields) {
const nestedParentPath = getNestedParentPath(fieldName, hit.fields);
const isEcs = ECS_METADATA_FIELDS.includes(fieldName);
if (!nestedParentPath && !has(fieldName, hit.fields) && !isEcs) {
// eslint-disable-next-line no-continue
continue;
}
if (dataFieldSet.has(fieldName)) {
const values = isEcs
? processMetadataField(fieldName, hit)
: processFieldData(fieldName, hit, nestedParentPath);
result.node.data.push(...values);
}
if (ecsFieldSet.has(fieldName)) {
deepMerge(result.node.ecs, buildObjectRecursive(fieldName, hit.fields));
}
}
results[i] = result;
} else {
return flattenedFields;
results[i] = createBaseTimelineEdges();
}
} else {
return flattenedFields;
}
return results;
};