github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Timeline] Rebuild nested fields structure from fields response (#96187)

Browse source 
* First pass at rebuilding nested object structure from fields response

* Always requests TIMELINE_CTI_FIELDS as part of request

This only works for one level of nesting; will be extending tests to
allow for multiple levels momentarily.

* Build objects from arbitrary levels of nesting

This is a recursive implementation, but recursion depth is limited to
the number of levels of nesting, with arguments reducing in size as we
go (i.e. logarithmic)

* Simplify parsing logic, perf improvements

* Order short-circuiting conditions by cost, ascending
* Simplify object building for non-nested objects from fields
  * The non-nested case is the same as the base recursive case, so
    always call our recursive function if building from .fields
* Simplify getNestedParentPath
  * We can do a few simple string comparison rather than building up
    multiple strings/arrays
* Don't call getNestedParentPath unnecessarily, only if we have a field

* Simplify if branching

By definition, nestedParentFieldName can never be equal to fieldName, which means
there are only two branches here.

* Declare/export a more accurate fields type

Each top-level field value can be either an array of leaf values
(unknown[]), or an array of nested fields.

* Remove unnecessary condition

If fieldName is null or undefined, there is no reason to search for it
in dataFields. Looking through the git history this looks to be dead
code as a result of refactoring, as opposed to a legitimate bugfix, so
I'm removing it.

* Fix failing tests

* one was a test failure due to my modifying mock data
* one may have been a legitimate bug where we don't handle a hit without
  a fields response; I need to follow up with Xavier to verify.

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Ryland Herrick 2021-04-12 17:52:42 -05:00 • committed by GitHub
parent 4d593bbc08
commit 39f87f4560
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 260 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

4
x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts
View file

@ -26,10 +26,12 @@ export interface EventsActionGroupData {
doc_count: number;
}
export type Fields = Record<string, unknown[] | Fields[]>;
export interface EventHit extends SearchHit {
sort: string[];
_source: EventSource;
fields: Record<string, unknown[]>;
fields: Fields;
aggregations: {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
[agg: string]: any;

6
x-pack/plugins/security_solution/common/utils/mock_event_details.ts
View file

@ -40,7 +40,7 @@ export const eventHit = {
'source.geo.location': [{ coordinates: [118.7778, 32.0617], type: 'Point' }],
'threat.indicator': [
{
'matched.field': ['matched_field'],
'matched.field': ['matched_field', 'other_matched_field'],
first_seen: ['2021-02-22T17:29:25.195Z'],
provider: ['yourself'],
type: ['custom'],
@ -259,8 +259,8 @@ export const eventDetailsFormattedFields = [
{
category: 'threat',
field: 'threat.indicator.matched.field',
values: ['matched_field', 'matched_field_2'],
originalValue: ['matched_field', 'matched_field_2'],
values: ['matched_field', 'other_matched_field', 'matched_field_2'],
originalValue: ['matched_field', 'other_matched_field', 'matched_field_2'],
isObjectArray: false,
},
{

10
x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts
View file

@ -5,6 +5,15 @@
* 2.0.
*/
export const TIMELINE_CTI_FIELDS = [
'threat.indicator.event.dataset',
'threat.indicator.event.reference',
'threat.indicator.matched.atomic',
'threat.indicator.matched.field',
'threat.indicator.matched.type',
'threat.indicator.provider',
];
export const TIMELINE_EVENTS_FIELDS = [
'@timestamp',
'signal.status',
@ -230,4 +239,5 @@ export const TIMELINE_EVENTS_FIELDS = [
'zeek.ssl.established',
'zeek.ssl.resumed',
'zeek.ssl.version',
...TIMELINE_CTI_FIELDS,
];

209
x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts
View file

@ -5,10 +5,10 @@
* 2.0.
*/
import { eventHit } from '../../../../../../common/utils/mock_event_details';
import { EventHit } from '../../../../../../common/search_strategy';
import { TIMELINE_EVENTS_FIELDS } from './constants';
import { formatTimelineData } from './helpers';
import { eventHit } from '../../../../../../common/utils/mock_event_details';
import { buildObjectForFieldPath, formatTimelineData } from './helpers';
describe('#formatTimelineData', () => {
it('happy path', async () => {
@ -42,12 +42,12 @@ describe('#formatTimelineData', () => {
value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
},
{
field: 'source.geo.location',
value: [`{"lon":118.7778,"lat":32.0617}`],
field: 'threat.indicator.matched.field',
value: ['matched_field', 'other_matched_field', 'matched_field_2'],
},
{
field: 'threat.indicator.matched.field',
value: ['matched_field', 'matched_field_2'],
field: 'source.geo.location',
value: [`{"lon":118.7778,"lat":32.0617}`],
},
],
ecs: {
@ -94,6 +94,34 @@ describe('#formatTimelineData', () => {
user: {
name: ['jenkins'],
},
threat: {
indicator: [
{
event: {
dataset: [],
reference: [],
},
matched: {
atomic: ['matched_atomic'],
field: ['matched_field', 'other_matched_field'],
type: [],
},
provider: ['yourself'],
},
{
event: {
dataset: [],
reference: [],
},
matched: {
atomic: ['matched_atomic_2'],
field: ['matched_field_2'],
type: [],
},
provider: ['other_you'],
},
],
},
},
},
});
@ -371,4 +399,173 @@ describe('#formatTimelineData', () => {
},
});
});
describe('buildObjectForFieldPath', () => {
it('builds an object from a single non-nested field', () => {
expect(buildObjectForFieldPath('@timestamp', eventHit)).toEqual({
'@timestamp': ['2020-11-17T14:48:08.922Z'],
});
});
it('builds an object with no fields response', () => {
const { fields, ...fieldLessHit } = eventHit;
// @ts-expect-error fieldLessHit is intentionally missing fields
expect(buildObjectForFieldPath('@timestamp', fieldLessHit)).toEqual({
'@timestamp': [],
});
});
it('does not misinterpret non-nested fields with a common prefix', () => {
// @ts-expect-error hit is minimal
const hit: EventHit = {
fields: {
'foo.bar': ['baz'],
'foo.barBaz': ['foo'],
},
};
expect(buildObjectForFieldPath('foo.barBaz', hit)).toEqual({
foo: { barBaz: ['foo'] },
});
});
it('builds an array of objects from a nested field', () => {
// @ts-expect-error hit is minimal
const hit: EventHit = {
fields: {
foo: [{ bar: ['baz'] }],
},
};
expect(buildObjectForFieldPath('foo.bar', hit)).toEqual({
foo: [{ bar: ['baz'] }],
});
});
it('builds intermediate objects for nested fields', () => {
// @ts-expect-error nestedHit is minimal
const nestedHit: EventHit = {
fields: {
'foo.bar': [
{
baz: ['host.name'],
},
],
},
};
expect(buildObjectForFieldPath('foo.bar.baz', nestedHit)).toEqual({
foo: {
bar: [
{
baz: ['host.name'],
},
],
},
});
});
it('builds intermediate objects at multiple levels', () => {
expect(buildObjectForFieldPath('threat.indicator.matched.atomic', eventHit)).toEqual({
threat: {
indicator: [
{
matched: {
atomic: ['matched_atomic'],
},
},
{
matched: {
atomic: ['matched_atomic_2'],
},
},
],
},
});
});
it('preserves multiple values for a single leaf', () => {
expect(buildObjectForFieldPath('threat.indicator.matched.field', eventHit)).toEqual({
threat: {
indicator: [
{
matched: {
field: ['matched_field', 'other_matched_field'],
},
},
{
matched: {
field: ['matched_field_2'],
},
},
],
},
});
});
describe('multiple levels of nested fields', () => {
let nestedHit: EventHit;
beforeEach(() => {
// @ts-expect-error nestedHit is minimal
nestedHit = {
fields: {
'nested_1.foo': [
{
'nested_2.bar': [
{ leaf: ['leaf_value'], leaf_2: ['leaf_2_value'] },
{ leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] },
],
},
{
'nested_2.bar': [
{ leaf: ['leaf_value_2'], leaf_2: ['leaf_2_value_4'] },
{ leaf: ['leaf_value_3'], leaf_2: ['leaf_2_value_5'] },
],
},
],
},
};
});
it('includes objects without the field', () => {
expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf', nestedHit)).toEqual({
nested_1: {
foo: [
{
nested_2: {
bar: [{ leaf: ['leaf_value'] }, { leaf: [] }],
},
},
{
nested_2: {
bar: [{ leaf: ['leaf_value_2'] }, { leaf: ['leaf_value_3'] }],
},
},
],
},
});
});
it('groups multiple leaf values', () => {
expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf_2', nestedHit)).toEqual({
nested_1: {
foo: [
{
nested_2: {
bar: [
{ leaf_2: ['leaf_2_value'] },
{ leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] },
],
},
},
{
nested_2: {
bar: [{ leaf_2: ['leaf_2_value_4'] }, { leaf_2: ['leaf_2_value_5'] }],
},
},
],
},
});
});
});
});
});

65
x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts
View file

@ -5,9 +5,12 @@
* 2.0.
*/
import { set } from '@elastic/safer-lodash-set';
import { get, has, merge, uniq } from 'lodash/fp';
import { Ecs } from '../../../../../../common/ecs';
import {
EventHit,
Fields,
TimelineEdges,
TimelineNonEcsData,
} from '../../../../../../common/search_strategy';
@ -78,19 +81,14 @@ const getValuesFromFields = async (
[fieldName]: get(fieldName, hit._source),
};
} else {
if (nestedParentFieldName == null || nestedParentFieldName === fieldName) {
if (nestedParentFieldName == null) {
fieldToEval = {
[fieldName]: hit.fields[fieldName],
};
} else if (nestedParentFieldName != null) {
} else {
fieldToEval = {
[nestedParentFieldName]: hit.fields[nestedParentFieldName],
};
} else {
// fallback, should never hit
fieldToEval = {
[fieldName]: [],
};
}
}
const formattedData = await getDataSafety(getDataFromFieldsHits, fieldToEval);
@ -102,6 +100,37 @@ const getValuesFromFields = async (
);
};
const buildObjectRecursive = (fieldPath: string, fields: Fields): Partial<Ecs> => {
const nestedParentPath = getNestedParentPath(fieldPath, fields);
if (!nestedParentPath) {
return set({}, fieldPath, toStringArray(get(fieldPath, fields)));
}
const subPath = fieldPath.replace(`${nestedParentPath}.`, '');
const subFields = (get(nestedParentPath, fields) ?? []) as Fields[];
return set(
{},
nestedParentPath,
subFields.map((subField) => buildObjectRecursive(subPath, subField))
);
};
export const buildObjectForFieldPath = (fieldPath: string, hit: EventHit): Partial<Ecs> => {
if (has(fieldPath, hit._source)) {
const value = get(fieldPath, hit._source);
return set({}, fieldPath, toStringArray(value));
}
return buildObjectRecursive(fieldPath, hit.fields);
};
/**
* If a prefix of our full field path is present as a field, we know that our field is nested
*/
const getNestedParentPath = (fieldPath: string, fields: Fields | undefined): string | undefined =>
fields &&
Object.keys(fields).find((field) => field !== fieldPath && fieldPath.startsWith(`${field}.`));
const mergeTimelineFieldsWithHit = async <T>(
fieldName: string,
flattenedFields: T,
@ -109,15 +138,12 @@ const mergeTimelineFieldsWithHit = async <T>(
dataFields: readonly string[],
ecsFields: readonly string[]
) => {
if (fieldName != null || dataFields.includes(fieldName)) {
const fieldNameAsArray = fieldName.split('.');
const nestedParentFieldName = Object.keys(hit.fields ?? []).find((f) => {
return f === fieldNameAsArray.slice(0, f.split('.').length).join('.');
});
if (fieldName != null) {
const nestedParentPath = getNestedParentPath(fieldName, hit.fields);
if (
nestedParentPath != null ||
has(fieldName, hit._source) ||
has(fieldName, hit.fields) ||
nestedParentFieldName != null ||
specialFields.includes(fieldName)
) {
const objectWithProperty = {
@ -126,22 +152,13 @@ const mergeTimelineFieldsWithHit = async <T>(
data: dataFields.includes(fieldName)
? [
...get('node.data', flattenedFields),
...(await getValuesFromFields(fieldName, hit, nestedParentFieldName)),
...(await getValuesFromFields(fieldName, hit, nestedParentPath)),
]
: get('node.data', flattenedFields),
ecs: ecsFields.includes(fieldName)
? {
...get('node.ecs', flattenedFields),
// @ts-expect-error
...fieldName.split('.').reduceRight(
// @ts-expect-error
(obj, next) => ({ [next]: obj }),
toStringArray(
has(fieldName, hit._source)
? get(fieldName, hit._source)
: hit.fields[fieldName]
)
),
...buildObjectForFieldPath(fieldName, hit),
}
: get('node.ecs', flattenedFields),
},