github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solutio][Investigations] Update Timeline Details API with ECS field (#120683)

Browse source 
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Michael Olorunnisola 2021-12-13 11:56:40 -05:00 • committed by GitHub
parent 6c4c5e1299
commit cc9be33dad
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 3214 additions and 2927 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/security_solution/common/types/timeline/index.ts
View file

@ -22,7 +22,6 @@ import {
import { FlowTarget } from '../../search_strategy/security_solution/network';
import { errorSchema } from '../../detection_engine/schemas/response/error_schema';
import { Direction, Maybe } from '../../search_strategy';
import { Ecs } from '../../ecs';
export * from './actions';
export * from './cells';
@ -503,7 +502,6 @@ export type TimelineExpandedEventType =
eventId: string;
indexName: string;
refetch?: () => void;
ecsData?: Ecs;
};
}
| EmptyObject;

2641
x-pack/plugins/security_solution/public/timelines/components/side_panel/__snapshots__/index.test.tsx.snap generated
View file

File diff suppressed because it is too large Load diff

5
x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/footer.tsx
View file

@ -24,10 +24,10 @@ import { inputsModel, inputsSelectors, State } from '../../../../common/store';
interface EventDetailsFooterProps {
detailsData: TimelineEventsDetailsItem[] | null;
detailsEcsData: Ecs | null;
expandedEvent: {
eventId: string;
indexName: string;
ecsData?: Ecs;
refetch?: () => void;
};
handleOnEventClosed: () => void;
@ -47,6 +47,7 @@ interface AddExceptionModalWrapperData {
export const EventDetailsFooterComponent = React.memo(
({
detailsData,
detailsEcsData,
expandedEvent,
handleOnEventClosed,
isHostIsolationPanelOpen,
@ -116,7 +117,7 @@ export const EventDetailsFooterComponent = React.memo(
skip: expandedEvent?.eventId == null,
});
const ecsData = expandedEvent.ecsData ?? get(0, alertsEcsData);
const ecsData = detailsEcsData ?? get(0, alertsEcsData);
return (
<>
<EuiFlyoutFooter>

7
x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.tsx
View file

@ -31,8 +31,6 @@ import {
import { getFieldValue } from '../../../../detections/components/host_isolation/helpers';
import { ALERT_DETAILS } from './translations';
import { useWithCaseDetailsRefresh } from '../../../../common/components/endpoint/host_isolation/endpoint_host_isolation_cases_context';
import { TimelineNonEcsData } from '../../../../../common/search_strategy';
import { Ecs } from '../../../../../common/ecs';
import { EventDetailsFooter } from './footer';
import { EntityType } from '../../../../../../timelines/common';
import { useHostsRiskScore } from '../../../../common/containers/hosts_risk/use_hosts_risk_score';
@ -58,8 +56,6 @@ interface EventDetailsPanelProps {
expandedEvent: {
eventId: string;
indexName: string;
ecsData?: Ecs;
nonEcsData?: TimelineNonEcsData[];
refetch?: () => void;
};
handleOnEventClosed: () => void;
@ -82,7 +78,7 @@ const EventDetailsPanelComponent: React.FC<EventDetailsPanelProps> = ({
tabType,
timelineId,
}) => {
const [loading, detailsData, rawEventData] = useTimelineEventsDetails({
const [loading, detailsData, rawEventData, ecsData] = useTimelineEventsDetails({
docValueFields,
entityType,
indexName: expandedEvent.indexName ?? '',
@ -220,6 +216,7 @@ const EventDetailsPanelComponent: React.FC<EventDetailsPanelProps> = ({
<EventDetailsFooter
detailsData={detailsData}
detailsEcsData={ecsData}
expandedEvent={expandedEvent}
handleOnEventClosed={handleOnEventClosed}
isHostIsolationPanelOpen={isHostIsolationPanelOpen}

2685
x-pack/plugins/security_solution/public/timelines/components/side_panel/index.test.tsx
View file

File diff suppressed because it is too large Load diff

13
x-pack/plugins/security_solution/public/timelines/containers/details/index.tsx
View file

@ -24,9 +24,11 @@ import { isCompleteResponse, isErrorResponse } from '../../../../../../../src/pl
import { useAppToasts } from '../../../common/hooks/use_app_toasts';
import * as i18n from './translations';
import { EntityType } from '../../../../../timelines/common';
import { Ecs } from '../../../../common/ecs';
export interface EventsArgs {
detailsData: TimelineEventsDetailsItem[] | null;
ecs: Ecs | null;
}
export interface UseTimelineEventsDetailsProps {
@ -45,7 +47,12 @@ export const useTimelineEventsDetails = ({
eventId,
runtimeMappings,
skip,
}: UseTimelineEventsDetailsProps): [boolean, EventsArgs['detailsData'], object | undefined] => {
}: UseTimelineEventsDetailsProps): [
boolean,
EventsArgs['detailsData'],
object | undefined,
EventsArgs['ecs']
] => {
const { data } = useKibana().services;
const refetch = useRef<inputsModel.Refetch>(noop);
const abortCtrl = useRef(new AbortController());
@ -57,6 +64,7 @@ export const useTimelineEventsDetails = ({
const [timelineDetailsResponse, setTimelineDetailsResponse] =
useState<EventsArgs['detailsData']>(null);
const [ecsData, setEcsData] = useState<EventsArgs['ecs']>(null);
const [rawEventData, setRawEventData] = useState<object | undefined>(undefined);
@ -84,6 +92,7 @@ export const useTimelineEventsDetails = ({
setLoading(false);
setTimelineDetailsResponse(response.data || []);
setRawEventData(response.rawResponse.hits.hits[0]);
setEcsData(response.ecs || null);
searchSubscription$.current.unsubscribe();
} else if (isErrorResponse(response)) {
setLoading(false);
@ -132,5 +141,5 @@ export const useTimelineEventsDetails = ({
};
}, [timelineDetailsRequest, timelineDetailsSearch]);
return [loading, timelineDetailsResponse, rawEventData];
return [loading, timelineDetailsResponse, rawEventData, ecsData];
};

2
x-pack/plugins/timelines/common/search_strategy/timeline/events/details/index.ts
View file

@ -10,6 +10,7 @@ import { JsonObject } from '@kbn/utility-types';
import type { IEsSearchResponse } from '../../../../../../../../src/plugins/data/common';
import { Inspect, Maybe } from '../../../common';
import { TimelineRequestOptionsPaginated } from '../..';
import { Ecs } from '../../../../../common/ecs';
export interface TimelineEventsDetailsItem {
ariaRowindex?: Maybe<number>;
@ -23,6 +24,7 @@ export interface TimelineEventsDetailsItem {
export interface TimelineEventsDetailsStrategyResponse extends IEsSearchResponse {
data?: Maybe<TimelineEventsDetailsItem[]>;
ecs?: Maybe<Ecs>;
inspect?: Maybe<Inspect>;
rawEventData?: Maybe<object>;
}

3
x-pack/plugins/timelines/public/components/t_grid/body/events/stateful_event.tsx
View file

@ -128,7 +128,6 @@ const StatefulEventComponent: React.FC<Props> = ({
const updatedExpandedDetail: TimelineExpandedDetailType = {
panelView: 'eventDetail',
params: {
ecsData: event.ecs,
eventId,
indexName,
},
@ -141,7 +140,7 @@ const StatefulEventComponent: React.FC<Props> = ({
timelineId,
})
);
}, [dispatch, event._id, event._index, event.ecs, tabType, timelineId]);
}, [dispatch, event._id, event._index, tabType, timelineId]);
const setEventsLoading = useCallback<SetEventsLoading>(
({ eventIds, isLoading }) => {

3
x-pack/plugins/timelines/public/components/t_grid/body/row_action/index.tsx
View file

@ -95,7 +95,6 @@ const RowActionComponent = ({
params: {
eventId: eventId ?? '',
indexName: indexName ?? '',
ecsData,
},
};
@ -106,7 +105,7 @@ const RowActionComponent = ({
timelineId,
})
);
}, [dispatch, ecsData, eventId, indexName, tabType, timelineId]);
}, [dispatch, eventId, indexName, tabType, timelineId]);
const Action = controlColumn.rowCellRender;

4
x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts
View file

@ -19,8 +19,8 @@ import {
TimelineEqlResponse,
} from '../../../../common/search_strategy/timeline/events/eql';
import { inspectStringifyObject } from '../../../utils/build_query';
import { TIMELINE_EVENTS_FIELDS } from '../factory/events/all/constants';
import { formatTimelineData } from '../factory/events/all/helpers';
import { TIMELINE_EVENTS_FIELDS } from '../factory/helpers/constants';
import { formatTimelineData } from '../factory/helpers/format_timeline_data';
export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record<string, unknown> => {
if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) {

5
x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts
View file

@ -17,9 +17,10 @@ import {
} from '../../../../../../common/search_strategy';
import { TimelineFactory } from '../../types';
import { buildTimelineEventsAllQuery } from './query.events_all.dsl';
import { TIMELINE_EVENTS_FIELDS } from './constants';
import { buildFieldsRequest, formatTimelineData } from './helpers';
import { inspectStringifyObject } from '../../../../../utils/build_query';
import { buildFieldsRequest } from '../../helpers/build_fields_request';
import { formatTimelineData } from '../../helpers/format_timeline_data';
import { TIMELINE_EVENTS_FIELDS } from '../../helpers/constants';
export const timelineEventsAll: TimelineFactory<TimelineEventsQueries.all> = {
buildDsl: ({ authFilter, ...options }: TimelineEventsAllRequestOptions) => {

4
x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/index.ts
View file

@ -24,6 +24,7 @@ import {
getDataFromSourceHits,
getDataSafety,
} from '../../../../../../common/utils/field_formatters';
import { buildEcsObjects } from '../../helpers/build_ecs_objects';
export const timelineEventsDetails: TimelineFactory<TimelineEventsQueries.details> = {
buildDsl: ({ authFilter, ...options }: TimelineEventsDetailsRequestOptions) => {
@ -67,12 +68,13 @@ export const timelineEventsDetails: TimelineFactory<TimelineEventsQueries.detail
);
const data = unionBy('field', fieldsData, sourceData);
const rawEventData = response.rawResponse.hits.hits[0];
const ecs = buildEcsObjects(rawEventData as EventHit);
return {
...response,
data,
ecs,
inspect,
rawEventData,
};

134
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/build_ecs_objects.test.ts Normal file
View file

@ -0,0 +1,134 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { eventHit } from '@kbn/securitysolution-t-grid';
import { EventHit } from '../../../../../common/search_strategy';
import { buildEcsObjects } from './build_ecs_objects';
describe('buildEcsObjects', () => {
it('should not populate null ecs fields', () => {
const hitWithMissingInfo: EventHit = {
_index: '.test-index',
_id: 'test-id',
_score: 0,
_source: {
'@timestamp': 123456,
host: {
architecture: 'windows98',
hostname: 'test-name',
id: 'some-id',
ip: [],
name: 'test-name',
},
},
fields: {
'@timestamp': [123456],
'host.architecture': ['windows98'],
'host.hostname': ['test-name'],
'host.id': ['some-id'],
'host.ip': [],
'host.name': ['test-name'],
},
_type: '',
sort: ['1610199700517'],
};
const ecsObject = buildEcsObjects(hitWithMissingInfo);
expect(ecsObject).toEqual({
_id: 'test-id',
_index: '.test-index',
host: {
id: ['some-id'],
ip: [],
name: ['test-name'],
},
timestamp: '123456',
'@timestamp': ['123456'],
});
});
it('should build nested ecs fields', () => {
const ecsObject = buildEcsObjects(eventHit);
expect(ecsObject).toEqual({
'@timestamp': ['2020-11-17T14:48:08.922Z'],
_id: 'tkCt1nUBaEgqnrVSZ8R_',
_index: 'auditbeat-7.8.0-2020.11.05-000003',
agent: {
type: ['auditbeat'],
},
event: {
action: ['process_started'],
category: ['process'],
dataset: ['process'],
kind: ['event'],
module: ['system'],
type: ['start'],
},
host: {
id: ['e59991e835905c65ed3e455b33e13bd6'],
ip: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'],
name: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
os: {
family: ['debian'],
},
},
message: ['Process go (PID: 4313) by user jenkins STARTED'],
process: {
args: ['go', 'vet', './...'],
entity_id: ['Z59cIkAAIw8ZoK0H'],
executable: [
'/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go',
],
hash: {
sha1: ['1eac22336a41e0660fb302add9d97daa2bcc7040'],
},
name: ['go'],
pid: ['4313'],
ppid: ['3977'],
working_directory: [
'/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat',
],
},
threat: {
enrichments: [
{
feed: {
name: [],
},
indicator: {
provider: ['yourself'],
reference: [],
},
matched: {
atomic: ['matched_atomic'],
field: ['matched_field', 'other_matched_field'],
type: [],
},
},
{
feed: {
name: [],
},
indicator: {
provider: ['other_you'],
reference: [],
},
matched: {
atomic: ['matched_atomic_2'],
field: ['matched_field_2'],
type: [],
},
},
],
},
timestamp: '2020-11-17T14:48:08.922Z',
user: {
name: ['jenkins'],
},
});
});
});

33
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/build_ecs_objects.ts Normal file
View file

@ -0,0 +1,33 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { has, merge } from 'lodash/fp';
import { EventHit } from '../../../../../common/search_strategy';
import { ECS_METADATA_FIELDS, TIMELINE_EVENTS_FIELDS } from './constants';
import { Ecs } from '../../../../../common/ecs';
import { getTimestamp } from './get_timestamp';
import { buildObjectForFieldPath } from './build_object_for_field_path';
import { getNestedParentPath } from './get_nested_parent_path';
export const buildEcsObjects = (hit: EventHit): Ecs => {
const ecsFields = [...TIMELINE_EVENTS_FIELDS];
return ecsFields.reduce(
(acc, field) => {
const nestedParentPath = getNestedParentPath(field, hit.fields);
if (
nestedParentPath != null ||
has(field, hit._source) ||
has(field, hit.fields) ||
ECS_METADATA_FIELDS.includes(field)
) {
return merge(acc, buildObjectForFieldPath(field, hit));
}
return acc;
},
{ _id: hit._id, timestamp: getTimestamp(hit), _index: hit._index }
);
};

34
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/build_fields_request.test.ts Normal file
View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { buildFieldsRequest } from './build_fields_request';
import { TIMELINE_EVENTS_FIELDS } from './constants';
describe('buildFieldsRequest', () => {
it('should include ecs fields by default', () => {
const fields: string[] = [];
const fieldsRequest = buildFieldsRequest(fields);
expect(fieldsRequest).toHaveLength(TIMELINE_EVENTS_FIELDS.length);
});
it('should not show ecs fields', () => {
const fields: string[] = [];
const fieldsRequest = buildFieldsRequest(fields, true);
expect(fieldsRequest).toHaveLength(0);
});
it('should map the expected (non underscore prefixed) fields', () => {
const fields = ['_dontShow1', '_dontShow2', 'showsup'];
const fieldsRequest = buildFieldsRequest(fields, true);
expect(fieldsRequest).toEqual([{ field: 'showsup', include_unmapped: true }]);
});
it('should map provided fields with ecs fields', () => {
const fields = ['showsup'];
const fieldsRequest = buildFieldsRequest(fields);
expect(fieldsRequest).toHaveLength(TIMELINE_EVENTS_FIELDS.length + fields.length);
});
});

18
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/build_fields_request.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { uniq } from 'lodash/fp';
import { TIMELINE_EVENTS_FIELDS } from './constants';
export const buildFieldsRequest = (fields: string[], excludeEcsData?: boolean) =>
uniq([
...fields.filter((field) => !field.startsWith('_')),
...(excludeEcsData ? [] : TIMELINE_EVENTS_FIELDS),
]).map((field) => ({
field,
include_unmapped: true,
}));

179
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/build_object_for_field_path.test.ts Normal file
View file

@ -0,0 +1,179 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { eventHit } from '@kbn/securitysolution-t-grid';
import { EventHit } from '../../../../../common/search_strategy';
import { buildObjectForFieldPath } from './build_object_for_field_path';
describe('buildObjectForFieldPath', () => {
it('builds an object from a single non-nested field', () => {
expect(buildObjectForFieldPath('@timestamp', eventHit)).toEqual({
'@timestamp': ['2020-11-17T14:48:08.922Z'],
});
});
it('builds an object with no fields response', () => {
const { fields, ...fieldLessHit } = eventHit;
// @ts-expect-error fieldLessHit is intentionally missing fields
expect(buildObjectForFieldPath('@timestamp', fieldLessHit)).toEqual({
'@timestamp': [],
});
});
it('does not misinterpret non-nested fields with a common prefix', () => {
// @ts-expect-error hit is minimal
const hit: EventHit = {
fields: {
'foo.bar': ['baz'],
'foo.barBaz': ['foo'],
},
};
expect(buildObjectForFieldPath('foo.barBaz', hit)).toEqual({
foo: { barBaz: ['foo'] },
});
});
it('builds an array of objects from a nested field', () => {
// @ts-expect-error hit is minimal
const hit: EventHit = {
fields: {
foo: [{ bar: ['baz'] }],
},
};
expect(buildObjectForFieldPath('foo.bar', hit)).toEqual({
foo: [{ bar: ['baz'] }],
});
});
it('builds intermediate objects for nested fields', () => {
// @ts-expect-error nestedHit is minimal
const nestedHit: EventHit = {
fields: {
'foo.bar': [
{
baz: ['host.name'],
},
],
},
};
expect(buildObjectForFieldPath('foo.bar.baz', nestedHit)).toEqual({
foo: {
bar: [
{
baz: ['host.name'],
},
],
},
});
});
it('builds intermediate objects at multiple levels', () => {
expect(buildObjectForFieldPath('threat.enrichments.matched.atomic', eventHit)).toEqual({
threat: {
enrichments: [
{
matched: {
atomic: ['matched_atomic'],
},
},
{
matched: {
atomic: ['matched_atomic_2'],
},
},
],
},
});
});
it('preserves multiple values for a single leaf', () => {
expect(buildObjectForFieldPath('threat.enrichments.matched.field', eventHit)).toEqual({
threat: {
enrichments: [
{
matched: {
field: ['matched_field', 'other_matched_field'],
},
},
{
matched: {
field: ['matched_field_2'],
},
},
],
},
});
});
describe('multiple levels of nested fields', () => {
let nestedHit: EventHit;
beforeEach(() => {
// @ts-expect-error nestedHit is minimal
nestedHit = {
fields: {
'nested_1.foo': [
{
'nested_2.bar': [
{ leaf: ['leaf_value'], leaf_2: ['leaf_2_value'] },
{ leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] },
],
},
{
'nested_2.bar': [
{ leaf: ['leaf_value_2'], leaf_2: ['leaf_2_value_4'] },
{ leaf: ['leaf_value_3'], leaf_2: ['leaf_2_value_5'] },
],
},
],
},
};
});
it('includes objects without the field', () => {
expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf', nestedHit)).toEqual({
nested_1: {
foo: [
{
nested_2: {
bar: [{ leaf: ['leaf_value'] }, { leaf: [] }],
},
},
{
nested_2: {
bar: [{ leaf: ['leaf_value_2'] }, { leaf: ['leaf_value_3'] }],
},
},
],
},
});
});
it('groups multiple leaf values', () => {
expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf_2', nestedHit)).toEqual({
nested_1: {
foo: [
{
nested_2: {
bar: [
{ leaf_2: ['leaf_2_value'] },
{ leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] },
],
},
},
{
nested_2: {
bar: [{ leaf_2: ['leaf_2_value_4'] }, { leaf_2: ['leaf_2_value_5'] }],
},
},
],
},
});
});
});
});

37
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/build_object_for_field_path.ts Normal file
View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { set } from '@elastic/safer-lodash-set';
import { get, has } from 'lodash/fp';
import { Ecs } from '../../../../../common/ecs';
import { EventHit, Fields } from '../../../../../common/search_strategy';
import { toStringArray } from '../../../../../common/utils/to_array';
import { getNestedParentPath } from './get_nested_parent_path';
const buildObjectRecursive = (fieldPath: string, fields: Fields): Partial<Ecs> => {
const nestedParentPath = getNestedParentPath(fieldPath, fields);
if (!nestedParentPath) {
return set({}, fieldPath, toStringArray(get(fieldPath, fields)));
}
const subPath = fieldPath.replace(`${nestedParentPath}.`, '');
const subFields = (get(nestedParentPath, fields) ?? []) as Fields[];
return set(
{},
nestedParentPath,
subFields.map((subField) => buildObjectRecursive(subPath, subField))
);
};
export const buildObjectForFieldPath = (fieldPath: string, hit: EventHit): Partial<Ecs> => {
if (has(fieldPath, hit._source)) {
const value = get(fieldPath, hit._source);
return set({}, fieldPath, toStringArray(value));
}
return buildObjectRecursive(fieldPath, hit.fields);
};

3
x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/constants.ts → x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/constants.ts
View file

@ -6,7 +6,6 @@
*/
import { ALERT_RULE_CONSUMER } from '@kbn/rule-data-utils';
// import { CTI_ROW_RENDERER_FIELDS } from '../../../../../../common/cti/constants';
// TODO: share with security_solution/common/cti/constants.ts
export const ENRICHMENT_DESTINATION_PATH = 'threat.enrichments';
@ -273,3 +272,5 @@ export const TIMELINE_EVENTS_FIELDS = [
'zeek.ssl.version',
...CTI_ROW_RENDERER_FIELDS,
];
export const ECS_METADATA_FIELDS = ['_id', '_index', '_type', '_score'];

179
x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/helpers.test.ts → x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.test.ts
View file

@ -6,12 +6,12 @@
*/
import { eventHit } from '@kbn/securitysolution-t-grid';
import { EventHit } from '../../../../../../common/search_strategy';
import { EventHit } from '../../../../../common/search_strategy';
import { TIMELINE_EVENTS_FIELDS } from './constants';
import { buildObjectForFieldPath, formatTimelineData } from './helpers';
import { formatTimelineData } from './format_timeline_data';
describe('#formatTimelineData', () => {
it('happy path', async () => {
describe('formatTimelineData', () => {
it('should properly format the timeline data', async () => {
const res = await formatTimelineData(
[
'@timestamp',
@ -127,7 +127,7 @@ describe('#formatTimelineData', () => {
});
});
it('rule signal results', async () => {
it('should properly format the rule signal results', async () => {
const response: EventHit = {
_index: '.siem-signals-patrykkopycinski-default-000007',
_id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562',
@ -405,173 +405,4 @@ describe('#formatTimelineData', () => {
},
});
});
describe('buildObjectForFieldPath', () => {
it('builds an object from a single non-nested field', () => {
expect(buildObjectForFieldPath('@timestamp', eventHit)).toEqual({
'@timestamp': ['2020-11-17T14:48:08.922Z'],
});
});
it('builds an object with no fields response', () => {
const { fields, ...fieldLessHit } = eventHit;
// @ts-expect-error fieldLessHit is intentionally missing fields
expect(buildObjectForFieldPath('@timestamp', fieldLessHit)).toEqual({
'@timestamp': [],
});
});
it('does not misinterpret non-nested fields with a common prefix', () => {
// @ts-expect-error hit is minimal
const hit: EventHit = {
fields: {
'foo.bar': ['baz'],
'foo.barBaz': ['foo'],
},
};
expect(buildObjectForFieldPath('foo.barBaz', hit)).toEqual({
foo: { barBaz: ['foo'] },
});
});
it('builds an array of objects from a nested field', () => {
// @ts-expect-error hit is minimal
const hit: EventHit = {
fields: {
foo: [{ bar: ['baz'] }],
},
};
expect(buildObjectForFieldPath('foo.bar', hit)).toEqual({
foo: [{ bar: ['baz'] }],
});
});
it('builds intermediate objects for nested fields', () => {
// @ts-expect-error nestedHit is minimal
const nestedHit: EventHit = {
fields: {
'foo.bar': [
{
baz: ['host.name'],
},
],
},
};
expect(buildObjectForFieldPath('foo.bar.baz', nestedHit)).toEqual({
foo: {
bar: [
{
baz: ['host.name'],
},
],
},
});
});
it('builds intermediate objects at multiple levels', () => {
expect(buildObjectForFieldPath('threat.enrichments.matched.atomic', eventHit)).toEqual({
threat: {
enrichments: [
{
matched: {
atomic: ['matched_atomic'],
},
},
{
matched: {
atomic: ['matched_atomic_2'],
},
},
],
},
});
});
it('preserves multiple values for a single leaf', () => {
expect(buildObjectForFieldPath('threat.enrichments.matched.field', eventHit)).toEqual({
threat: {
enrichments: [
{
matched: {
field: ['matched_field', 'other_matched_field'],
},
},
{
matched: {
field: ['matched_field_2'],
},
},
],
},
});
});
describe('multiple levels of nested fields', () => {
let nestedHit: EventHit;
beforeEach(() => {
// @ts-expect-error nestedHit is minimal
nestedHit = {
fields: {
'nested_1.foo': [
{
'nested_2.bar': [
{ leaf: ['leaf_value'], leaf_2: ['leaf_2_value'] },
{ leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] },
],
},
{
'nested_2.bar': [
{ leaf: ['leaf_value_2'], leaf_2: ['leaf_2_value_4'] },
{ leaf: ['leaf_value_3'], leaf_2: ['leaf_2_value_5'] },
],
},
],
},
};
});
it('includes objects without the field', () => {
expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf', nestedHit)).toEqual({
nested_1: {
foo: [
{
nested_2: {
bar: [{ leaf: ['leaf_value'] }, { leaf: [] }],
},
},
{
nested_2: {
bar: [{ leaf: ['leaf_value_2'] }, { leaf: ['leaf_value_3'] }],
},
},
],
},
});
});
it('groups multiple leaf values', () => {
expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf_2', nestedHit)).toEqual({
nested_1: {
foo: [
{
nested_2: {
bar: [
{ leaf_2: ['leaf_2_value'] },
{ leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] },
],
},
},
{
nested_2: {
bar: [{ leaf_2: ['leaf_2_value_4'] }, { leaf_2: ['leaf_2_value_5'] }],
},
},
],
},
});
});
});
});
});

76
x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/helpers.ts → x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.ts
View file

@ -5,39 +5,14 @@
* 2.0.
*/
import { set } from '@elastic/safer-lodash-set';
import { get, has, merge, uniq } from 'lodash/fp';
import { Ecs } from '../../../../../../common/ecs';
import {
EventHit,
Fields,
TimelineEdges,
TimelineNonEcsData,
} from '../../../../../../common/search_strategy';
import { toStringArray } from '../../../../../../common/utils/to_array';
import {
getDataFromFieldsHits,
getDataSafety,
} from '../../../../../../common/utils/field_formatters';
import { TIMELINE_EVENTS_FIELDS } from './constants';
const getTimestamp = (hit: EventHit): string => {
if (hit.fields && hit.fields['@timestamp']) {
return `${hit.fields['@timestamp'][0] ?? ''}`;
} else if (hit._source && hit._source['@timestamp']) {
return hit._source['@timestamp'];
}
return '';
};
export const buildFieldsRequest = (fields: string[], excludeEcsData?: boolean) =>
uniq([
...fields.filter((f) => !f.startsWith('_')),
...(excludeEcsData ? [] : TIMELINE_EVENTS_FIELDS),
]).map((field) => ({
field,
include_unmapped: true,
}));
import { EventHit, TimelineEdges, TimelineNonEcsData } from '../../../../../common/search_strategy';
import { toStringArray } from '../../../../../common/utils/to_array';
import { getDataFromFieldsHits, getDataSafety } from '../../../../../common/utils/field_formatters';
import { getTimestamp } from './get_timestamp';
import { getNestedParentPath } from './get_nested_parent_path';
import { buildObjectForFieldPath } from './build_object_for_field_path';
import { ECS_METADATA_FIELDS } from './constants';
export const formatTimelineData = async (
dataFields: readonly string[],
@ -74,14 +49,12 @@ export const formatTimelineData = async (
})
);
const specialFields = ['_id', '_index', '_type', '_score'];
const getValuesFromFields = async (
fieldName: string,
hit: EventHit,
nestedParentFieldName?: string
): Promise<TimelineNonEcsData[]> => {
if (specialFields.includes(fieldName)) {
if (ECS_METADATA_FIELDS.includes(fieldName)) {
return [{ field: fieldName, value: toStringArray(get(fieldName, hit)) }];
}
@ -110,37 +83,6 @@ const getValuesFromFields = async (
);
};
const buildObjectRecursive = (fieldPath: string, fields: Fields): Partial<Ecs> => {
const nestedParentPath = getNestedParentPath(fieldPath, fields);
if (!nestedParentPath) {
return set({}, fieldPath, toStringArray(get(fieldPath, fields)));
}
const subPath = fieldPath.replace(`${nestedParentPath}.`, '');
const subFields = (get(nestedParentPath, fields) ?? []) as Fields[];
return set(
{},
nestedParentPath,
subFields.map((subField) => buildObjectRecursive(subPath, subField))
);
};
export const buildObjectForFieldPath = (fieldPath: string, hit: EventHit): Partial<Ecs> => {
if (has(fieldPath, hit._source)) {
const value = get(fieldPath, hit._source);
return set({}, fieldPath, toStringArray(value));
}
return buildObjectRecursive(fieldPath, hit.fields);
};
/**
* If a prefix of our full field path is present as a field, we know that our field is nested
*/
const getNestedParentPath = (fieldPath: string, fields: Fields | undefined): string | undefined =>
fields &&
Object.keys(fields).find((field) => field !== fieldPath && fieldPath.startsWith(`${field}.`));
const mergeTimelineFieldsWithHit = async <T>(
fieldName: string,
flattenedFields: T,
@ -154,7 +96,7 @@ const mergeTimelineFieldsWithHit = async <T>(
nestedParentPath != null ||
has(fieldName, hit._source) ||
has(fieldName, hit.fields) ||
specialFields.includes(fieldName)
ECS_METADATA_FIELDS.includes(fieldName)
) {
const objectWithProperty = {
node: {

39
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/get_nested_parent_path.test.ts Normal file
View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Fields } from '../../../../../common/search_strategy';
import { getNestedParentPath } from './get_nested_parent_path';
describe('getNestedParentPath', () => {
let testFields: Fields | undefined;
beforeAll(() => {
testFields = {
'not.nested': ['I am not nested'],
'is.nested': [
{
field: ['I am nested'],
},
],
};
});
it('should ignore fields that are not nested', () => {
const notNestedPath = 'not.nested';
const shouldBeUndefined = getNestedParentPath(notNestedPath, testFields);
expect(shouldBeUndefined).toBe(undefined);
});
it('should capture fields that are nested', () => {
const nestedPath = 'is.nested.field';
const nestedParentPath = getNestedParentPath(nestedPath, testFields);
expect(nestedParentPath).toEqual('is.nested');
});
it('should return undefined when the `fields` param is undefined', () => {
const nestedPath = 'is.nested.field';
expect(getNestedParentPath(nestedPath, undefined)).toBe(undefined);
});
});

18
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/get_nested_parent_path.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Fields } from '../../../../../common/search_strategy';
/**
* If a prefix of our full field path is present as a field, we know that our field is nested
*/
export const getNestedParentPath = (
fieldPath: string,
fields: Fields | undefined
): string | undefined =>
fields &&
Object.keys(fields).find((field) => field !== fieldPath && fieldPath.startsWith(`${field}.`));

17
x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/get_timestamp.ts Normal file
View file

@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EventHit } from '../../../../../common/search_strategy';
export const getTimestamp = (hit: EventHit): string => {
if (hit.fields && hit.fields['@timestamp']) {
return `${hit.fields['@timestamp'][0] ?? ''}`;
} else if (hit._source && hit._source['@timestamp']) {
return hit._source['@timestamp'];
}
return '';
};