github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution] Fallback to @timestamp should be configurable when timestamp override is defined (#135116)

Browse source 
* [Security Solution] Fallback to @timestamp should be configurable when timestamp override is defined #112315

* Fix CI

* Fix CI

* Fix CI

* Review feedback: more concise name

* Review comments

* Review feedback

- create `primaryTimestamp` and `secondaryTimestamp` variables within `createSecurityRuleTypeWrapper` and pass it from there to all executors

* CI fixes

* Fix types

* Fix threshold rules

* Fix CI

* Integration tests

* Updated comment

* Fix CI

* Fix types

* Review feedback: better naming

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Ievgen Sorokopud 2022-07-12 19:24:01 +02:00 • committed by GitHub
parent 3fb87f55e3
commit 68e2ff6cbf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
53 changed files with 697 additions and 211 deletions
Download patch file Download diff file Expand all files Collapse all files

13
x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts
View file

@ -188,6 +188,19 @@ export type AnomalyThreshold = t.TypeOf<typeof PositiveInteger>;
export const anomalyThresholdOrUndefined = t.union([anomaly_threshold, t.undefined]);
export type AnomalyThresholdOrUndefined = t.TypeOf<typeof anomalyThresholdOrUndefined>;
export const timestamp_override_fallback_disabled = t.boolean;
export type TimestampOverrideFallbackDisabled = t.TypeOf<
typeof timestamp_override_fallback_disabled
>;
export const timestampOverrideFallbackDisabledOrUndefined = t.union([
timestamp_override_fallback_disabled,
t.undefined,
]);
export type TimestampOverrideFallbackDisabledOrUndefined = t.TypeOf<
typeof timestampOverrideFallbackDisabledOrUndefined
>;
/**
* Note that this is a non-exact io-ts type as we allow extra meta information
* to be added to the meta object

2
x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts
View file

@ -45,6 +45,7 @@ import {
meta,
rule_name_override,
timestamp_override,
timestamp_override_fallback_disabled,
author,
description,
false_positives,
@ -163,6 +164,7 @@ const baseParams = {
meta,
rule_name_override,
timestamp_override,
timestamp_override_fallback_disabled,
namespace,
},
defaultable: {

23
x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx
View file

@ -144,10 +144,11 @@ const StepAboutRuleComponent: FC<StepAboutRuleProps> = ({
schema,
});
const { getFields, getFormData, submit } = form;
const [{ severity: formSeverity }] = useFormData<AboutStepRule>({
form,
watch: ['severity'],
});
const [{ severity: formSeverity, timestampOverride: formTimestampOverride }] =
useFormData<AboutStepRule>({
form,
watch: ['severity', 'timestampOverride'],
});
useEffect(() => {
const formSeverityValue = formSeverity?.value;
@ -403,6 +404,20 @@ const StepAboutRuleComponent: FC<StepAboutRuleProps> = ({
placeholder: '',
}}
/>
{!!formTimestampOverride && formTimestampOverride !== '@timestamp' && (
<>
<CommonUseField
path="timestampOverrideFallbackDisabled"
componentProps={{
idAria: 'detectionTimestampOverrideFallbackDisabled',
'data-test-subj': 'detectionTimestampOverrideFallbackDisabled',
euiFieldProps: {
disabled: isLoading,
},
}}
/>
</>
)}
</EuiAccordion>
</Form>
</StepContentWrapper>

10
x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx
View file

@ -236,6 +236,16 @@ export const schema: FormSchema<AboutStepRule> = {
),
labelAppend: OptionalFieldLabel,
},
timestampOverrideFallbackDisabled: {
type: FIELD_TYPES.CHECKBOX,
label: i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldTimestampOverrideFallbackDisabledLabel',
{
defaultMessage: 'Do not use @timestamp as a fallback timestamp field',
}
),
labelAppend: OptionalFieldLabel,
},
tags: {
type: FIELD_TYPES.COMBO_BOX,
label: i18n.translate(

2
x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts
View file

@ -35,6 +35,7 @@ import {
rule_name_override,
data_view_id,
timestamp_override,
timestamp_override_fallback_disabled,
timestamp_field,
event_category_override,
tiebreaker_field,
@ -155,6 +156,7 @@ export const RuleSchema = t.intersection([
timeline_id: t.string,
timeline_title: t.string,
timestamp_override,
timestamp_override_fallback_disabled,
timestamp_field,
event_category_override,
tiebreaker_field,

1
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts
View file

@ -158,6 +158,7 @@ export const mockRuleWithEverything = (id: string): Rule => ({
},
throttle: 'no_actions',
timestamp_override: 'event.ingested',
timestamp_override_fallback_disabled: false,
note: '# this is some markdown documentation',
version: 1,
});

28
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts
View file

@ -710,6 +710,34 @@ describe('helpers', () => {
expect(result).toEqual(expected);
});
test('returns formatted object with timestamp override', () => {
const mockStepData: AboutStepRule = {
...mockData,
timestampOverride: 'event.ingest',
timestampOverrideFallbackDisabled: true,
};
const result = formatAboutStepData(mockStepData);
const expected: AboutStepRuleJson = {
author: ['Elastic'],
description: '24/7',
false_positives: ['test'],
license: 'Elastic License',
name: 'Query with rule-id',
note: '# this is some markdown documentation',
references: ['www.test.co'],
risk_score: 21,
risk_score_mapping: [],
severity: 'low',
severity_mapping: [],
tags: ['tag1', 'tag2'],
threat: getThreatMock(),
timestamp_override: 'event.ingest',
timestamp_override_fallback_disabled: true,
};
expect(result).toEqual(expected);
});
});
describe('formatActionsStepData', () => {

2
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts
View file

@ -392,6 +392,7 @@ export const formatAboutStepData = (
ruleNameOverride,
threatIndicatorPath,
timestampOverride,
timestampOverrideFallbackDisabled,
...rest
} = aboutStepData;
@ -435,6 +436,7 @@ export const formatAboutStepData = (
})),
threat_indicator_path: threatIndicatorPath,
timestamp_override: timestampOverride !== '' ? timestampOverride : undefined,
timestamp_override_fallback_disabled: timestampOverrideFallbackDisabled,
...(!isEmpty(note) ? { note } : {}),
...rest,
};

1
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx
View file

@ -128,6 +128,7 @@ describe('rule helpers', () => {
tags: ['tag1', 'tag2'],
threat: getThreatMock(),
timestampOverride: 'event.ingested',
timestampOverrideFallbackDisabled: false,
};
const scheduleRuleStepData = { from: '0s', interval: '5m' };
const ruleActionsStepData = {

2
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx
View file

@ -164,6 +164,7 @@ export const getAboutStepsData = (rule: Rule, detailsView: boolean): AboutStepRu
rule_name_override: ruleNameOverride,
severity_mapping: severityMapping,
timestamp_override: timestampOverride,
timestamp_override_fallback_disabled: timestampOverrideFallbackDisabled,
references,
severity,
false_positives: falsePositives,
@ -180,6 +181,7 @@ export const getAboutStepsData = (rule: Rule, detailsView: boolean): AboutStepRu
license: license ?? '',
ruleNameOverride: ruleNameOverride ?? '',
timestampOverride: timestampOverride ?? '',
timestampOverrideFallbackDisabled,
name,
description,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion

2
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts
View file

@ -109,6 +109,7 @@ export interface AboutStepRule {
ruleNameOverride: string;
tags: string[];
timestampOverride: string;
timestampOverrideFallbackDisabled?: boolean;
threatIndicatorPath?: string;
threat: Threats;
note: string;
@ -215,6 +216,7 @@ export interface AboutStepRuleJson {
threat: Threats;
threat_indicator_path?: string;
timestamp_override?: TimestampOverride;
timestamp_override_fallback_disabled?: boolean;
note?: string;
}

1
x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.test.ts
View file

@ -44,6 +44,7 @@ describe('schedule_notification_actions', () => {
severityMapping: [],
threat: [],
timestampOverride: undefined,
timestampOverrideFallbackDisabled: undefined,
to: 'now',
type: 'query',
references: ['http://www.example.com'],

1
x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_throttle_notification_actions.test.ts
View file

@ -50,6 +50,7 @@ describe('schedule_throttle_notification_actions', () => {
severityMapping: [],
threat: [],
timestampOverride: undefined,
timestampOverrideFallbackDisabled: undefined,
dataViewId: undefined,
to: 'now',
type: 'query',

29
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts
View file

@ -10,6 +10,7 @@ import { isEmpty } from 'lodash';
import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils';
import agent from 'elastic-apm-node';
import { TIMESTAMP } from '@kbn/rule-data-utils';
import { createPersistenceRuleTypeWrapper } from '@kbn/rule-registry-plugin/server';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
@ -79,7 +80,15 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
let hasError = false;
let inputIndex: string[] = [];
let runtimeMappings: estypes.MappingRuntimeFields | undefined;
const { from, maxSignals, meta, ruleId, timestampOverride, to } = params;
const {
from,
maxSignals,
meta,
ruleId,
timestampOverride,
timestampOverrideFallbackDisabled,
to,
} = params;
const {
alertWithPersistence,
savedObjectsClient,
@ -142,6 +151,12 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
id: alertId,
};
const primaryTimestamp = timestampOverride ?? TIMESTAMP;
const secondaryTimestamp =
primaryTimestamp !== TIMESTAMP && !timestampOverrideFallbackDisabled
? TIMESTAMP
: undefined;
/**
* Data Views Logic
* Use of data views is supported for all rules other than ML.
@ -180,8 +195,6 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
// so that we can use it in create rules route, bulk, etc.
try {
if (!isMachineLearningParams(params)) {
const hasTimestampOverride = !!timestampOverride;
const privileges = await checkPrivilegesFromEsClient(esClient, inputIndex);
wroteWarningStatus = await hasReadIndexPrivileges({
@ -197,9 +210,9 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
services.scopedClusterClient.asCurrentUser.fieldCaps(
{
index: inputIndex,
fields: hasTimestampOverride
? ['@timestamp', timestampOverride]
: ['@timestamp'],
fields: secondaryTimestamp
? [primaryTimestamp, secondaryTimestamp]
: [primaryTimestamp],
include_unmapped: true,
runtime_mappings: runtimeMappings,
},
@ -208,7 +221,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
);
wroteWarningStatus = await hasTimestampFields({
timestampField: hasTimestampOverride ? timestampOverride : '@timestamp',
timestampField: primaryTimestamp,
timestampFieldCapsResponse: timestampFieldCaps,
inputIndices: inputIndex,
ruleExecutionLogger,
@ -309,6 +322,8 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
wrapHits,
wrapSequences,
ruleDataReader: ruleDataClient.getReader({ namespace: options.spaceId }),
primaryTimestamp,
secondaryTimestamp,
},
});

4
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts
View file

@ -71,6 +71,8 @@ export const createEqlAlertType = (
tuple,
wrapHits,
wrapSequences,
primaryTimestamp,
secondaryTimestamp,
},
services,
state,
@ -89,6 +91,8 @@ export const createEqlAlertType = (
version,
wrapHits,
wrapSequences,
primaryTimestamp,
secondaryTimestamp,
});
return { ...result, state };
},

2
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts
View file

@ -178,7 +178,7 @@ export const buildAlert = (
const originalTime = getValidDateFromDoc({
doc: docs[0],
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
return {

4
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts
View file

@ -74,6 +74,8 @@ export const createIndicatorMatchAlertType = (
searchAfterSize,
tuple,
wrapHits,
primaryTimestamp,
secondaryTimestamp,
},
services,
state,
@ -95,6 +97,8 @@ export const createIndicatorMatchAlertType = (
tuple,
version,
wrapHits,
primaryTimestamp,
secondaryTimestamp,
});
return { ...result, state };
},

4
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts
View file

@ -73,6 +73,8 @@ export const createQueryAlertType = (
searchAfterSize,
tuple,
wrapHits,
primaryTimestamp,
secondaryTimestamp,
},
services,
state,
@ -94,6 +96,8 @@ export const createQueryAlertType = (
wrapHits,
inputIndex,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
});
return { ...result, state };
},

4
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/saved_query/create_saved_query_alert_type.ts
View file

@ -73,6 +73,8 @@ export const createSavedQueryAlertType = (
searchAfterSize,
tuple,
wrapHits,
primaryTimestamp,
secondaryTimestamp,
},
services,
state,
@ -94,6 +96,8 @@ export const createSavedQueryAlertType = (
tuple,
version,
wrapHits,
primaryTimestamp,
secondaryTimestamp,
});
return { ...result, state };
},

4
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts
View file

@ -74,6 +74,8 @@ export const createThresholdAlertType = (
ruleDataReader,
inputIndex,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
},
services,
startedAt,
@ -96,6 +98,8 @@ export const createThresholdAlertType = (
ruleDataReader,
inputIndex,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
});
return result;

2
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts
View file

@ -69,6 +69,8 @@ export interface RunOpts<TParams extends RuleParams> {
ruleDataReader: IRuleDataReader;
inputIndex: string[];
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
}
export type SecurityAlertType<

1
x-pack/plugins/security_solution/server/lib/detection_engine/rules/duplicate_rule.test.ts
View file

@ -58,6 +58,7 @@ describe('duplicateRule', () => {
timelineTitle: undefined,
ruleNameOverride: undefined,
timestampOverride: undefined,
timestampOverrideFallbackDisabled: undefined,
dataViewId: undefined,
},
schedule: {

1
x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts
View file

@ -54,6 +54,7 @@ export const updateRules = async ({
severityMapping: ruleUpdate.severity_mapping ?? [],
threat: ruleUpdate.threat ?? [],
timestampOverride: ruleUpdate.timestamp_override,
timestampOverrideFallbackDisabled: ruleUpdate.timestamp_override_fallback_disabled,
to: ruleUpdate.to ?? 'now',
references: ruleUpdate.references ?? [],
namespace: ruleUpdate.namespace,

5
x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts
View file

@ -383,6 +383,9 @@ export const convertPatchAPIToInternalSchema = (
severityMapping: params.severity_mapping ?? existingParams.severityMapping,
threat: params.threat ?? existingParams.threat,
timestampOverride: params.timestamp_override ?? existingParams.timestampOverride,
timestampOverrideFallbackDisabled:
params.timestamp_override_fallback_disabled ??
existingParams.timestampOverrideFallbackDisabled,
to: params.to ?? existingParams.to,
references: params.references ?? existingParams.references,
namespace: params.namespace ?? existingParams.namespace,
@ -442,6 +445,7 @@ export const convertCreateAPIToInternalSchema = (
severityMapping: input.severity_mapping ?? [],
threat: input.threat ?? [],
timestampOverride: input.timestamp_override,
timestampOverrideFallbackDisabled: input.timestamp_override_fallback_disabled,
to: input.to ?? 'now',
references: input.references ?? [],
namespace: input.namespace,
@ -559,6 +563,7 @@ export const commonParamsCamelToSnake = (params: BaseRuleParams) => {
meta: params.meta,
rule_name_override: params.ruleNameOverride,
timestamp_override: params.timestampOverride,
timestamp_override_fallback_disabled: params.timestampOverrideFallbackDisabled,
author: params.author,
false_positives: params.falsePositives,
from: params.from,

1
x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.mock.ts
View file

@ -46,6 +46,7 @@ const getBaseRuleParams = (): BaseRuleParams => {
timelineId: 'some-timeline-id',
timelineTitle: 'some-timeline-title',
timestampOverride: undefined,
timestampOverrideFallbackDisabled: undefined,
meta: {
someMeta: 'someField',
},

2
x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts
View file

@ -78,6 +78,7 @@ import {
RelatedIntegrationArray,
RequiredFieldArray,
SetupGuide,
timestampOverrideFallbackDisabledOrUndefined,
} from '../../../../common/detection_engine/schemas/common';
import { SERVER_APP_ID } from '../../../../common/constants';
@ -107,6 +108,7 @@ export const baseRuleParams = t.exact(
severity,
severityMapping: severity_mapping,
timestampOverride: timestampOverrideOrUndefined,
timestampOverrideFallbackDisabled: timestampOverrideFallbackDisabledOrUndefined,
threat: threats,
to,
references,

237
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts
View file

@ -26,7 +26,8 @@ describe('create_signals', () => {
filter: {},
size: 100,
searchAfterSortIds: undefined,
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
});
expect(query).toEqual({
@ -84,7 +85,8 @@ describe('create_signals', () => {
filter: {},
size: 100,
searchAfterSortIds: undefined,
timestampOverride: 'event.ingested',
primaryTimestamp: 'event.ingested',
secondaryTimestamp: '@timestamp',
runtimeMappings: undefined,
});
expect(query).toEqual({
@ -175,6 +177,65 @@ describe('create_signals', () => {
});
});
test('it builds a filter without @timestamp fallback if `secondaryTimestamp` is undefined', () => {
const query = buildEventsSearchQuery({
index: ['auditbeat-*'],
from: 'now-5m',
to: 'today',
filter: {},
size: 100,
searchAfterSortIds: undefined,
primaryTimestamp: 'event.ingested',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
});
expect(query).toEqual({
allow_no_indices: true,
index: ['auditbeat-*'],
size: 100,
ignore_unavailable: true,
body: {
query: {
bool: {
filter: [
{},
{
range: {
'event.ingested': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
{
match_all: {},
},
],
},
},
fields: [
{
field: '*',
include_unmapped: true,
},
{
field: 'event.ingested',
format: 'strict_date_optional_time',
},
],
sort: [
{
'event.ingested': {
order: 'asc',
unmapped_type: 'date',
},
},
],
},
});
});
test('if searchAfterSortIds is a valid sortId string', () => {
const fakeSortId = '123456789012';
const query = buildEventsSearchQuery({
@ -184,7 +245,8 @@ describe('create_signals', () => {
filter: {},
size: 100,
searchAfterSortIds: [fakeSortId],
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
});
expect(query).toEqual({
@ -243,7 +305,8 @@ describe('create_signals', () => {
filter: {},
size: 100,
searchAfterSortIds: [fakeSortIdNumber],
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
});
expect(query).toEqual({
@ -301,7 +364,8 @@ describe('create_signals', () => {
filter: {},
size: 100,
searchAfterSortIds: undefined,
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
});
expect(query).toEqual({
@ -366,7 +430,8 @@ describe('create_signals', () => {
filter: {},
size: 100,
searchAfterSortIds: undefined,
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
});
expect(query).toEqual({
@ -431,7 +496,8 @@ describe('create_signals', () => {
filter: {},
size: 100,
searchAfterSortIds: undefined,
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
trackTotalHits: false,
runtimeMappings: undefined,
});
@ -446,7 +512,8 @@ describe('create_signals', () => {
filter: {},
size: 100,
searchAfterSortIds: undefined,
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
sortOrder: 'desc',
trackTotalHits: false,
runtimeMappings: undefined,
@ -467,7 +534,8 @@ describe('create_signals', () => {
filter: {},
size: 100,
searchAfterSortIds: undefined,
timestampOverride: 'event.ingested',
primaryTimestamp: 'event.ingested',
secondaryTimestamp: '@timestamp',
sortOrder: 'desc',
runtimeMappings: undefined,
});
@ -487,18 +555,19 @@ describe('create_signals', () => {
describe('buildEqlSearchRequest', () => {
test('should build a basic request with time range', () => {
const request = buildEqlSearchRequest(
'process where true',
['testindex1', 'testindex2'],
'now-5m',
'now',
100,
undefined,
undefined,
[],
undefined,
undefined
);
const request = buildEqlSearchRequest({
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
exceptionLists: [],
runtimeMappings: undefined,
eventCategoryOverride: undefined,
});
expect(request).toEqual({
allow_no_indices: true,
index: ['testindex1', 'testindex2'],
@ -537,19 +606,20 @@ describe('create_signals', () => {
});
test('should build a request with timestamp and event category overrides', () => {
const request = buildEqlSearchRequest(
'process where true',
['testindex1', 'testindex2'],
'now-5m',
'now',
100,
undefined,
'event.ingested',
[],
undefined,
'event.other_category',
undefined
);
const request = buildEqlSearchRequest({
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters: undefined,
primaryTimestamp: 'event.ingested',
secondaryTimestamp: '@timestamp',
exceptionLists: [],
runtimeMappings: undefined,
eventCategoryOverride: 'event.other_category',
timestampField: undefined,
});
expect(request).toEqual({
allow_no_indices: true,
index: ['testindex1', 'testindex2'],
@ -623,19 +693,73 @@ describe('create_signals', () => {
});
});
test('should build a request without @timestamp fallback if secondaryTimestamp is not specified', () => {
const request = buildEqlSearchRequest({
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters: undefined,
primaryTimestamp: 'event.ingested',
secondaryTimestamp: undefined,
exceptionLists: [],
runtimeMappings: undefined,
eventCategoryOverride: 'event.other_category',
timestampField: undefined,
});
expect(request).toEqual({
allow_no_indices: true,
index: ['testindex1', 'testindex2'],
body: {
event_category_field: 'event.other_category',
size: 100,
query: 'process where true',
runtime_mappings: undefined,
filter: {
bool: {
filter: [
{
range: {
'event.ingested': {
lte: 'now',
gte: 'now-5m',
format: 'strict_date_optional_time',
},
},
},
emptyFilter,
],
},
},
fields: [
{
field: '*',
include_unmapped: true,
},
{
field: 'event.ingested',
format: 'strict_date_optional_time',
},
],
},
});
});
test('should build a request with exceptions', () => {
const request = buildEqlSearchRequest(
'process where true',
['testindex1', 'testindex2'],
'now-5m',
'now',
100,
undefined,
undefined,
[getExceptionListItemSchemaMock()],
undefined,
undefined
);
const request = buildEqlSearchRequest({
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
exceptionLists: [getExceptionListItemSchemaMock()],
runtimeMappings: undefined,
eventCategoryOverride: undefined,
});
expect(request).toEqual({
allow_no_indices: true,
index: ['testindex1', 'testindex2'],
@ -758,17 +882,18 @@ describe('create_signals', () => {
},
},
];
const request = buildEqlSearchRequest(
'process where true',
['testindex1', 'testindex2'],
'now-5m',
'now',
100,
const request = buildEqlSearchRequest({
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters,
undefined,
[],
undefined
);
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
exceptionLists: [],
runtimeMappings: undefined,
});
expect(request).toEqual({
allow_no_indices: true,
index: ['testindex1', 'testindex2'],

123
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts
View file

@ -10,6 +10,7 @@ import { isEmpty } from 'lodash';
import type {
FiltersOrUndefined,
TimestampOverrideOrUndefined,
TimestampOverride,
} from '../../../../common/detection_engine/schemas/common/schemas';
import { getQueryFilter } from '../../../../common/detection_engine/get_query_filter';
@ -23,30 +24,56 @@ interface BuildEventsSearchQuery {
size: number;
sortOrder?: estypes.SortOrder;
searchAfterSortIds: estypes.SortResults | undefined;
timestampOverride: TimestampOverrideOrUndefined;
primaryTimestamp: TimestampOverride;
secondaryTimestamp: TimestampOverrideOrUndefined;
trackTotalHits?: boolean;
}
interface BuildEqlSearchRequestParams {
query: string;
index: string[];
from: string;
to: string;
size: number;
filters: FiltersOrUndefined;
primaryTimestamp: TimestampOverride;
secondaryTimestamp: TimestampOverrideOrUndefined;
exceptionLists: ExceptionListItemSchema[];
runtimeMappings: estypes.MappingRuntimeFields | undefined;
eventCategoryOverride?: string;
timestampField?: string;
tiebreakerField?: string;
}
const buildTimeRangeFilter = ({
to,
from,
timestampOverride,
primaryTimestamp,
secondaryTimestamp,
}: {
to: string;
from: string;
timestampOverride?: string;
primaryTimestamp: TimestampOverride;
secondaryTimestamp: TimestampOverrideOrUndefined;
}): estypes.QueryDslQueryContainer => {
// If the timestampOverride is provided, documents must either populate timestampOverride with a timestamp in the range
// or must NOT populate the timestampOverride field at all and `@timestamp` must fall in the range.
// If timestampOverride is not provided, we simply use `@timestamp`
return timestampOverride != null
// The primaryTimestamp is always provided and will contain either the timestamp override field or `@timestamp` otherwise.
// The secondaryTimestamp is `undefined` if
// 1. timestamp override field is not specified
// 2. timestamp override field is set and timestamp fallback is disabled
// 3. timestamp override field is set to `@timestamp`
// or `@timestamp` otherwise.
//
// If the secondaryTimestamp is provided, documents must either populate primaryTimestamp with a timestamp in the range
// or must NOT populate the primaryTimestamp field at all and secondaryTimestamp must fall in the range.
// If secondaryTimestamp is not provided, we simply use primaryTimestamp
return secondaryTimestamp != null
? {
bool: {
minimum_should_match: 1,
should: [
{
range: {
[timestampOverride]: {
[primaryTimestamp]: {
lte: to,
gte: from,
format: 'strict_date_optional_time',
@ -58,7 +85,7 @@ const buildTimeRangeFilter = ({
filter: [
{
range: {
'@timestamp': {
[secondaryTimestamp]: {
lte: to,
gte: from,
format: 'strict_date_optional_time',
@ -69,7 +96,7 @@ const buildTimeRangeFilter = ({
bool: {
must_not: {
exists: {
field: timestampOverride,
field: primaryTimestamp,
},
},
},
@ -82,7 +109,7 @@ const buildTimeRangeFilter = ({
}
: {
range: {
'@timestamp': {
[primaryTimestamp]: {
lte: to,
gte: from,
format: 'strict_date_optional_time',
@ -101,36 +128,42 @@ export const buildEventsSearchQuery = ({
runtimeMappings,
searchAfterSortIds,
sortOrder,
timestampOverride,
primaryTimestamp,
secondaryTimestamp,
trackTotalHits,
}: BuildEventsSearchQuery) => {
const defaultTimeFields = ['@timestamp'];
const timestamps =
timestampOverride != null ? [timestampOverride, ...defaultTimeFields] : defaultTimeFields;
const timestamps = secondaryTimestamp
? [primaryTimestamp, secondaryTimestamp]
: [primaryTimestamp];
const docFields = timestamps.map((tstamp) => ({
field: tstamp,
format: 'strict_date_optional_time',
}));
const rangeFilter = buildTimeRangeFilter({ to, from, timestampOverride });
const rangeFilter = buildTimeRangeFilter({
to,
from,
primaryTimestamp,
secondaryTimestamp,
});
const filterWithTime: estypes.QueryDslQueryContainer[] = [filter, rangeFilter];
const sort: estypes.Sort = [];
if (timestampOverride) {
sort.push({
[primaryTimestamp]: {
order: sortOrder ?? 'asc',
unmapped_type: 'date',
},
});
if (secondaryTimestamp) {
sort.push({
[timestampOverride]: {
[secondaryTimestamp]: {
order: sortOrder ?? 'asc',
unmapped_type: 'date',
},
});
}
sort.push({
'@timestamp': {
order: sortOrder ?? 'asc',
unmapped_type: 'date',
},
});
const searchQuery = {
allow_no_indices: true,
@ -174,23 +207,24 @@ export const buildEventsSearchQuery = ({
return searchQuery;
};
export const buildEqlSearchRequest = (
query: string,
index: string[],
from: string,
to: string,
size: number,
filters: FiltersOrUndefined,
timestampOverride: TimestampOverrideOrUndefined,
exceptionLists: ExceptionListItemSchema[],
runtimeMappings: estypes.MappingRuntimeFields | undefined,
eventCategoryOverride?: string,
timestampField?: string,
tiebreakerField?: string
): estypes.EqlSearchRequest => {
const defaultTimeFields = ['@timestamp'];
const timestamps =
timestampOverride != null ? [timestampOverride, ...defaultTimeFields] : defaultTimeFields;
export const buildEqlSearchRequest = ({
query,
index,
from,
to,
size,
filters,
primaryTimestamp,
secondaryTimestamp,
exceptionLists,
runtimeMappings,
eventCategoryOverride,
timestampField,
tiebreakerField,
}: BuildEqlSearchRequestParams): estypes.EqlSearchRequest => {
const timestamps = secondaryTimestamp
? [primaryTimestamp, secondaryTimestamp]
: [primaryTimestamp];
const docFields = timestamps.map((tstamp) => ({
field: tstamp,
format: 'strict_date_optional_time',
@ -198,7 +232,12 @@ export const buildEqlSearchRequest = (
const esFilter = getQueryFilter('', 'eql', filters || [], index, exceptionLists);
const rangeFilter = buildTimeRangeFilter({ to, from, timestampOverride });
const rangeFilter = buildTimeRangeFilter({
to,
from,
primaryTimestamp,
secondaryTimestamp,
});
const requestFilter: estypes.QueryDslQueryContainer[] = [rangeFilter, esFilter];
const fields = [
{

1
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts
View file

@ -61,6 +61,7 @@ describe('eql_executor', () => {
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
wrapSequences: jest.fn(),
primaryTimestamp: '@timestamp',
});
expect(response.warningMessages.length).toEqual(1);
});

31
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts
View file

@ -49,6 +49,8 @@ export const eqlExecutor = async ({
bulkCreate,
wrapHits,
wrapSequences,
primaryTimestamp,
secondaryTimestamp,
}: {
inputIndex: string[];
runtimeMappings: estypes.MappingRuntimeFields | undefined;
@ -62,6 +64,8 @@ export const eqlExecutor = async ({
bulkCreate: BulkCreate;
wrapHits: WrapHits;
wrapSequences: WrapSequences;
primaryTimestamp: string;
secondaryTimestamp?: string;
}): Promise<SearchAfterAndBulkCreateReturnType> => {
const ruleParams = completeRule.ruleParams;
@ -74,20 +78,21 @@ export const eqlExecutor = async ({
result.warning = true;
}
const request = buildEqlSearchRequest(
ruleParams.query,
inputIndex,
tuple.from.toISOString(),
tuple.to.toISOString(),
completeRule.ruleParams.maxSignals,
ruleParams.filters,
ruleParams.timestampOverride,
exceptionItems,
const request = buildEqlSearchRequest({
query: ruleParams.query,
index: inputIndex,
from: tuple.from.toISOString(),
to: tuple.to.toISOString(),
size: completeRule.ruleParams.maxSignals,
filters: ruleParams.filters,
primaryTimestamp,
secondaryTimestamp,
exceptionLists: exceptionItems,
runtimeMappings,
ruleParams.eventCategoryOverride,
ruleParams.timestampField,
ruleParams.tiebreakerField
);
eventCategoryOverride: ruleParams.eventCategoryOverride,
timestampField: ruleParams.timestampField,
tiebreakerField: ruleParams.tiebreakerField,
});
const eqlSignalSearchStart = performance.now();
logger.debug(`EQL query request: ${JSON.stringify(request)}`);

6
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts
View file

@ -45,6 +45,8 @@ export const queryExecutor = async ({
buildRuleMessage,
bulkCreate,
wrapHits,
primaryTimestamp,
secondaryTimestamp,
}: {
inputIndex: string[];
runtimeMappings: estypes.MappingRuntimeFields | undefined;
@ -61,6 +63,8 @@ export const queryExecutor = async ({
buildRuleMessage: BuildRuleMessage;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
primaryTimestamp: string;
secondaryTimestamp?: string;
}) => {
const ruleParams = completeRule.ruleParams;
@ -93,6 +97,8 @@ export const queryExecutor = async ({
bulkCreate,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
});
});
};

6
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts
View file

@ -40,6 +40,8 @@ export const threatMatchExecutor = async ({
buildRuleMessage,
bulkCreate,
wrapHits,
primaryTimestamp,
secondaryTimestamp,
}: {
inputIndex: string[];
runtimeMappings: estypes.MappingRuntimeFields | undefined;
@ -56,6 +58,8 @@ export const threatMatchExecutor = async ({
buildRuleMessage: BuildRuleMessage;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
primaryTimestamp: string;
secondaryTimestamp?: string;
}) => {
const ruleParams = completeRule.ruleParams;
@ -89,6 +93,8 @@ export const threatMatchExecutor = async ({
type: ruleParams.type,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
});
});
};

2
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts
View file

@ -20,6 +20,7 @@ import { sampleEmptyDocSearchResults } from '../__mocks__/es_results';
import { allowedExperimentalValues } from '../../../../../common/experimental_features';
import type { ThresholdRuleParams } from '../../schemas/rule_schemas';
import { createRuleDataClientMock } from '@kbn/rule-registry-plugin/server/rule_data_client/rule_data_client.mock';
import { TIMESTAMP } from '@kbn/rule-data-utils';
describe('threshold_executor', () => {
const version = '8.0.0';
@ -75,6 +76,7 @@ describe('threshold_executor', () => {
ruleDataReader: ruleDataClientMock.getReader({ namespace: 'default' }),
runtimeMappings: {},
inputIndex: ['auditbeat-*'],
primaryTimestamp: TIMESTAMP,
});
expect(response.warningMessages.length).toEqual(1);
});

11
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts
View file

@ -59,6 +59,8 @@ export const thresholdExecutor = async ({
bulkCreate,
wrapHits,
ruleDataReader,
primaryTimestamp,
secondaryTimestamp,
}: {
inputIndex: string[];
runtimeMappings: estypes.MappingRuntimeFields | undefined;
@ -75,6 +77,8 @@ export const thresholdExecutor = async ({
bulkCreate: BulkCreate;
wrapHits: WrapHits;
ruleDataReader: IRuleDataReader;
primaryTimestamp: string;
secondaryTimestamp?: string;
}): Promise<SearchAfterAndBulkCreateReturnType & { state: ThresholdAlertState }> => {
let result = createSearchAfterReturnType();
const ruleParams = completeRule.ruleParams;
@ -114,7 +118,7 @@ export const thresholdExecutor = async ({
// Eliminate dupes
const bucketFilters = await getThresholdBucketFilters({
signalHistory,
timestampOverride: ruleParams.timestampOverride,
primaryTimestamp,
});
// Combine dupe filter with other filters
@ -142,9 +146,10 @@ export const thresholdExecutor = async ({
logger,
filter: esFilter,
threshold: ruleParams.threshold,
timestampOverride: ruleParams.timestampOverride,
buildRuleMessage,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
});
// Build and index new alerts
@ -168,7 +173,7 @@ export const thresholdExecutor = async ({
result,
createSearchAfterReturnTypeFromResponse({
searchResult: thresholdResults,
timestampOverride: ruleParams.timestampOverride,
primaryTimestamp,
}),
createSearchAfterReturnType({
success,

12
x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts
View file

@ -217,6 +217,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(5);
@ -310,6 +311,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(4);
@ -385,6 +387,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
@ -445,6 +448,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
@ -514,6 +518,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
@ -570,6 +575,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(1);
@ -639,6 +645,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(1);
@ -710,6 +717,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
@ -758,6 +766,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(createdSignalsCount).toEqual(0);
@ -805,6 +814,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(false);
expect(createdSignalsCount).toEqual(0); // should not create signals if search threw error
@ -930,6 +940,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(false);
expect(errors).toEqual(['error on creation']);
@ -1015,6 +1026,7 @@ describe('searchAfterAndBulkCreate', () => {
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(mockEnrichment).toHaveBeenCalledWith(

8
x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts
View file

@ -42,9 +42,10 @@ export const searchAfterAndBulkCreate = async ({
tuple,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
}: SearchAfterAndBulkCreateParams): Promise<SearchAfterAndBulkCreateReturnType> => {
return withSecuritySpan('searchAfterAndBulkCreate', async () => {
const ruleParams = completeRule.ruleParams;
let toReturn = createSearchAfterReturnType();
// sortId tells us where to start our next consecutive search_after query
@ -80,7 +81,8 @@ export const searchAfterAndBulkCreate = async ({
logger,
filter,
pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)),
timestampOverride: ruleParams.timestampOverride,
primaryTimestamp,
secondaryTimestamp,
trackTotalHits,
sortOrder,
});
@ -89,7 +91,7 @@ export const searchAfterAndBulkCreate = async ({
toReturn,
createSearchAfterReturnTypeFromResponse({
searchResult: mergedSearchResults,
timestampOverride: ruleParams.timestampOverride,
primaryTimestamp,
}),
createSearchAfterReturnType({
searchAfterTimes: [searchDuration],

15
x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts
View file

@ -43,7 +43,8 @@ describe('singleSearchAfter', () => {
logger: mockLogger,
pageSize: 1,
filter: {},
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
buildRuleMessage,
runtimeMappings: undefined,
});
@ -62,7 +63,8 @@ describe('singleSearchAfter', () => {
logger: mockLogger,
pageSize: 1,
filter: {},
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
buildRuleMessage,
runtimeMappings: undefined,
});
@ -113,7 +115,8 @@ describe('singleSearchAfter', () => {
logger: mockLogger,
pageSize: 1,
filter: {},
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
buildRuleMessage,
runtimeMappings: undefined,
});
@ -137,7 +140,8 @@ describe('singleSearchAfter', () => {
logger: mockLogger,
pageSize: 1,
filter: {},
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
buildRuleMessage,
runtimeMappings: undefined,
});
@ -158,7 +162,8 @@ describe('singleSearchAfter', () => {
logger: mockLogger,
pageSize: 1,
filter: {},
timestampOverride: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
buildRuleMessage,
runtimeMappings: undefined,
})

19
x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts
View file

@ -16,7 +16,10 @@ import type { SignalSearchResponse, SignalSource } from './types';
import type { BuildRuleMessage } from './rule_messages';
import { buildEventsSearchQuery } from './build_events_query';
import { createErrorsFromShard, makeFloatString } from './utils';
import type { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas';
import type {
TimestampOverride,
TimestampOverrideOrUndefined,
} from '../../../../common/detection_engine/schemas/common/schemas';
import { withSecuritySpan } from '../../../utils/with_security_span';
interface SingleSearchAfterParams {
@ -30,7 +33,8 @@ interface SingleSearchAfterParams {
pageSize: number;
sortOrder?: estypes.SortOrder;
filter: estypes.QueryDslQueryContainer;
timestampOverride: TimestampOverrideOrUndefined;
primaryTimestamp: TimestampOverride;
secondaryTimestamp: TimestampOverrideOrUndefined;
buildRuleMessage: BuildRuleMessage;
trackTotalHits?: boolean;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
@ -49,7 +53,8 @@ export const singleSearchAfter = async ({
logger,
pageSize,
sortOrder,
timestampOverride,
primaryTimestamp,
secondaryTimestamp,
buildRuleMessage,
trackTotalHits,
}: SingleSearchAfterParams): Promise<{
@ -69,7 +74,8 @@ export const singleSearchAfter = async ({
size: pageSize,
sortOrder,
searchAfterSortIds,
timestampOverride,
primaryTimestamp,
secondaryTimestamp,
trackTotalHits,
});
@ -93,8 +99,9 @@ export const singleSearchAfter = async ({
} catch (exc) {
logger.error(buildRuleMessage(`[-] nextSearchAfter threw an error ${exc}`));
if (
exc.message.includes('No mapping found for [@timestamp] in order to sort on') ||
exc.message.includes(`No mapping found for [${timestampOverride}] in order to sort on`)
exc.message.includes(`No mapping found for [${primaryTimestamp}] in order to sort on`) ||
(secondaryTimestamp &&
exc.message.includes(`No mapping found for [${secondaryTimestamp}] in order to sort on`))
) {
logger.error(buildRuleMessage(`[-] failure reason: ${exc.message}`));

4
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts
View file

@ -48,6 +48,8 @@ export const createEventSignal = async ({
threatPitId,
reassignThreatPitId,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
}: CreateEventSignalOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
const threatFilter = buildThreatMappingFilter({
threatMapping,
@ -143,6 +145,8 @@ export const createEventSignal = async ({
tuple,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
});
logger.debug(

4
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts
View file

@ -38,6 +38,8 @@ export const createThreatSignal = async ({
type,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
}: CreateThreatSignalOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
const threatFilter = buildThreatMappingFilter({
threatMapping,
@ -92,6 +94,8 @@ export const createThreatSignal = async ({
tuple,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
});
logger.debug(

10
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts
View file

@ -52,6 +52,8 @@ export const createThreatSignals = async ({
type,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
}: CreateThreatSignalsOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
const params = completeRule.ruleParams;
logger.debug(buildRuleMessage('Indicator matching rule starting'));
@ -84,6 +86,8 @@ export const createThreatSignals = async ({
query,
language,
filters: allEventFilters,
primaryTimestamp,
secondaryTimestamp,
});
logger.debug(`Total event count: ${eventCount}`);
@ -198,6 +202,8 @@ export const createThreatSignals = async ({
perPage,
tuple,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
}),
createSignal: (slicedChunk) =>
@ -233,6 +239,8 @@ export const createThreatSignals = async ({
type,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
}),
});
} else {
@ -283,6 +291,8 @@ export const createThreatSignals = async ({
type,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
}),
});
}

41
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_event_count.test.ts
View file

@ -24,6 +24,7 @@ describe('getEventCount', () => {
exceptionItems: [],
index: ['test-index'],
tuple: { to: moment('2022-01-14'), from: moment('2022-01-13'), maxSignals: 1337 },
primaryTimestamp: '@timestamp',
});
expect(esClient.count).toHaveBeenCalledWith({
@ -60,7 +61,8 @@ describe('getEventCount', () => {
exceptionItems: [],
index: ['test-index'],
tuple: { to: moment('2022-01-14'), from: moment('2022-01-13'), maxSignals: 1337 },
timestampOverride: 'event.ingested',
primaryTimestamp: 'event.ingested',
secondaryTimestamp: '@timestamp',
});
expect(esClient.count).toHaveBeenCalledWith({
@ -110,4 +112,41 @@ describe('getEventCount', () => {
index: ['test-index'],
});
});
it('can override timestamp without fallback to @timestamp', () => {
getEventCount({
esClient,
query: '*:*',
language: 'kuery',
filters: [],
exceptionItems: [],
index: ['test-index'],
tuple: { to: moment('2022-01-14'), from: moment('2022-01-13'), maxSignals: 1337 },
primaryTimestamp: 'event.ingested',
});
expect(esClient.count).toHaveBeenCalledWith({
body: {
query: {
bool: {
filter: [
{ bool: { must: [], filter: [], should: [], must_not: [] } },
{
range: {
'event.ingested': {
lte: '2022-01-14T05:00:00.000Z',
gte: '2022-01-13T05:00:00.000Z',
format: 'strict_date_optional_time',
},
},
},
{ match_all: {} },
],
},
},
},
ignore_unavailable: true,
index: ['test-index'],
});
});
});

12
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_event_count.ts
View file

@ -25,7 +25,8 @@ export const getEventList = async ({
buildRuleMessage,
logger,
tuple,
timestampOverride,
primaryTimestamp,
secondaryTimestamp,
runtimeMappings,
}: EventsOptions): Promise<estypes.SearchResponse<EventDoc>> => {
const calculatedPerPage = perPage ?? MAX_PER_PAGE;
@ -51,7 +52,8 @@ export const getEventList = async ({
logger,
filter,
pageSize: Math.ceil(Math.min(tuple.maxSignals, calculatedPerPage)),
timestampOverride,
primaryTimestamp,
secondaryTimestamp,
sortOrder: 'desc',
trackTotalHits: false,
runtimeMappings,
@ -71,7 +73,8 @@ export const getEventCount = async ({
index,
exceptionItems,
tuple,
timestampOverride,
primaryTimestamp,
secondaryTimestamp,
}: EventCountOptions): Promise<number> => {
const filter = getQueryFilter(query, language ?? 'kuery', filters, index, exceptionItems);
const eventSearchQueryBodyQuery = buildEventsSearchQuery({
@ -80,7 +83,8 @@ export const getEventCount = async ({
to: tuple.to.toISOString(),
filter,
size: 0,
timestampOverride,
primaryTimestamp,
secondaryTimestamp,
searchAfterSortIds: undefined,
runtimeMappings: undefined,
}).body.query;

12
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts
View file

@ -68,6 +68,8 @@ export interface CreateThreatSignalsOptions {
type: Type;
wrapHits: WrapHits;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
}
export interface CreateThreatSignalOptions {
@ -95,6 +97,8 @@ export interface CreateThreatSignalOptions {
type: Type;
wrapHits: WrapHits;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
}
export interface CreateEventSignalOptions {
@ -130,6 +134,8 @@ export interface CreateEventSignalOptions {
threatPitId: OpenPointInTimeResponse['id'];
reassignThreatPitId: (newPitId: OpenPointInTimeResponse['id'] | undefined) => void;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
}
type EntryKey = 'field' | 'value';
@ -261,7 +267,8 @@ export interface EventsOptions {
perPage?: number;
logger: Logger;
filters: unknown[];
timestampOverride?: string;
primaryTimestamp: string;
secondaryTimestamp?: string;
tuple: RuleRangeTuple;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
}
@ -279,7 +286,8 @@ export interface EventCountOptions {
query: string;
filters: unknown[];
tuple: RuleRangeTuple;
timestampOverride?: string;
primaryTimestamp: string;
secondaryTimestamp?: string;
}
export interface SignalMatch {

16
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.test.ts
View file

@ -12,6 +12,7 @@ import { mockLogger } from '../__mocks__/es_results';
import { buildRuleMessageFactory } from '../rule_messages';
import * as single_search_after from '../single_search_after';
import { findThresholdSignals } from './find_threshold_signals';
import { TIMESTAMP } from '@kbn/rule-data-utils';
const buildRuleMessage = buildRuleMessageFactory({
id: 'fake id',
@ -45,8 +46,9 @@ describe('findThresholdSignals', () => {
value: 100,
},
buildRuleMessage,
timestampOverride: undefined,
runtimeMappings: undefined,
primaryTimestamp: TIMESTAMP,
secondaryTimestamp: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
@ -90,8 +92,9 @@ describe('findThresholdSignals', () => {
value: 100,
},
buildRuleMessage,
timestampOverride: undefined,
runtimeMappings: undefined,
primaryTimestamp: TIMESTAMP,
secondaryTimestamp: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
@ -134,8 +137,9 @@ describe('findThresholdSignals', () => {
cardinality: [],
},
buildRuleMessage,
timestampOverride: undefined,
runtimeMappings: undefined,
primaryTimestamp: TIMESTAMP,
secondaryTimestamp: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
@ -192,8 +196,9 @@ describe('findThresholdSignals', () => {
],
},
buildRuleMessage,
timestampOverride: undefined,
runtimeMappings: undefined,
primaryTimestamp: TIMESTAMP,
secondaryTimestamp: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
@ -264,8 +269,9 @@ describe('findThresholdSignals', () => {
value: 200,
},
buildRuleMessage,
timestampOverride: undefined,
runtimeMappings: undefined,
primaryTimestamp: TIMESTAMP,
secondaryTimestamp: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({

21
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts
View file

@ -6,7 +6,6 @@
*/
import { set } from '@elastic/safer-lodash-set';
import { TIMESTAMP } from '@kbn/rule-data-utils';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type {
@ -15,8 +14,10 @@ import type {
RuleExecutorServices,
} from '@kbn/alerting-plugin/server';
import type { Logger } from '@kbn/core/server';
import type { ESBoolQuery } from '../../../../../common/typed_json';
import type {
ThresholdNormalized,
TimestampOverride,
TimestampOverrideOrUndefined,
} from '../../../../../common/detection_engine/schemas/common/schemas';
import type { BuildRuleMessage } from '../rule_messages';
@ -29,11 +30,12 @@ interface FindThresholdSignalsParams {
inputIndexPattern: string[];
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
logger: Logger;
filter: unknown;
filter: ESBoolQuery;
threshold: ThresholdNormalized;
buildRuleMessage: BuildRuleMessage;
timestampOverride: TimestampOverrideOrUndefined;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: TimestampOverride;
secondaryTimestamp: TimestampOverrideOrUndefined;
}
export const findThresholdSignals = async ({
@ -45,8 +47,9 @@ export const findThresholdSignals = async ({
filter,
threshold,
buildRuleMessage,
timestampOverride,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
}: FindThresholdSignalsParams): Promise<{
searchResult: SignalSearchResponse;
searchDuration: string;
@ -56,12 +59,12 @@ export const findThresholdSignals = async ({
const leafAggs = {
max_timestamp: {
max: {
field: timestampOverride != null ? timestampOverride : TIMESTAMP,
field: primaryTimestamp,
},
},
min_timestamp: {
min: {
field: timestampOverride != null ? timestampOverride : TIMESTAMP,
field: primaryTimestamp,
},
},
...(threshold.cardinality?.length
@ -134,18 +137,18 @@ export const findThresholdSignals = async ({
return singleSearchAfter({
aggregations,
searchAfterSortId: undefined,
timestampOverride,
searchAfterSortIds: undefined,
index: inputIndexPattern,
from,
to,
services,
logger,
// @ts-expect-error refactor to pass type explicitly instead of unknown
filter,
pageSize: 0,
sortOrder: 'desc',
buildRuleMessage,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
});
};

3
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.test.ts
View file

@ -5,6 +5,7 @@
* 2.0.
*/
import { TIMESTAMP } from '@kbn/rule-data-utils';
import { sampleThresholdSignalHistory } from '../__mocks__/threshold_signal_history.mock';
import { getThresholdBucketFilters } from './get_threshold_bucket_filters';
@ -12,7 +13,7 @@ describe('getThresholdBucketFilters', () => {
it('should generate filters for threshold signal detection with dupe mitigation', async () => {
const result = await getThresholdBucketFilters({
signalHistory: sampleThresholdSignalHistory(),
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
expect(result).toEqual([
{

6
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.ts
View file

@ -15,10 +15,10 @@ import type { ThresholdSignalHistory, ThresholdSignalHistoryRecord } from '../ty
*/
export const getThresholdBucketFilters = async ({
signalHistory,
timestampOverride,
primaryTimestamp,
}: {
signalHistory: ThresholdSignalHistory;
timestampOverride: string | undefined;
primaryTimestamp: string;
}): Promise<Filter[]> => {
const filters = Object.values(signalHistory).reduce(
(acc: ESFilter[], bucket: ThresholdSignalHistoryRecord): ESFilter[] => {
@ -27,7 +27,7 @@ export const getThresholdBucketFilters = async ({
filter: [
{
range: {
[timestampOverride ?? '@timestamp']: {
[primaryTimestamp]: {
// Timestamp of last event signaled on for this set of terms.
lte: new Date(bucket.lastSignalTimestamp).toISOString(),
},

2
x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts
View file

@ -328,6 +328,8 @@ export interface SearchAfterAndBulkCreateParams {
trackTotalHits?: boolean;
sortOrder?: estypes.SortOrder;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
}
export interface SearchAfterAndBulkCreateReturnType {

56
x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts
View file

@ -8,7 +8,7 @@
import moment from 'moment';
import sinon from 'sinon';
import type { TransportResult } from '@elastic/elasticsearch';
import { ALERT_REASON, ALERT_RULE_PARAMETERS, ALERT_UUID } from '@kbn/rule-data-utils';
import { ALERT_REASON, ALERT_RULE_PARAMETERS, ALERT_UUID, TIMESTAMP } from '@kbn/rule-data-utils';
import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks';
import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
@ -981,7 +981,7 @@ describe('utils', () => {
const searchResult = sampleEmptyDocSearchResults();
const newSearchResult = createSearchAfterReturnTypeFromResponse({
searchResult,
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
const expected: SearchAfterAndBulkCreateReturnType = {
bulkCreateTimes: [],
@ -1001,7 +1001,7 @@ describe('utils', () => {
const searchResult = sampleDocSearchResultsWithSortId();
const newSearchResult = createSearchAfterReturnTypeFromResponse({
searchResult,
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
const expected: SearchAfterAndBulkCreateReturnType = {
bulkCreateTimes: [],
@ -1024,7 +1024,7 @@ describe('utils', () => {
searchResult._shards.failures = [{ reason: { reason: 'Not a sort failure' } }];
const { success } = createSearchAfterReturnTypeFromResponse({
searchResult,
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
expect(success).toEqual(false);
});
@ -1036,7 +1036,7 @@ describe('utils', () => {
searchResult._shards.failures = [{ reason: { reason: 'Not a sort failure' } }];
const { success } = createSearchAfterReturnTypeFromResponse({
searchResult,
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
expect(success).toEqual(false);
});
@ -1052,7 +1052,7 @@ describe('utils', () => {
];
const { success } = createSearchAfterReturnTypeFromResponse({
searchResult,
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
expect(success).toEqual(false);
});
@ -1062,7 +1062,7 @@ describe('utils', () => {
searchResult._shards.failed = 0;
const { success } = createSearchAfterReturnTypeFromResponse({
searchResult,
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
expect(success).toEqual(true);
});
@ -1078,7 +1078,7 @@ describe('utils', () => {
];
const { success } = createSearchAfterReturnTypeFromResponse({
searchResult,
timestampOverride: 'event.ingested',
primaryTimestamp: 'event.ingested',
});
expect(success).toEqual(true);
});
@ -1091,7 +1091,7 @@ describe('utils', () => {
}
const { lastLookBackDate } = createSearchAfterReturnTypeFromResponse({
searchResult,
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
expect(lastLookBackDate).toEqual(null);
});
@ -1104,7 +1104,7 @@ describe('utils', () => {
}
const { lastLookBackDate } = createSearchAfterReturnTypeFromResponse({
searchResult,
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
expect(lastLookBackDate).toEqual(null);
});
@ -1117,7 +1117,7 @@ describe('utils', () => {
}
const { lastLookBackDate } = createSearchAfterReturnTypeFromResponse({
searchResult,
timestampOverride: undefined,
primaryTimestamp: TIMESTAMP,
});
expect(lastLookBackDate).toEqual(null);
});
@ -1130,7 +1130,7 @@ describe('utils', () => {
if (searchResult.hits.hits[0].fields != null) {
(searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null;
}
const date = lastValidDate({ searchResult, timestampOverride: undefined });
const date = lastValidDate({ searchResult, primaryTimestamp: TIMESTAMP });
expect(date).toEqual(undefined);
});
@ -1140,7 +1140,7 @@ describe('utils', () => {
if (searchResult.hits.hits[0].fields != null) {
(searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined;
}
const date = lastValidDate({ searchResult, timestampOverride: undefined });
const date = lastValidDate({ searchResult, primaryTimestamp: TIMESTAMP });
expect(date).toEqual(undefined);
});
@ -1150,13 +1150,13 @@ describe('utils', () => {
if (searchResult.hits.hits[0].fields != null) {
(searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid value'];
}
const date = lastValidDate({ searchResult, timestampOverride: undefined });
const date = lastValidDate({ searchResult, primaryTimestamp: TIMESTAMP });
expect(date).toEqual(undefined);
});
test('It returns normal date time if set', () => {
const searchResult = sampleDocSearchResultsNoSortId();
const date = lastValidDate({ searchResult, timestampOverride: undefined });
const date = lastValidDate({ searchResult, primaryTimestamp: TIMESTAMP });
expect(date?.toISOString()).toEqual('2020-04-20T21:27:45.000Z');
});
@ -1172,7 +1172,7 @@ describe('utils', () => {
'@timestamp': [timestamp],
},
};
const date = lastValidDate({ searchResult, timestampOverride: undefined });
const date = lastValidDate({ searchResult, primaryTimestamp: TIMESTAMP });
expect(date?.toISOString()).toEqual(timestamp);
});
@ -1180,7 +1180,7 @@ describe('utils', () => {
const override = '2020-10-07T19:20:28.049Z';
const searchResult = sampleDocSearchResultsNoSortId();
searchResult.hits.hits[0]._source.different_timestamp = new Date(override).toISOString();
const date = lastValidDate({ searchResult, timestampOverride: 'different_timestamp' });
const date = lastValidDate({ searchResult, primaryTimestamp: 'different_timestamp' });
expect(date?.toISOString()).toEqual(override);
});
@ -1196,7 +1196,7 @@ describe('utils', () => {
different_timestamp: [override],
},
};
const date = lastValidDate({ searchResult, timestampOverride: 'different_timestamp' });
const date = lastValidDate({ searchResult, primaryTimestamp: 'different_timestamp' });
expect(date?.toISOString()).toEqual(override);
});
});
@ -1208,7 +1208,7 @@ describe('utils', () => {
if (doc.fields != null) {
(doc.fields['@timestamp'] as unknown) = null;
}
const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
const date = getValidDateFromDoc({ doc, primaryTimestamp: TIMESTAMP });
expect(date).toEqual(undefined);
});
@ -1218,7 +1218,7 @@ describe('utils', () => {
if (doc.fields != null) {
(doc.fields['@timestamp'] as unknown) = undefined;
}
const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
const date = getValidDateFromDoc({ doc, primaryTimestamp: TIMESTAMP });
expect(date).toEqual(undefined);
});
@ -1228,13 +1228,13 @@ describe('utils', () => {
if (doc.fields != null) {
(doc.fields['@timestamp'] as unknown) = ['invalid value'];
}
const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
const date = getValidDateFromDoc({ doc, primaryTimestamp: TIMESTAMP });
expect(date).toEqual(undefined);
});
test('It returns normal date time if set', () => {
const doc = sampleDocNoSortId();
const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
const date = getValidDateFromDoc({ doc, primaryTimestamp: TIMESTAMP });
expect(date?.toISOString()).toEqual('2020-04-20T21:27:45.000Z');
});
@ -1250,7 +1250,7 @@ describe('utils', () => {
'@timestamp': [timestamp],
},
};
const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
const date = getValidDateFromDoc({ doc, primaryTimestamp: TIMESTAMP });
expect(date?.toISOString()).toEqual(timestamp);
});
@ -1258,7 +1258,7 @@ describe('utils', () => {
const override = '2020-10-07T19:20:28.049Z';
const doc = sampleDocNoSortId();
doc._source.different_timestamp = new Date(override).toISOString();
const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' });
const date = getValidDateFromDoc({ doc, primaryTimestamp: 'different_timestamp' });
expect(date?.toISOString()).toEqual(override);
});
@ -1274,7 +1274,7 @@ describe('utils', () => {
different_timestamp: [override],
},
};
const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' });
const date = getValidDateFromDoc({ doc, primaryTimestamp: 'different_timestamp' });
expect(date?.toISOString()).toEqual(override);
});
@ -1286,7 +1286,7 @@ describe('utils', () => {
if (doc.fields != null) {
doc.fields['@timestamp'] = [testDate];
}
const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
const date = getValidDateFromDoc({ doc, primaryTimestamp: TIMESTAMP });
expect(date?.toISOString()).toEqual(testDateString);
});
@ -1296,7 +1296,7 @@ describe('utils', () => {
const testDate = `${new Date(testDateString).valueOf()}`;
doc._source['@timestamp'] = testDate;
doc.fields = undefined;
const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
const date = getValidDateFromDoc({ doc, primaryTimestamp: TIMESTAMP });
expect(date?.toISOString()).toEqual(testDateString);
});
@ -1313,7 +1313,7 @@ describe('utils', () => {
different_timestamp: [testDate],
},
};
const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' });
const date = getValidDateFromDoc({ doc, primaryTimestamp: 'different_timestamp' });
expect(date?.toISOString()).toEqual(override);
});
});

31
x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts
View file

@ -34,7 +34,7 @@ import type {
import { parseDuration } from '@kbn/alerting-plugin/server';
import type { ExceptionListClient, ListClient, ListPluginSetup } from '@kbn/lists-plugin/server';
import type {
TimestampOverrideOrUndefined,
TimestampOverride,
Privilege,
} from '../../../../common/detection_engine/schemas/common';
import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common';
@ -604,20 +604,20 @@ export const createErrorsFromShard = ({ errors }: { errors: ShardError[] }): str
* it cannot it will resort to using the "_source" fields second which can be problematic if the date time
* is not correctly ISO8601 or epoch milliseconds formatted.
* @param searchResult The result to try and parse out the timestamp.
* @param timestampOverride The timestamp override to use its values if we have it.
* @param primaryTimestamp The primary timestamp to use.
*/
export const lastValidDate = ({
searchResult,
timestampOverride,
primaryTimestamp,
}: {
searchResult: SignalSearchResponse;
timestampOverride: TimestampOverrideOrUndefined;
primaryTimestamp: TimestampOverride;
}): Date | undefined => {
if (searchResult.hits.hits.length === 0) {
return undefined;
} else {
const lastRecord = searchResult.hits.hits[searchResult.hits.hits.length - 1];
return getValidDateFromDoc({ doc: lastRecord, timestampOverride });
return getValidDateFromDoc({ doc: lastRecord, primaryTimestamp });
}
};
@ -627,21 +627,20 @@ export const lastValidDate = ({
* it cannot it will resort to using the "_source" fields second which can be problematic if the date time
* is not correctly ISO8601 or epoch milliseconds formatted.
* @param searchResult The result to try and parse out the timestamp.
* @param timestampOverride The timestamp override to use its values if we have it.
* @param primaryTimestamp The primary timestamp to use.
*/
export const getValidDateFromDoc = ({
doc,
timestampOverride,
primaryTimestamp,
}: {
doc: BaseSignalHit;
timestampOverride: TimestampOverrideOrUndefined;
primaryTimestamp: TimestampOverride;
}): Date | undefined => {
const timestamp = timestampOverride ?? '@timestamp';
const timestampValue =
doc.fields != null && doc.fields[timestamp] != null
? doc.fields[timestamp][0]
doc.fields != null && doc.fields[primaryTimestamp] != null
? doc.fields[primaryTimestamp][0]
: doc._source != null
? (doc._source as { [key: string]: unknown })[timestamp]
? (doc._source as { [key: string]: unknown })[primaryTimestamp]
: undefined;
const lastTimestamp =
typeof timestampValue === 'string' || typeof timestampValue === 'number'
@ -668,10 +667,10 @@ export const getValidDateFromDoc = ({
export const createSearchAfterReturnTypeFromResponse = ({
searchResult,
timestampOverride,
primaryTimestamp,
}: {
searchResult: SignalSearchResponse;
timestampOverride: TimestampOverrideOrUndefined;
primaryTimestamp: TimestampOverride;
}): SearchAfterAndBulkCreateReturnType => {
return createSearchAfterReturnType({
success:
@ -682,11 +681,11 @@ export const createSearchAfterReturnTypeFromResponse = ({
'No mapping found for [@timestamp] in order to sort on'
) ||
failure.reason?.reason?.includes(
`No mapping found for [${timestampOverride}] in order to sort on`
`No mapping found for [${primaryTimestamp}] in order to sort on`
)
);
}),
lastLookBackDate: lastValidDate({ searchResult, timestampOverride }),
lastLookBackDate: lastValidDate({ searchResult, primaryTimestamp }),
});
};

62
x-pack/test/detection_engine_api_integration/security_and_spaces/group1/timestamps.ts
View file

@ -22,6 +22,7 @@ import {
createRule,
waitForRuleSuccessOrStatus,
waitForSignalsToBePresent,
getOpenSignals,
getRuleForSignalTesting,
getSignalsByIds,
getEqlRuleForSignalTesting,
@ -31,6 +32,7 @@ import {
export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertest');
const esArchiver = getService('esArchiver');
const es = getService('es');
const log = getService('log');
/**
@ -184,6 +186,30 @@ export default ({ getService }: FtrProviderContext) => {
expect(signalsOrderedByEventId.length).equal(3);
});
it('should generate 2 signals with event.ingested when timestamp fallback is disabled', async () => {
const rule: QueryCreateSchema = {
...getRuleForSignalTesting(['myfa*']),
rule_id: 'rule-without-timestamp-fallback',
timestamp_override: 'event.ingested',
timestamp_override_fallback_disabled: true,
};
const { id } = await createRule(supertest, log, rule);
await waitForRuleSuccessOrStatus(
supertest,
log,
id,
RuleExecutionStatus['partial failure']
);
await waitForSignalsToBePresent(supertest, log, 2, [id]);
const signalsResponse = await getSignalsByIds(supertest, log, [id], 2);
const signals = signalsResponse.hits.hits.map((hit) => hit._source);
const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
expect(signalsOrderedByEventId.length).equal(2);
});
it('should generate 2 signals with @timestamp', async () => {
const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']);
@ -224,6 +250,25 @@ export default ({ getService }: FtrProviderContext) => {
expect(signalsOrderedByEventId.length).equal(2);
});
it('should not generate any signals when timestamp override does not exist and timestamp fallback is disabled', async () => {
const rule: QueryCreateSchema = {
...getRuleForSignalTesting(['myfa*']),
rule_id: 'rule-without-timestamp-fallback',
timestamp_override: 'event.fakeingestfield',
timestamp_override_fallback_disabled: true,
};
const createdRule = await createRule(supertest, log, rule);
const signalsOpen = await getOpenSignals(
supertest,
log,
es,
createdRule,
RuleExecutionStatus['partial failure']
);
expect(signalsOpen.hits.hits.length).eql(0);
});
/**
* We should not use the timestamp override as the "original_time" as that can cause
* confusion if you have both a timestamp and an override in the source event. Instead the "original_time"
@ -287,6 +332,23 @@ export default ({ getService }: FtrProviderContext) => {
expect(signalsOrderedByEventId.length).equal(2);
});
it('should not generate any signals when timestamp override does not exist and timestamp fallback is disabled', async () => {
const rule: EqlCreateSchema = {
...getEqlRuleForSignalTesting(['myfa*']),
timestamp_override: 'event.fakeingestfield',
timestamp_override_fallback_disabled: true,
};
const createdRule = await createRule(supertest, log, rule);
const signalsOpen = await getOpenSignals(
supertest,
log,
es,
createdRule,
RuleExecutionStatus['partial failure']
);
expect(signalsOpen.hits.hits.length).eql(0);
});
});
});

6
x-pack/test/detection_engine_api_integration/utils/get_open_signals.ts
View file

@ -8,6 +8,7 @@
import type { ToolingLog } from '@kbn/tooling-log';
import type SuperTest from 'supertest';
import type { Client } from '@elastic/elasticsearch';
import { RuleExecutionStatus } from '@kbn/security-solution-plugin/common/detection_engine/schemas/common';
import type { FullResponseSchema } from '@kbn/security-solution-plugin/common/detection_engine/schemas/request';
import { waitForRuleSuccessOrStatus } from './wait_for_rule_success_or_status';
@ -18,9 +19,10 @@ export const getOpenSignals = async (
supertest: SuperTest.SuperTest<SuperTest.Test>,
log: ToolingLog,
es: Client,
rule: FullResponseSchema
rule: FullResponseSchema,
status: RuleExecutionStatus = RuleExecutionStatus.succeeded
) => {
await waitForRuleSuccessOrStatus(supertest, log, rule.id);
await waitForRuleSuccessOrStatus(supertest, log, rule.id, status);
// Critically important that we wait for rule success AND refresh the write index in that order before we
// assert that no signals were created. Otherwise, signals could be written but not available to query yet
// when we search, causing tests that check that signals are NOT created to pass when they should fail.