mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[Security Solution][Alerts] New terms security rule type (#134526)
* WIP new value rule type * Finish implementation and add integration tests * Remove experimental value list exception implementation * Reorganize aggregation and runtime mapping builders * Add new terms field to UI and tests * Add new fields in more places * Add Cypress test for new terms rule creation * Change historyWindowStart references on UI to historyWindowSize * Fix more tests that break when more rule types are added * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Fix UI form schema description * Test implementation for phased new terms search implementation * New terms using composite agg for history search phase * Implementation using terms agg for phase 2 * Add alert creation logic back, add more unit tests * Update buildNewTermsAggregation snapshot * Type and test fixes * Fix merge * More merge conflict fixes * Mock and test fixes * API test fix * More test fixes * Try fixing cypress test * Fix cypress test again * Fix new terms field text * Add new terms rule type to patch converter function * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * UX feedback: rule card icon and field box description * Fix types post merge main * Remove duplicate switch case * Add special investigate in timeline action for new terms alerts * PR comments: naming, improved schema error message * PR comments: update cypress test, fix copied error messages * Add README in new terms folder Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Marshall Main
GitHub
parent
4bc0cb7cae
commit
0fe480c87b
77 changed files with 2764 additions and 60 deletions
|
|
@ -15,6 +15,7 @@ export const type = t.keyof({
|
|||
saved_query: null,
|
||||
threshold: null,
|
||||
threat_match: null,
|
||||
new_terms: null,
|
||||
});
|
||||
export type Type = t.TypeOf<typeof type>;
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ export * from './empty_string_array';
|
|||
export * from './enumeration';
|
||||
export * from './iso_date_string';
|
||||
export * from './import_query_schema';
|
||||
export * from './limited_size_array';
|
||||
export * from './non_empty_array';
|
||||
export * from './non_empty_or_nullable_string_array';
|
||||
export * from './non_empty_string_array';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,112 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import * as t from 'io-ts';
|
||||
|
||||
import { pipe } from 'fp-ts/lib/pipeable';
|
||||
import { left } from 'fp-ts/lib/Either';
|
||||
import { LimitedSizeArray } from '.';
|
||||
import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils';
|
||||
|
||||
const testSchema = t.keyof({
|
||||
valid: true,
|
||||
also_valid: true,
|
||||
});
|
||||
type TestSchema = t.TypeOf<typeof testSchema>;
|
||||
|
||||
const limitedSizeArraySchema = LimitedSizeArray({
|
||||
codec: testSchema,
|
||||
minSize: 1,
|
||||
maxSize: 2,
|
||||
name: 'TestSchemaArray',
|
||||
});
|
||||
|
||||
describe('limited size array', () => {
|
||||
test('it should generate the correct name for limited size array', () => {
|
||||
const newTestSchema = LimitedSizeArray({ codec: testSchema });
|
||||
expect(newTestSchema.name).toEqual('LimitedSizeArray<"valid" | "also_valid">');
|
||||
});
|
||||
|
||||
test('it should use a supplied name override', () => {
|
||||
const newTestSchema = LimitedSizeArray({ codec: testSchema, name: 'someName' });
|
||||
expect(newTestSchema.name).toEqual('someName');
|
||||
});
|
||||
|
||||
test('it should not validate an array smaller than min size', () => {
|
||||
const payload: string[] = [];
|
||||
const decoded = limitedSizeArraySchema.decode(payload);
|
||||
const message = pipe(decoded, foldLeftRight);
|
||||
|
||||
expect(getPaths(left(message.errors))).toEqual([
|
||||
'Array size (0) is out of bounds: min: 1, max: 2',
|
||||
]);
|
||||
expect(message.schema).toEqual({});
|
||||
});
|
||||
|
||||
test('it should validate an array of testSchema that is within min and max size', () => {
|
||||
const payload: TestSchema[] = ['valid'];
|
||||
const decoded = limitedSizeArraySchema.decode(payload);
|
||||
const message = pipe(decoded, foldLeftRight);
|
||||
|
||||
expect(getPaths(left(message.errors))).toEqual([]);
|
||||
expect(message.schema).toEqual(payload);
|
||||
});
|
||||
|
||||
test('it should validate an array of valid testSchema strings that is within min and max size', () => {
|
||||
const payload: TestSchema[] = ['valid', 'also_valid'];
|
||||
const decoded = limitedSizeArraySchema.decode(payload);
|
||||
const message = pipe(decoded, foldLeftRight);
|
||||
|
||||
expect(getPaths(left(message.errors))).toEqual([]);
|
||||
expect(message.schema).toEqual(payload);
|
||||
});
|
||||
|
||||
test('it should not validate an array bigger than max size', () => {
|
||||
const payload: TestSchema[] = ['valid', 'also_valid', 'also_valid'];
|
||||
const decoded = limitedSizeArraySchema.decode(payload);
|
||||
const message = pipe(decoded, foldLeftRight);
|
||||
|
||||
expect(getPaths(left(message.errors))).toEqual([
|
||||
'Array size (3) is out of bounds: min: 1, max: 2',
|
||||
]);
|
||||
expect(message.schema).toEqual({});
|
||||
});
|
||||
|
||||
test('it should not validate an array with a number', () => {
|
||||
const payload = ['valid', 123];
|
||||
const decoded = limitedSizeArraySchema.decode(payload);
|
||||
const message = pipe(decoded, foldLeftRight);
|
||||
|
||||
expect(getPaths(left(message.errors))).toEqual([
|
||||
'Invalid value "123" supplied to "TestSchemaArray"',
|
||||
]);
|
||||
expect(message.schema).toEqual({});
|
||||
});
|
||||
|
||||
test('it should not validate an array with an invalid string', () => {
|
||||
const payload = ['valid', 'invalid'];
|
||||
const decoded = limitedSizeArraySchema.decode(payload);
|
||||
const message = pipe(decoded, foldLeftRight);
|
||||
|
||||
expect(getPaths(left(message.errors))).toEqual([
|
||||
'Invalid value "invalid" supplied to "TestSchemaArray"',
|
||||
]);
|
||||
expect(message.schema).toEqual({});
|
||||
});
|
||||
|
||||
test('it should not validate a null value', () => {
|
||||
const payload = null;
|
||||
const decoded = limitedSizeArraySchema.decode(payload);
|
||||
const message = pipe(decoded, foldLeftRight);
|
||||
|
||||
expect(getPaths(left(message.errors))).toEqual([
|
||||
'Invalid value "null" supplied to "TestSchemaArray"',
|
||||
]);
|
||||
expect(message.schema).toEqual({});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import * as t from 'io-ts';
|
||||
import { Either } from 'fp-ts/lib/Either';
|
||||
|
||||
export const LimitedSizeArray = <C extends t.Mixed>({
|
||||
codec,
|
||||
minSize,
|
||||
maxSize,
|
||||
name = `LimitedSizeArray<${codec.name}>`,
|
||||
}: {
|
||||
codec: C;
|
||||
minSize?: number;
|
||||
maxSize?: number;
|
||||
name?: string;
|
||||
}) => {
|
||||
const arrType = t.array(codec);
|
||||
type ArrType = t.TypeOf<typeof arrType>;
|
||||
return new t.Type<ArrType, ArrType, unknown>(
|
||||
name,
|
||||
arrType.is,
|
||||
(input, context): Either<t.Errors, ArrType> => {
|
||||
if (
|
||||
Array.isArray(input) &&
|
||||
((minSize && input.length < minSize) || (maxSize && input.length > maxSize))
|
||||
) {
|
||||
return t.failure(
|
||||
input,
|
||||
context,
|
||||
`Array size (${input.length}) is out of bounds: min: ${
|
||||
minSize ?? 'not specified'
|
||||
}, max: ${maxSize ?? 'not specified'}`
|
||||
);
|
||||
} else {
|
||||
return arrType.validate(input, context);
|
||||
}
|
||||
},
|
||||
t.identity
|
||||
);
|
||||
};
|
||||
|
|
@ -21,3 +21,4 @@ export const ML_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.mlRule` as const;
|
|||
export const QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.queryRule` as const;
|
||||
export const SAVED_QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.savedQueryRule` as const;
|
||||
export const THRESHOLD_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.thresholdRule` as const;
|
||||
export const NEW_TERMS_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.newTermsRule` as const;
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
EQL_RULE_TYPE_ID,
|
||||
INDICATOR_RULE_TYPE_ID,
|
||||
ML_RULE_TYPE_ID,
|
||||
NEW_TERMS_RULE_TYPE_ID,
|
||||
QUERY_RULE_TYPE_ID,
|
||||
SAVED_QUERY_RULE_TYPE_ID,
|
||||
THRESHOLD_RULE_TYPE_ID,
|
||||
|
|
@ -25,6 +26,7 @@ export const ruleTypeMappings = {
|
|||
saved_query: SAVED_QUERY_RULE_TYPE_ID,
|
||||
threat_match: INDICATOR_RULE_TYPE_ID,
|
||||
threshold: THRESHOLD_RULE_TYPE_ID,
|
||||
new_terms: NEW_TERMS_RULE_TYPE_ID,
|
||||
};
|
||||
type RuleTypeMappings = typeof ruleTypeMappings;
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
|
|||
...options,
|
||||
services: {
|
||||
...options.services,
|
||||
alertWithPersistence: async (alerts, refresh) => {
|
||||
alertWithPersistence: async (alerts, refresh, maxAlerts = undefined) => {
|
||||
const numAlerts = alerts.length;
|
||||
logger.debug(`Found ${numAlerts} alerts.`);
|
||||
|
||||
|
|
@ -82,7 +82,13 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
|
|||
}
|
||||
|
||||
if (filteredAlerts.length === 0) {
|
||||
return { createdAlerts: [], errors: {} };
|
||||
return { createdAlerts: [], errors: {}, alertsWereTruncated: false };
|
||||
}
|
||||
|
||||
let alertsWereTruncated = false;
|
||||
if (maxAlerts && filteredAlerts.length > maxAlerts) {
|
||||
filteredAlerts.length = maxAlerts;
|
||||
alertsWereTruncated = true;
|
||||
}
|
||||
|
||||
const augmentedAlerts = filteredAlerts.map((alert) => {
|
||||
|
|
@ -105,7 +111,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
|
|||
});
|
||||
|
||||
if (response == null) {
|
||||
return { createdAlerts: [], errors: {} };
|
||||
return { createdAlerts: [], errors: {}, alertsWereTruncated };
|
||||
}
|
||||
|
||||
return {
|
||||
|
|
@ -120,10 +126,11 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
|
|||
})
|
||||
.filter((_, idx) => response.body.items[idx].create?.status === 201),
|
||||
errors: errorAggregator(response.body, [409]),
|
||||
alertsWereTruncated,
|
||||
};
|
||||
} else {
|
||||
logger.debug('Writing is disabled.');
|
||||
return { createdAlerts: [], errors: {} };
|
||||
return { createdAlerts: [], errors: {}, alertsWereTruncated: false };
|
||||
}
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -24,12 +24,14 @@ export type PersistenceAlertService = <T>(
|
|||
_id: string;
|
||||
_source: T;
|
||||
}>,
|
||||
refresh: boolean | 'wait_for'
|
||||
refresh: boolean | 'wait_for',
|
||||
maxAlerts?: number
|
||||
) => Promise<PersistenceAlertServiceResult<T>>;
|
||||
|
||||
export interface PersistenceAlertServiceResult<T> {
|
||||
createdAlerts: Array<AlertWithCommonFieldsLatest<T> & { _id: string; _index: string }>;
|
||||
errors: BulkResponseErrorAggregation;
|
||||
alertsWereTruncated: boolean;
|
||||
}
|
||||
|
||||
export interface PersistenceServices {
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import type { AlertWithCommonFields800 } from '@kbn/rule-registry-plugin/common/
|
|||
import type {
|
||||
ALERT_GROUP_ID,
|
||||
ALERT_GROUP_INDEX,
|
||||
ALERT_NEW_TERMS,
|
||||
ALERT_RULE_INDICES,
|
||||
} from '../../../../field_maps/field_names';
|
||||
import type {
|
||||
|
|
@ -19,38 +20,48 @@ import type {
|
|||
EqlShellAlert800,
|
||||
} from '../8.0.0';
|
||||
|
||||
export type { Ancestor800 as Ancestor830 };
|
||||
export type { Ancestor800 as Ancestor840 };
|
||||
|
||||
/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.3.0.
|
||||
Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.3.0.
|
||||
/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.4.0.
|
||||
Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.4.0.
|
||||
If you are adding new fields for a new release of Kibana, create a new sibling folder to this one
|
||||
for the version to be released and add the field(s) to the schema in that folder.
|
||||
Then, update `../index.ts` to import from the new folder that has the latest schemas, add the
|
||||
new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas.
|
||||
*/
|
||||
export interface BaseFields830 extends BaseFields800 {
|
||||
export interface BaseFields840 extends BaseFields800 {
|
||||
[ALERT_RULE_INDICES]: string[];
|
||||
}
|
||||
|
||||
export interface WrappedFields830<T extends BaseFields830> {
|
||||
export interface WrappedFields840<T extends BaseFields840> {
|
||||
_id: string;
|
||||
_index: string;
|
||||
_source: T & { [ALERT_UUID]: string };
|
||||
}
|
||||
|
||||
export type GenericAlert830 = AlertWithCommonFields800<BaseFields830>;
|
||||
export type GenericAlert840 = AlertWithCommonFields800<BaseFields840>;
|
||||
|
||||
// This is the type of the final generated alert including base fields, common fields
|
||||
// added by the alertWithPersistence function, and arbitrary fields copied from source documents
|
||||
export type DetectionAlert830 = GenericAlert830 | EqlShellAlert800 | EqlBuildingBlockAlert800;
|
||||
|
||||
export interface EqlShellFields830 extends BaseFields830 {
|
||||
export interface EqlShellFields840 extends BaseFields840 {
|
||||
[ALERT_GROUP_ID]: string;
|
||||
[ALERT_UUID]: string;
|
||||
}
|
||||
|
||||
export interface EqlBuildingBlockFields830 extends BaseFields830 {
|
||||
export interface EqlBuildingBlockFields840 extends BaseFields840 {
|
||||
[ALERT_GROUP_ID]: string;
|
||||
[ALERT_GROUP_INDEX]: number;
|
||||
[ALERT_BUILDING_BLOCK_TYPE]: 'default';
|
||||
}
|
||||
|
||||
export interface NewTermsFields840 extends BaseFields840 {
|
||||
[ALERT_NEW_TERMS]: Array<string | number | null>;
|
||||
}
|
||||
|
||||
export type NewTermsAlert840 = AlertWithCommonFields800<NewTermsFields840>;
|
||||
|
||||
// This is the type of the final generated alert including base fields, common fields
|
||||
// added by the alertWithPersistence function, and arbitrary fields copied from source documents
|
||||
export type DetectionAlert840 =
|
||||
| GenericAlert840
|
||||
| EqlShellAlert800
|
||||
| EqlBuildingBlockAlert800
|
||||
| NewTermsAlert840;
|
||||
|
|
@ -5,26 +5,28 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import type {
|
||||
EqlBuildingBlockFields830,
|
||||
EqlShellFields830,
|
||||
WrappedFields830,
|
||||
DetectionAlert830,
|
||||
BaseFields830,
|
||||
Ancestor830,
|
||||
} from './8.3.0';
|
||||
|
||||
import type { DetectionAlert800 } from './8.0.0';
|
||||
|
||||
import type {
|
||||
Ancestor840,
|
||||
BaseFields840,
|
||||
DetectionAlert840,
|
||||
WrappedFields840,
|
||||
EqlBuildingBlockFields840,
|
||||
EqlShellFields840,
|
||||
NewTermsFields840,
|
||||
} from './8.4.0';
|
||||
|
||||
// When new Alert schemas are created for new Kibana versions, add the DetectionAlert type from the new version
|
||||
// here, e.g. `export type DetectionAlert = DetectionAlert800 | DetectionAlert820` if a new schema is created in 8.2.0
|
||||
export type DetectionAlert = DetectionAlert800 | DetectionAlert830;
|
||||
export type DetectionAlert = DetectionAlert800 | DetectionAlert840;
|
||||
|
||||
export type {
|
||||
Ancestor830 as AncestorLatest,
|
||||
BaseFields830 as BaseFieldsLatest,
|
||||
DetectionAlert830 as DetectionAlertLatest,
|
||||
WrappedFields830 as WrappedFieldsLatest,
|
||||
EqlBuildingBlockFields830 as EqlBuildingBlockFieldsLatest,
|
||||
EqlShellFields830 as EqlShellFieldsLatest,
|
||||
Ancestor840 as AncestorLatest,
|
||||
BaseFields840 as BaseFieldsLatest,
|
||||
DetectionAlert840 as DetectionAlertLatest,
|
||||
WrappedFields840 as WrappedFieldsLatest,
|
||||
EqlBuildingBlockFields840 as EqlBuildingBlockFieldsLatest,
|
||||
EqlShellFields840 as EqlShellFieldsLatest,
|
||||
NewTermsFields840 as NewTermsFieldsLatest,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
PositiveInteger,
|
||||
PositiveIntegerGreaterThanZero,
|
||||
UUID,
|
||||
LimitedSizeArray,
|
||||
} from '@kbn/securitysolution-io-ts-types';
|
||||
import * as t from 'io-ts';
|
||||
|
||||
|
|
@ -281,6 +282,13 @@ export const thresholdWithCardinality = t.intersection([
|
|||
]);
|
||||
export type ThresholdWithCardinality = t.TypeOf<typeof thresholdWithCardinality>;
|
||||
|
||||
// New terms rule type currently only supports a single term, but should support more in the future
|
||||
export const newTermsFields = LimitedSizeArray({ codec: t.string, minSize: 1, maxSize: 1 });
|
||||
export type NewTermsFields = t.TypeOf<typeof newTermsFields>;
|
||||
|
||||
export const historyWindowStart = NonEmptyString;
|
||||
export type HistoryWindowStart = t.TypeOf<typeof historyWindowStart>;
|
||||
|
||||
export const created_at = IsoDateString;
|
||||
|
||||
export const updated_at = IsoDateString;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ import type {
|
|||
SavedQueryCreateSchema,
|
||||
ThreatMatchCreateSchema,
|
||||
ThresholdCreateSchema,
|
||||
NewTermsCreateSchema,
|
||||
NewTermsUpdateSchema,
|
||||
} from './rule_schemas';
|
||||
|
||||
export const getCreateRulesSchemaMock = (ruleId = 'rule-1'): QueryCreateSchema => ({
|
||||
|
|
@ -128,6 +130,26 @@ export const getCreateThresholdRulesSchemaMock = (ruleId = 'rule-1'): ThresholdC
|
|||
},
|
||||
});
|
||||
|
||||
export const getCreateNewTermsRulesSchemaMock = (
|
||||
ruleId = 'rule-1',
|
||||
enabled = false
|
||||
): NewTermsCreateSchema => ({
|
||||
description: 'Detecting root and admin users',
|
||||
enabled,
|
||||
index: ['auditbeat-*'],
|
||||
name: 'Query with a rule id',
|
||||
query: '*',
|
||||
severity: 'high',
|
||||
type: 'new_terms',
|
||||
risk_score: 55,
|
||||
language: 'kuery',
|
||||
rule_id: ruleId,
|
||||
interval: '5m',
|
||||
from: 'now-6m',
|
||||
new_terms_fields: ['user.name'],
|
||||
history_window_start: 'now-7d',
|
||||
});
|
||||
|
||||
export const getUpdateRulesSchemaMock = (
|
||||
id = '04128c15-0d1b-4716-a4c5-46997ac7f3bd'
|
||||
): QueryUpdateSchema => ({
|
||||
|
|
@ -153,3 +175,21 @@ export const getUpdateMachineLearningSchemaMock = (
|
|||
anomaly_threshold: 58,
|
||||
machine_learning_job_id: 'typical-ml-job-id',
|
||||
});
|
||||
|
||||
export const getUpdateNewTermsSchemaMock = (
|
||||
id = '04128c15-0d1b-4716-a4c5-46997ac7f3bd'
|
||||
): NewTermsUpdateSchema => ({
|
||||
description: 'Detecting root and admin users',
|
||||
index: ['auditbeat-*'],
|
||||
name: 'Query with a rule id',
|
||||
query: '*',
|
||||
severity: 'high',
|
||||
type: 'new_terms',
|
||||
risk_score: 55,
|
||||
language: 'kuery',
|
||||
id,
|
||||
interval: '5m',
|
||||
from: 'now-6m',
|
||||
new_terms_fields: ['user.name'],
|
||||
history_window_start: 'now-7d',
|
||||
});
|
||||
|
|
|
|||
|
|
@ -74,6 +74,8 @@ import {
|
|||
RelatedIntegrationArray,
|
||||
RequiredFieldArray,
|
||||
SetupGuide,
|
||||
newTermsFields,
|
||||
historyWindowStart,
|
||||
} from '../common';
|
||||
|
||||
export const createSchema = <
|
||||
|
|
@ -358,6 +360,30 @@ const {
|
|||
} = buildAPISchemas(machineLearningRuleParams);
|
||||
|
||||
export { machineLearningCreateParams };
|
||||
|
||||
const newTermsRuleParams = {
|
||||
required: {
|
||||
type: t.literal('new_terms'),
|
||||
query,
|
||||
new_terms_fields: newTermsFields,
|
||||
history_window_start: historyWindowStart,
|
||||
},
|
||||
optional: {
|
||||
index,
|
||||
data_view_id,
|
||||
filters,
|
||||
},
|
||||
defaultable: {
|
||||
language: t.keyof({ kuery: null, lucene: null }),
|
||||
},
|
||||
};
|
||||
const {
|
||||
create: newTermsCreateParams,
|
||||
patch: newTermsPatchParams,
|
||||
response: newTermsResponseParams,
|
||||
} = buildAPISchemas(newTermsRuleParams);
|
||||
|
||||
export { newTermsCreateParams };
|
||||
// ---------------------------------------
|
||||
// END type specific parameter definitions
|
||||
|
||||
|
|
@ -368,6 +394,7 @@ export const createTypeSpecific = t.union([
|
|||
savedQueryCreateParams,
|
||||
thresholdCreateParams,
|
||||
machineLearningCreateParams,
|
||||
newTermsCreateParams,
|
||||
]);
|
||||
export type CreateTypeSpecific = t.TypeOf<typeof createTypeSpecific>;
|
||||
|
||||
|
|
@ -381,6 +408,7 @@ export type ThresholdCreateSchema = CreateSchema<t.TypeOf<typeof thresholdCreate
|
|||
export type MachineLearningCreateSchema = CreateSchema<
|
||||
t.TypeOf<typeof machineLearningCreateParams>
|
||||
>;
|
||||
export type NewTermsCreateSchema = CreateSchema<t.TypeOf<typeof newTermsCreateParams>>;
|
||||
|
||||
export const createRulesSchema = t.intersection([sharedCreateSchema, createTypeSpecific]);
|
||||
export type CreateRulesSchema = t.TypeOf<typeof createRulesSchema>;
|
||||
|
|
@ -396,6 +424,7 @@ export type QueryUpdateSchema = UpdateSchema<t.TypeOf<typeof queryCreateParams>>
|
|||
export type MachineLearningUpdateSchema = UpdateSchema<
|
||||
t.TypeOf<typeof machineLearningCreateParams>
|
||||
>;
|
||||
export type NewTermsUpdateSchema = UpdateSchema<t.TypeOf<typeof newTermsCreateParams>>;
|
||||
|
||||
export const patchTypeSpecific = t.union([
|
||||
eqlPatchParams,
|
||||
|
|
@ -404,6 +433,7 @@ export const patchTypeSpecific = t.union([
|
|||
savedQueryPatchParams,
|
||||
thresholdPatchParams,
|
||||
machineLearningPatchParams,
|
||||
newTermsPatchParams,
|
||||
]);
|
||||
export {
|
||||
eqlPatchParams,
|
||||
|
|
@ -412,6 +442,7 @@ export {
|
|||
savedQueryPatchParams,
|
||||
thresholdPatchParams,
|
||||
machineLearningPatchParams,
|
||||
newTermsPatchParams,
|
||||
};
|
||||
|
||||
export type EqlPatchParams = t.TypeOf<typeof eqlPatchParams>;
|
||||
|
|
@ -420,6 +451,7 @@ export type QueryPatchParams = t.TypeOf<typeof queryPatchParams>;
|
|||
export type SavedQueryPatchParams = t.TypeOf<typeof savedQueryPatchParams>;
|
||||
export type ThresholdPatchParams = t.TypeOf<typeof thresholdPatchParams>;
|
||||
export type MachineLearningPatchParams = t.TypeOf<typeof machineLearningPatchParams>;
|
||||
export type NewTermsPatchParams = t.TypeOf<typeof newTermsPatchParams>;
|
||||
|
||||
const responseTypeSpecific = t.union([
|
||||
eqlResponseParams,
|
||||
|
|
@ -428,6 +460,7 @@ const responseTypeSpecific = t.union([
|
|||
savedQueryResponseParams,
|
||||
thresholdResponseParams,
|
||||
machineLearningResponseParams,
|
||||
newTermsResponseParams,
|
||||
]);
|
||||
export type ResponseTypeSpecific = t.TypeOf<typeof responseTypeSpecific>;
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ export const isQueryRule = (ruleType: Type | undefined): boolean =>
|
|||
export const isThreatMatchRule = (ruleType: Type | undefined): boolean =>
|
||||
ruleType === 'threat_match';
|
||||
export const isMlRule = (ruleType: Type | undefined): boolean => ruleType === 'machine_learning';
|
||||
export const isNewTermsRule = (ruleType: Type | undefined): boolean => ruleType === 'new_terms';
|
||||
|
||||
export const normalizeThresholdField = (
|
||||
thresholdField: string | string[] | null | undefined
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ export const ALERT_GROUP_INDEX = `${ALERT_NAMESPACE}.group.index` as const;
|
|||
export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const;
|
||||
export const ALERT_THRESHOLD_RESULT = `${ALERT_NAMESPACE}.threshold_result` as const;
|
||||
export const ALERT_THRESHOLD_RESULT_COUNT = `${ALERT_THRESHOLD_RESULT}.count` as const;
|
||||
export const ALERT_NEW_TERMS = `${ALERT_NAMESPACE}.new_terms` as const;
|
||||
|
||||
export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const;
|
||||
export const ALERT_ORIGINAL_EVENT_ACTION = `${ALERT_ORIGINAL_EVENT}.action` as const;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,157 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { formatMitreAttackDescription } from '../../helpers/rules';
|
||||
import { getNewTermsRule, getIndexPatterns } from '../../objects/rule';
|
||||
|
||||
import { ALERT_DATA_GRID } from '../../screens/alerts';
|
||||
import {
|
||||
CUSTOM_RULES_BTN,
|
||||
RISK_SCORE,
|
||||
RULE_NAME,
|
||||
RULES_ROW,
|
||||
RULES_TABLE,
|
||||
RULE_SWITCH,
|
||||
SEVERITY,
|
||||
} from '../../screens/alerts_detection_rules';
|
||||
import {
|
||||
ABOUT_DETAILS,
|
||||
ABOUT_INVESTIGATION_NOTES,
|
||||
ABOUT_RULE_DESCRIPTION,
|
||||
ADDITIONAL_LOOK_BACK_DETAILS,
|
||||
CUSTOM_QUERY_DETAILS,
|
||||
DEFINITION_DETAILS,
|
||||
FALSE_POSITIVES_DETAILS,
|
||||
removeExternalLinkText,
|
||||
INDEX_PATTERNS_DETAILS,
|
||||
INVESTIGATION_NOTES_MARKDOWN,
|
||||
INVESTIGATION_NOTES_TOGGLE,
|
||||
MITRE_ATTACK_DETAILS,
|
||||
REFERENCE_URLS_DETAILS,
|
||||
RISK_SCORE_DETAILS,
|
||||
RULE_NAME_HEADER,
|
||||
RULE_TYPE_DETAILS,
|
||||
RUNS_EVERY_DETAILS,
|
||||
SCHEDULE_DETAILS,
|
||||
SEVERITY_DETAILS,
|
||||
TAGS_DETAILS,
|
||||
TIMELINE_TEMPLATE_DETAILS,
|
||||
NEW_TERMS_HISTORY_WINDOW_DETAILS,
|
||||
NEW_TERMS_FIELDS_DETAILS,
|
||||
} from '../../screens/rule_details';
|
||||
|
||||
import { getDetails } from '../../tasks/rule_details';
|
||||
import { goToRuleDetails } from '../../tasks/alerts_detection_rules';
|
||||
import { createTimeline } from '../../tasks/api_calls/timelines';
|
||||
import { cleanKibana, deleteAlertsAndRules } from '../../tasks/common';
|
||||
import {
|
||||
createAndEnableRule,
|
||||
fillAboutRuleAndContinue,
|
||||
fillDefineNewTermsRuleAndContinue,
|
||||
fillScheduleRuleAndContinue,
|
||||
selectNewTermsRuleType,
|
||||
waitForAlertsToPopulate,
|
||||
waitForTheRuleToBeExecuted,
|
||||
} from '../../tasks/create_new_rule';
|
||||
import { login, visit } from '../../tasks/login';
|
||||
|
||||
import { RULE_CREATION } from '../../urls/navigation';
|
||||
|
||||
describe('New Terms rules', () => {
|
||||
before(() => {
|
||||
cleanKibana();
|
||||
login();
|
||||
});
|
||||
describe('Detection rules, New Terms', () => {
|
||||
const expectedUrls = getNewTermsRule().referenceUrls.join('');
|
||||
const expectedFalsePositives = getNewTermsRule().falsePositivesExamples.join('');
|
||||
const expectedTags = getNewTermsRule().tags.join('');
|
||||
const expectedMitre = formatMitreAttackDescription(getNewTermsRule().mitre);
|
||||
const expectedNumberOfRules = 1;
|
||||
|
||||
beforeEach(() => {
|
||||
deleteAlertsAndRules();
|
||||
createTimeline(getNewTermsRule().timeline).then((response) => {
|
||||
cy.wrap({
|
||||
...getNewTermsRule(),
|
||||
timeline: {
|
||||
...getNewTermsRule().timeline,
|
||||
id: response.body.data.persistTimeline.timeline.savedObjectId,
|
||||
},
|
||||
}).as('rule');
|
||||
});
|
||||
});
|
||||
|
||||
it('Creates and enables a new terms rule', function () {
|
||||
visit(RULE_CREATION);
|
||||
selectNewTermsRuleType();
|
||||
fillDefineNewTermsRuleAndContinue(this.rule);
|
||||
fillAboutRuleAndContinue(this.rule);
|
||||
fillScheduleRuleAndContinue(this.rule);
|
||||
createAndEnableRule();
|
||||
|
||||
cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)');
|
||||
|
||||
cy.get(RULES_TABLE).then(($table) => {
|
||||
cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules);
|
||||
});
|
||||
|
||||
cy.get(RULE_NAME).should('have.text', this.rule.name);
|
||||
cy.get(RISK_SCORE).should('have.text', this.rule.riskScore);
|
||||
cy.get(SEVERITY).should('have.text', this.rule.severity);
|
||||
cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true');
|
||||
|
||||
goToRuleDetails();
|
||||
|
||||
cy.get(RULE_NAME_HEADER).should('contain', `${this.rule.name}`);
|
||||
cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', this.rule.description);
|
||||
cy.get(ABOUT_DETAILS).within(() => {
|
||||
getDetails(SEVERITY_DETAILS).should('have.text', this.rule.severity);
|
||||
getDetails(RISK_SCORE_DETAILS).should('have.text', this.rule.riskScore);
|
||||
getDetails(REFERENCE_URLS_DETAILS).should((details) => {
|
||||
expect(removeExternalLinkText(details.text())).equal(expectedUrls);
|
||||
});
|
||||
getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives);
|
||||
getDetails(MITRE_ATTACK_DETAILS).should((mitre) => {
|
||||
expect(removeExternalLinkText(mitre.text())).equal(expectedMitre);
|
||||
});
|
||||
getDetails(TAGS_DETAILS).should('have.text', expectedTags);
|
||||
});
|
||||
cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true });
|
||||
cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN);
|
||||
cy.get(DEFINITION_DETAILS).within(() => {
|
||||
getDetails(INDEX_PATTERNS_DETAILS).should('have.text', getIndexPatterns().join(''));
|
||||
getDetails(CUSTOM_QUERY_DETAILS).should('have.text', this.rule.customQuery);
|
||||
getDetails(RULE_TYPE_DETAILS).should('have.text', 'New Terms');
|
||||
getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None');
|
||||
getDetails(NEW_TERMS_FIELDS_DETAILS).should('have.text', 'host.name');
|
||||
getDetails(NEW_TERMS_HISTORY_WINDOW_DETAILS).should('have.text', '50000h');
|
||||
});
|
||||
cy.get(SCHEDULE_DETAILS).within(() => {
|
||||
getDetails(RUNS_EVERY_DETAILS).should(
|
||||
'have.text',
|
||||
`${this.rule.runsEvery.interval}${this.rule.runsEvery.type}`
|
||||
);
|
||||
getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should(
|
||||
'have.text',
|
||||
`${this.rule.lookBack.interval}${this.rule.lookBack.type}`
|
||||
);
|
||||
});
|
||||
|
||||
waitForTheRuleToBeExecuted();
|
||||
waitForAlertsToPopulate();
|
||||
|
||||
cy.get(ALERT_DATA_GRID)
|
||||
.invoke('text')
|
||||
.then((text) => {
|
||||
expect(text).contains(this.rule.name);
|
||||
expect(text).contains(this.rule.severity.toLowerCase());
|
||||
expect(text).contains(this.rule.riskScore);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -83,6 +83,11 @@ export interface ThreatIndicatorRule extends CustomRule {
|
|||
matchedIndex?: string;
|
||||
}
|
||||
|
||||
export interface NewTermsRule extends CustomRule {
|
||||
newTermsFields: string[];
|
||||
historyWindowSize: Interval;
|
||||
}
|
||||
|
||||
export interface MachineLearningRule {
|
||||
machineLearningJobs: string[];
|
||||
anomalyScoreThreshold: number;
|
||||
|
|
@ -318,6 +323,26 @@ export const getNewThresholdRule = (): ThresholdRule => ({
|
|||
maxSignals: 100,
|
||||
});
|
||||
|
||||
export const getNewTermsRule = (): NewTermsRule => ({
|
||||
customQuery: 'host.name: *',
|
||||
index: getIndexPatterns(),
|
||||
name: 'New Terms Rule',
|
||||
description: 'The new rule description.',
|
||||
severity: 'High',
|
||||
riskScore: '17',
|
||||
tags: ['test', 'newRule'],
|
||||
referenceUrls: ['http://example.com/', 'https://example.com/'],
|
||||
falsePositivesExamples: ['False1', 'False2'],
|
||||
mitre: [getMitre1(), getMitre2()],
|
||||
note: '# test markdown',
|
||||
newTermsFields: ['host.name'],
|
||||
historyWindowSize: getLookBack(),
|
||||
runsEvery: getRunsEvery(),
|
||||
lookBack: getLookBack(),
|
||||
timeline: getTimeline(),
|
||||
maxSignals: 100,
|
||||
});
|
||||
|
||||
export const getMachineLearningRule = (): MachineLearningRule => ({
|
||||
machineLearningJobs: [
|
||||
'v3_linux_anomalous_process_all_hosts',
|
||||
|
|
|
|||
|
|
@ -218,8 +218,18 @@ export const TAGS_INPUT =
|
|||
export const TAGS_CLEAR_BUTTON =
|
||||
'[data-test-subj="detectionEngineStepAboutRuleTags"] [data-test-subj="comboBoxClearButton"]';
|
||||
|
||||
export const THRESHOLD_FIELD_SELECTION = '.euiFilterSelectItem';
|
||||
export const EUI_FILTER_SELECT_ITEM = '.euiFilterSelectItem';
|
||||
|
||||
export const THRESHOLD_INPUT_AREA = '[data-test-subj="thresholdInput"]';
|
||||
|
||||
export const THRESHOLD_TYPE = '[data-test-subj="thresholdRuleType"]';
|
||||
|
||||
export const NEW_TERMS_TYPE = '[data-test-subj="newTermsRuleType"]';
|
||||
|
||||
export const NEW_TERMS_INPUT_AREA = '[data-test-subj="newTermsInput"]';
|
||||
|
||||
export const NEW_TERMS_HISTORY_SIZE =
|
||||
'[data-test-subj="detectionEngineStepDefineRuleHistoryWindowSize"] [data-test-subj="interval"]';
|
||||
|
||||
export const NEW_TERMS_HISTORY_TIME_TYPE =
|
||||
'[data-test-subj="detectionEngineStepDefineRuleHistoryWindowSize"] [data-test-subj="timeType"]';
|
||||
|
|
|
|||
|
|
@ -55,6 +55,10 @@ export const MACHINE_LEARNING_JOB_STATUS = '[data-test-subj="machineLearningJobS
|
|||
|
||||
export const MITRE_ATTACK_DETAILS = 'MITRE ATT&CK';
|
||||
|
||||
export const NEW_TERMS_FIELDS_DETAILS = 'Fields';
|
||||
|
||||
export const NEW_TERMS_HISTORY_WINDOW_DETAILS = 'History Window Size';
|
||||
|
||||
export const FIELDS_BROWSER_BTN =
|
||||
'[data-test-subj="events-viewer-panel"] [data-test-subj="show-field-browser"]';
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import type {
|
|||
OverrideRule,
|
||||
ThreatIndicatorRule,
|
||||
ThresholdRule,
|
||||
NewTermsRule,
|
||||
} from '../objects/rule';
|
||||
import { getMachineLearningRule } from '../objects/rule';
|
||||
import {
|
||||
|
|
@ -82,7 +83,7 @@ import {
|
|||
THREAT_MATCH_INDICATOR_INDICATOR_INDEX,
|
||||
THREAT_MATCH_OR_BUTTON,
|
||||
THREAT_MATCH_QUERY_INPUT,
|
||||
THRESHOLD_FIELD_SELECTION,
|
||||
EUI_FILTER_SELECT_ITEM,
|
||||
THRESHOLD_INPUT_AREA,
|
||||
THRESHOLD_TYPE,
|
||||
CONNECTOR_NAME_INPUT,
|
||||
|
|
@ -93,6 +94,10 @@ import {
|
|||
EMAIL_CONNECTOR_PASSWORD_INPUT,
|
||||
EMAIL_CONNECTOR_SERVICE_SELECTOR,
|
||||
PREVIEW_HISTOGRAM,
|
||||
NEW_TERMS_TYPE,
|
||||
NEW_TERMS_HISTORY_SIZE,
|
||||
NEW_TERMS_HISTORY_TIME_TYPE,
|
||||
NEW_TERMS_INPUT_AREA,
|
||||
} from '../screens/create_new_rule';
|
||||
import { TOAST_ERROR } from '../screens/shared';
|
||||
import { SERVER_SIDE_EVENT_COUNT } from '../screens/timeline';
|
||||
|
|
@ -286,7 +291,7 @@ export const fillDefineThresholdRule = (rule: ThresholdRule) => {
|
|||
.find(INPUT)
|
||||
.then((inputs) => {
|
||||
cy.wrap(inputs[thresholdField]).type(rule.thresholdField);
|
||||
cy.get(THRESHOLD_FIELD_SELECTION).click({ force: true });
|
||||
cy.get(EUI_FILTER_SELECT_ITEM).click({ force: true });
|
||||
cy.wrap(inputs[threshold]).clear().type(rule.threshold);
|
||||
});
|
||||
};
|
||||
|
|
@ -306,7 +311,7 @@ export const fillDefineThresholdRuleAndContinue = (rule: ThresholdRule) => {
|
|||
.then((inputs) => {
|
||||
cy.wrap(inputs[thresholdField]).click();
|
||||
cy.wrap(inputs[thresholdField]).pipe(typeThresholdField);
|
||||
cy.get(THRESHOLD_FIELD_SELECTION).click({ force: true });
|
||||
cy.get(EUI_FILTER_SELECT_ITEM).click({ force: true });
|
||||
cy.wrap(inputs[threshold]).clear().type(rule.threshold);
|
||||
});
|
||||
cy.get(DEFINE_CONTINUE_BUTTON).should('exist').click({ force: true });
|
||||
|
|
@ -341,6 +346,24 @@ export const fillDefineEqlRuleAndContinue = (rule: CustomRule) => {
|
|||
cy.get(`${RULES_CREATION_FORM} ${EQL_QUERY_INPUT}`).should('not.exist');
|
||||
};
|
||||
|
||||
export const fillDefineNewTermsRuleAndContinue = (rule: NewTermsRule) => {
|
||||
cy.get(IMPORT_QUERY_FROM_SAVED_TIMELINE_LINK).click();
|
||||
cy.get(TIMELINE(rule.timeline.id)).click();
|
||||
cy.get(CUSTOM_QUERY_INPUT).should('have.value', rule.customQuery);
|
||||
cy.get(NEW_TERMS_INPUT_AREA).find(INPUT).click().type(rule.newTermsFields[0], { delay: 35 });
|
||||
cy.get(EUI_FILTER_SELECT_ITEM).click({ force: true });
|
||||
cy.get(NEW_TERMS_INPUT_AREA)
|
||||
.find(NEW_TERMS_HISTORY_SIZE)
|
||||
.type('{selectAll}')
|
||||
.type(rule.historyWindowSize.interval);
|
||||
cy.get(NEW_TERMS_INPUT_AREA)
|
||||
.find(NEW_TERMS_HISTORY_TIME_TYPE)
|
||||
.select(rule.historyWindowSize.timeType);
|
||||
cy.get(DEFINE_CONTINUE_BUTTON).should('exist').click({ force: true });
|
||||
|
||||
cy.get(CUSTOM_QUERY_INPUT).should('not.exist');
|
||||
};
|
||||
|
||||
/**
|
||||
* Fills in the indicator match rows for tests by giving it an optional rowNumber,
|
||||
* a indexField, a indicatorIndexField, and an optional validRows which indicates
|
||||
|
|
@ -525,6 +548,10 @@ export const selectThresholdRuleType = () => {
|
|||
cy.get(THRESHOLD_TYPE).click({ force: true });
|
||||
};
|
||||
|
||||
export const selectNewTermsRuleType = () => {
|
||||
cy.get(NEW_TERMS_TYPE).click({ force: true });
|
||||
};
|
||||
|
||||
export const previewResults = () => {
|
||||
cy.get(QUERY_PREVIEW_BUTTON).click();
|
||||
};
|
||||
|
|
|
|||
|
|
@ -685,6 +685,40 @@ describe('AlertSummaryView', () => {
|
|||
});
|
||||
});
|
||||
|
||||
test('New terms events have special fields', () => {
|
||||
const enhancedData = [
|
||||
...mockAlertDetailsData.map((item) => {
|
||||
if (item.category === 'kibana' && item.field === 'kibana.alert.rule.type') {
|
||||
return {
|
||||
...item,
|
||||
values: ['new_terms'],
|
||||
originalValue: ['new_terms'],
|
||||
};
|
||||
}
|
||||
return item;
|
||||
}),
|
||||
{
|
||||
category: 'kibana',
|
||||
field: 'kibana.alert.new_terms',
|
||||
values: ['127.0.0.1'],
|
||||
originalValue: ['127.0.0.1'],
|
||||
},
|
||||
] as TimelineEventsDetailsItem[];
|
||||
const renderProps = {
|
||||
...props,
|
||||
data: enhancedData,
|
||||
};
|
||||
const { getByText } = render(
|
||||
<TestProvidersComponent>
|
||||
<AlertSummaryView {...renderProps} />
|
||||
</TestProvidersComponent>
|
||||
);
|
||||
|
||||
['New Terms'].forEach((fieldId) => {
|
||||
expect(getByText(fieldId));
|
||||
});
|
||||
});
|
||||
|
||||
test("doesn't render empty fields", async () => {
|
||||
const renderProps = {
|
||||
...props,
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@ import {
|
|||
ALERTS_HEADERS_THRESHOLD_COUNT,
|
||||
ALERTS_HEADERS_THRESHOLD_TERMS,
|
||||
ALERTS_HEADERS_RULE_DESCRIPTION,
|
||||
ALERTS_HEADERS_NEW_TERMS,
|
||||
} from '../../../detections/components/alerts_table/translations';
|
||||
import { ALERT_THRESHOLD_RESULT } from '../../../../common/field_maps/field_names';
|
||||
import { ALERT_NEW_TERMS, ALERT_THRESHOLD_RESULT } from '../../../../common/field_maps/field_names';
|
||||
import { AGENT_STATUS_FIELD_NAME } from '../../../timelines/components/timeline/body/renderers/constants';
|
||||
import type { AlertSummaryRow } from './helpers';
|
||||
import { getEnrichedFieldInfo } from './helpers';
|
||||
|
|
@ -168,6 +169,13 @@ function getFieldsByRuleType(ruleType?: string): EventSummaryField[] {
|
|||
legacyId: 'signal.rule.threat_query',
|
||||
},
|
||||
];
|
||||
case 'new_terms':
|
||||
return [
|
||||
{
|
||||
id: ALERT_NEW_TERMS,
|
||||
label: ALERTS_HEADERS_NEW_TERMS,
|
||||
},
|
||||
];
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,12 +27,15 @@ import {
|
|||
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
|
||||
import { buildExceptionFilter } from '@kbn/securitysolution-list-utils';
|
||||
|
||||
import type { TGridModel } from '@kbn/timelines-plugin/public';
|
||||
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
import {
|
||||
ALERT_ORIGINAL_TIME,
|
||||
ALERT_GROUP_ID,
|
||||
ALERT_RULE_TIMELINE_ID,
|
||||
ALERT_THRESHOLD_RESULT,
|
||||
ALERT_NEW_TERMS,
|
||||
} from '../../../../common/field_maps/field_names';
|
||||
import type { TimelineResult } from '../../../../common/types/timeline';
|
||||
import { TimelineId, TimelineStatus, TimelineType } from '../../../../common/types/timeline';
|
||||
|
|
@ -272,6 +275,14 @@ export const isThresholdAlert = (ecsData: Ecs): boolean => {
|
|||
);
|
||||
};
|
||||
|
||||
export const isNewTermsAlert = (ecsData: Ecs): boolean => {
|
||||
const ruleType = getField(ecsData, ALERT_RULE_TYPE);
|
||||
return (
|
||||
ruleType === 'new_terms' ||
|
||||
(Array.isArray(ruleType) && ruleType.length > 0 && ruleType[0] === 'new_terms')
|
||||
);
|
||||
};
|
||||
|
||||
export const buildAlertsKqlFilter = (
|
||||
key: '_id' | 'signal.group.id' | 'kibana.alert.group.id',
|
||||
alertIds: string[]
|
||||
|
|
@ -506,6 +517,155 @@ const createThresholdTimeline = async (
|
|||
}
|
||||
};
|
||||
|
||||
const getNewTermsData = (ecsData: Ecs | Ecs[]) => {
|
||||
const normalizedEcsData: Ecs = Array.isArray(ecsData) ? ecsData[0] : ecsData;
|
||||
const originalTimeValue = getField(normalizedEcsData, ALERT_ORIGINAL_TIME);
|
||||
const newTermsField = getField(normalizedEcsData, `${ALERT_RULE_PARAMETERS}.new_terms_fields`)[0];
|
||||
const newTermsValue = getField(normalizedEcsData, ALERT_NEW_TERMS)[0];
|
||||
const newTermsFieldId = newTermsField.replace('.', '-');
|
||||
const dataProviderPartial = {
|
||||
id: `send-alert-to-timeline-action-default-draggable-event-details-value-formatted-field-value-${TimelineId.active}-${newTermsFieldId}-${newTermsValue}`,
|
||||
name: newTermsField,
|
||||
enabled: true,
|
||||
excluded: false,
|
||||
kqlQuery: '',
|
||||
queryMatch: {
|
||||
field: newTermsField,
|
||||
value: newTermsValue,
|
||||
operator: ':' as const,
|
||||
},
|
||||
and: [],
|
||||
};
|
||||
return {
|
||||
from: originalTimeValue,
|
||||
to: moment().toISOString(),
|
||||
dataProviders: [dataProviderPartial],
|
||||
};
|
||||
};
|
||||
|
||||
const createNewTermsTimeline = async (
|
||||
ecsData: Ecs,
|
||||
createTimeline: ({ from, timeline, to }: CreateTimelineProps) => void,
|
||||
noteContent: string,
|
||||
templateValues: {
|
||||
filters?: Filter[];
|
||||
query?: string;
|
||||
dataProviders?: DataProvider[];
|
||||
columns?: TGridModel['columns'];
|
||||
},
|
||||
getExceptions: (ecs: Ecs) => Promise<ExceptionListItemSchema[]>
|
||||
) => {
|
||||
try {
|
||||
const alertResponse = await KibanaServices.get().http.fetch<
|
||||
estypes.SearchResponse<{ '@timestamp': string; [key: string]: unknown }>
|
||||
>(DETECTION_ENGINE_QUERY_SIGNALS_URL, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(buildAlertsQuery([ecsData._id])),
|
||||
});
|
||||
const formattedAlertData =
|
||||
alertResponse?.hits.hits.reduce<Ecs[]>((acc, { _id, _index, _source = {} }) => {
|
||||
return [
|
||||
...acc,
|
||||
{
|
||||
...formatAlertToEcsSignal(_source),
|
||||
_id,
|
||||
_index,
|
||||
timestamp: _source['@timestamp'],
|
||||
},
|
||||
];
|
||||
}, []) ?? [];
|
||||
|
||||
const alertDoc = formattedAlertData[0];
|
||||
const params = getField(alertDoc, ALERT_RULE_PARAMETERS);
|
||||
const filters: Filter[] =
|
||||
(params as MightHaveFilters).filters ??
|
||||
(alertDoc.signal?.rule as MightHaveFilters)?.filters ??
|
||||
[];
|
||||
// https://github.com/elastic/kibana/issues/126574 - if the provided filter has no `meta` field
|
||||
// we expect an empty object to be inserted before calling `createTimeline`
|
||||
const augmentedFilters = filters.map((filter) => {
|
||||
return filter.meta != null ? filter : { ...filter, meta: {} };
|
||||
});
|
||||
const language = params.language ?? alertDoc.signal?.rule?.language ?? 'kuery';
|
||||
const query = params.query ?? alertDoc.signal?.rule?.query ?? '';
|
||||
const indexNames = params.index ?? alertDoc.signal?.rule?.index ?? [];
|
||||
|
||||
const { from, to, dataProviders } = getNewTermsData(alertDoc);
|
||||
const exceptions = await getExceptions(ecsData);
|
||||
const exceptionsFilter =
|
||||
buildExceptionFilter({
|
||||
lists: exceptions,
|
||||
excludeExceptions: true,
|
||||
chunkSize: 10000,
|
||||
alias: 'Exceptions',
|
||||
}) ?? [];
|
||||
const allFilters = (templateValues.filters ?? augmentedFilters).concat(exceptionsFilter);
|
||||
return createTimeline({
|
||||
from,
|
||||
notes: null,
|
||||
timeline: {
|
||||
...timelineDefaults,
|
||||
columns: templateValues.columns ?? timelineDefaults.columns,
|
||||
description: `_id: ${alertDoc._id}`,
|
||||
filters: allFilters,
|
||||
dataProviders: templateValues.dataProviders ?? dataProviders,
|
||||
id: TimelineId.active,
|
||||
indexNames,
|
||||
dateRange: {
|
||||
start: from,
|
||||
end: to,
|
||||
},
|
||||
eventType: 'all',
|
||||
kqlQuery: {
|
||||
filterQuery: {
|
||||
kuery: {
|
||||
kind: language,
|
||||
expression: templateValues.query ?? query,
|
||||
},
|
||||
serializedQuery: templateValues.query ?? query,
|
||||
},
|
||||
},
|
||||
},
|
||||
to,
|
||||
ruleNote: noteContent,
|
||||
});
|
||||
} catch (error) {
|
||||
const { toasts } = KibanaServices.get().notifications;
|
||||
toasts.addError(error, {
|
||||
toastMessage: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.alerts.createNewTermsTimelineFailure',
|
||||
{
|
||||
defaultMessage: 'Failed to create timeline for document _id: {id}',
|
||||
values: { id: ecsData._id },
|
||||
}
|
||||
),
|
||||
title: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.alerts.createNewTermsTimelineFailureTitle',
|
||||
{
|
||||
defaultMessage: 'Failed to create new terms alert timeline',
|
||||
}
|
||||
),
|
||||
});
|
||||
const from = DEFAULT_FROM_MOMENT.toISOString();
|
||||
const to = DEFAULT_TO_MOMENT.toISOString();
|
||||
return createTimeline({
|
||||
from,
|
||||
notes: null,
|
||||
timeline: {
|
||||
...timelineDefaults,
|
||||
id: TimelineId.active,
|
||||
indexNames: [],
|
||||
dateRange: {
|
||||
start: from,
|
||||
end: to,
|
||||
},
|
||||
eventType: 'all',
|
||||
},
|
||||
to,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const sendAlertToTimelineAction = async ({
|
||||
createTimeline,
|
||||
ecsData: ecs,
|
||||
|
|
@ -585,6 +745,19 @@ export const sendAlertToTimelineAction = async ({
|
|||
},
|
||||
getExceptions
|
||||
);
|
||||
} else if (isNewTermsAlert(ecsData)) {
|
||||
return createNewTermsTimeline(
|
||||
ecsData,
|
||||
createTimeline,
|
||||
noteContent,
|
||||
{
|
||||
filters,
|
||||
query,
|
||||
dataProviders,
|
||||
columns: timeline.columns,
|
||||
},
|
||||
getExceptions
|
||||
);
|
||||
} else {
|
||||
return createTimeline({
|
||||
from,
|
||||
|
|
@ -639,6 +812,8 @@ export const sendAlertToTimelineAction = async ({
|
|||
}
|
||||
} else if (isThresholdAlert(ecsData)) {
|
||||
return createThresholdTimeline(ecsData, createTimeline, noteContent, {}, getExceptions);
|
||||
} else if (isNewTermsAlert(ecsData)) {
|
||||
return createNewTermsTimeline(ecsData, createTimeline, noteContent, {}, getExceptions);
|
||||
} else {
|
||||
let { dataProviders, filters } = buildTimelineDataProviderOrFilter(alertIds ?? [], ecsData._id);
|
||||
if (isEqlAlertWithGroupId(ecsData)) {
|
||||
|
|
|
|||
|
|
@ -116,6 +116,13 @@ export const ALERTS_HEADERS_THRESHOLD_CARDINALITY = i18n.translate(
|
|||
}
|
||||
);
|
||||
|
||||
export const ALERTS_HEADERS_NEW_TERMS = i18n.translate(
|
||||
'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTerms',
|
||||
{
|
||||
defaultMessage: 'New Terms',
|
||||
}
|
||||
);
|
||||
|
||||
export const ACTION_INVESTIGATE_IN_TIMELINE = i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.alerts.actions.investigateInTimelineTitle',
|
||||
{
|
||||
|
|
|
|||
|
|
@ -494,6 +494,14 @@ export const buildRuleTypeDescription = (label: string, ruleType: Type): ListIte
|
|||
},
|
||||
];
|
||||
}
|
||||
case 'new_terms': {
|
||||
return [
|
||||
{
|
||||
title: label,
|
||||
description: i18n.NEW_TERMS_TYPE_DESCRIPTION,
|
||||
},
|
||||
];
|
||||
}
|
||||
default:
|
||||
return assertUnreachable(ruleType);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,6 +70,13 @@ export const THREAT_MATCH_TYPE_DESCRIPTION = i18n.translate(
|
|||
}
|
||||
);
|
||||
|
||||
export const NEW_TERMS_TYPE_DESCRIPTION = i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.newTermsRuleTypeDescription',
|
||||
{
|
||||
defaultMessage: 'New Terms',
|
||||
}
|
||||
);
|
||||
|
||||
export const ML_JOB_STARTED = i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.ruleDescription.mlJobStartedDescription',
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import React, { useMemo } from 'react';
|
||||
|
||||
import type { FieldHook } from '../../../../shared_imports';
|
||||
import { Field } from '../../../../shared_imports';
|
||||
import type { BrowserFields } from '../../../../common/containers/source';
|
||||
import { getCategorizedFieldNames } from '../../../../timelines/components/edit_data_provider/helpers';
|
||||
import { NEW_TERMS_FIELD_PLACEHOLDER } from './translations';
|
||||
|
||||
interface NewTermsFieldsProps {
|
||||
browserFields: BrowserFields;
|
||||
field: FieldHook;
|
||||
}
|
||||
|
||||
const FIELD_COMBO_BOX_WIDTH = 410;
|
||||
|
||||
const fieldDescribedByIds = 'detectionEngineStepDefineRuleNewTermsField';
|
||||
|
||||
export const NewTermsFieldsComponent: React.FC<NewTermsFieldsProps> = ({
|
||||
browserFields,
|
||||
field,
|
||||
}: NewTermsFieldsProps) => {
|
||||
const fieldEuiFieldProps = useMemo(
|
||||
() => ({
|
||||
fullWidth: true,
|
||||
noSuggestions: false,
|
||||
options: getCategorizedFieldNames(browserFields),
|
||||
placeholder: NEW_TERMS_FIELD_PLACEHOLDER,
|
||||
onCreateOption: undefined,
|
||||
style: { width: `${FIELD_COMBO_BOX_WIDTH}px` },
|
||||
}),
|
||||
[browserFields]
|
||||
);
|
||||
return <Field field={field} idAria={fieldDescribedByIds} euiFieldProps={fieldEuiFieldProps} />;
|
||||
};
|
||||
|
||||
export const NewTermsFields = React.memo(NewTermsFieldsComponent);
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
|
||||
export const NEW_TERMS_FIELD_PLACEHOLDER = i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.newTermsField.placeholderText',
|
||||
{
|
||||
defaultMessage: 'Select a field',
|
||||
}
|
||||
);
|
||||
|
|
@ -59,6 +59,8 @@ const defaultProps: RulePreviewProps = {
|
|||
anomalyThreshold: 50,
|
||||
machineLearningJobId: ['test-ml-job-id'],
|
||||
eqlOptions: {},
|
||||
newTermsFields: ['host.ip'],
|
||||
historyWindowSize: '7d',
|
||||
};
|
||||
|
||||
describe('PreviewQuery', () => {
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ export interface RulePreviewProps {
|
|||
machineLearningJobId: string[];
|
||||
anomalyThreshold: number;
|
||||
eqlOptions: EqlOptionsSelected;
|
||||
newTermsFields: string[];
|
||||
historyWindowSize: string;
|
||||
}
|
||||
|
||||
const Select = styled(EuiSelect)`
|
||||
|
|
@ -77,6 +79,8 @@ const RulePreviewComponent: React.FC<RulePreviewProps> = ({
|
|||
machineLearningJobId,
|
||||
anomalyThreshold,
|
||||
eqlOptions,
|
||||
newTermsFields,
|
||||
historyWindowSize,
|
||||
}) => {
|
||||
const { spaces } = useKibana().services;
|
||||
const { loading: isMlLoading, jobs } = useSecurityJobs(false);
|
||||
|
|
@ -121,6 +125,8 @@ const RulePreviewComponent: React.FC<RulePreviewProps> = ({
|
|||
machineLearningJobId,
|
||||
anomalyThreshold,
|
||||
eqlOptions,
|
||||
newTermsFields,
|
||||
historyWindowSize,
|
||||
});
|
||||
|
||||
// Resets the timeFrame to default when rule type is changed because not all time frames are supported by all rule types
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ interface PreviewRouteParams {
|
|||
machineLearningJobId: string[];
|
||||
anomalyThreshold: number;
|
||||
eqlOptions: EqlOptionsSelected;
|
||||
newTermsFields: string[];
|
||||
historyWindowSize: string;
|
||||
}
|
||||
|
||||
export const usePreviewRoute = ({
|
||||
|
|
@ -45,6 +47,8 @@ export const usePreviewRoute = ({
|
|||
machineLearningJobId,
|
||||
anomalyThreshold,
|
||||
eqlOptions,
|
||||
newTermsFields,
|
||||
historyWindowSize,
|
||||
}: PreviewRouteParams) => {
|
||||
const [isRequestTriggered, setIsRequestTriggered] = useState(false);
|
||||
|
||||
|
|
@ -86,6 +90,8 @@ export const usePreviewRoute = ({
|
|||
machineLearningJobId,
|
||||
anomalyThreshold,
|
||||
eqlOptions,
|
||||
newTermsFields,
|
||||
historyWindowSize,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -104,6 +110,8 @@ export const usePreviewRoute = ({
|
|||
machineLearningJobId,
|
||||
anomalyThreshold,
|
||||
eqlOptions,
|
||||
newTermsFields,
|
||||
historyWindowSize,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
|
@ -123,6 +131,8 @@ export const usePreviewRoute = ({
|
|||
machineLearningJobId,
|
||||
anomalyThreshold,
|
||||
eqlOptions,
|
||||
newTermsFields,
|
||||
historyWindowSize,
|
||||
]);
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -28,12 +28,14 @@ interface ScheduleItemProps {
|
|||
idAria: string;
|
||||
isDisabled: boolean;
|
||||
minimumValue?: number;
|
||||
timeTypes?: string[];
|
||||
}
|
||||
|
||||
const timeTypeOptions = [
|
||||
{ value: 's', text: I18n.SECONDS },
|
||||
{ value: 'm', text: I18n.MINUTES },
|
||||
{ value: 'h', text: I18n.HOURS },
|
||||
{ value: 'd', text: I18n.DAYS },
|
||||
];
|
||||
|
||||
// move optional label to the end of input
|
||||
|
|
@ -79,8 +81,9 @@ export const ScheduleItem = ({
|
|||
idAria,
|
||||
isDisabled,
|
||||
minimumValue = 0,
|
||||
timeTypes = ['s', 'm', 'h'],
|
||||
}: ScheduleItemProps) => {
|
||||
const [timeType, setTimeType] = useState('s');
|
||||
const [timeType, setTimeType] = useState(timeTypes[0]);
|
||||
const [timeVal, setTimeVal] = useState<number>(0);
|
||||
const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(field);
|
||||
const { value, setValue } = field;
|
||||
|
|
@ -117,13 +120,13 @@ export const ScheduleItem = ({
|
|||
if (
|
||||
!isEmpty(filterTimeType) &&
|
||||
filterTimeType != null &&
|
||||
['s', 'm', 'h'].includes(filterTimeType[0]) &&
|
||||
timeTypes.includes(filterTimeType[0]) &&
|
||||
filterTimeType[0] !== timeType
|
||||
) {
|
||||
setTimeType(filterTimeType[0]);
|
||||
}
|
||||
}
|
||||
}, [timeType, timeVal, value]);
|
||||
}, [timeType, timeTypes, timeVal, value]);
|
||||
|
||||
// EUI missing some props
|
||||
const rest = { disabled: isDisabled };
|
||||
|
|
@ -155,7 +158,7 @@ export const ScheduleItem = ({
|
|||
append={
|
||||
<MyEuiSelect
|
||||
fullWidth={false}
|
||||
options={timeTypeOptions}
|
||||
options={timeTypeOptions.filter((type) => timeTypes.includes(type.value))}
|
||||
onChange={onChangeTimeType}
|
||||
value={timeType}
|
||||
data-test-subj="timeType"
|
||||
|
|
|
|||
|
|
@ -27,3 +27,10 @@ export const HOURS = i18n.translate(
|
|||
defaultMessage: 'Hours',
|
||||
}
|
||||
);
|
||||
|
||||
export const DAYS = i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.stepScheduleRuleForm.daysOptionDescription',
|
||||
{
|
||||
defaultMessage: 'Days',
|
||||
}
|
||||
);
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import {
|
|||
isEqlRule,
|
||||
isQueryRule,
|
||||
isThreatMatchRule,
|
||||
isNewTermsRule,
|
||||
} from '../../../../../common/detection_engine/utils';
|
||||
import type { FieldHook } from '../../../../shared_imports';
|
||||
import { useKibana } from '../../../../common/lib/kibana';
|
||||
|
|
@ -48,6 +49,7 @@ export const SelectRuleType: React.FC<SelectRuleTypeProps> = ({
|
|||
const setQuery = useCallback(() => setType('query'), [setType]);
|
||||
const setThreshold = useCallback(() => setType('threshold'), [setType]);
|
||||
const setThreatMatch = useCallback(() => setType('threat_match'), [setType]);
|
||||
const setNewTerms = useCallback(() => setType('new_terms'), [setType]);
|
||||
const licensingUrl = useKibana().services.application.getUrlForApp('kibana', {
|
||||
path: '#/management/stack/license_management',
|
||||
});
|
||||
|
|
@ -93,6 +95,14 @@ export const SelectRuleType: React.FC<SelectRuleTypeProps> = ({
|
|||
[ruleType, setThreatMatch]
|
||||
);
|
||||
|
||||
const newTermsSelectableConfig = useMemo(
|
||||
() => ({
|
||||
onClick: setNewTerms,
|
||||
isSelected: isNewTermsRule(ruleType),
|
||||
}),
|
||||
[ruleType, setNewTerms]
|
||||
);
|
||||
|
||||
return (
|
||||
<EuiFormRow
|
||||
fullWidth
|
||||
|
|
@ -172,6 +182,19 @@ export const SelectRuleType: React.FC<SelectRuleTypeProps> = ({
|
|||
/>
|
||||
</EuiFlexItem>
|
||||
)}
|
||||
{(!isUpdateView || newTermsSelectableConfig.isSelected) && (
|
||||
<EuiFlexItem>
|
||||
<EuiCard
|
||||
data-test-subj="newTermsRuleType"
|
||||
title={i18n.NEW_TERMS_TYPE_TITLE}
|
||||
titleSize="xs"
|
||||
description={i18n.NEW_TERMS_TYPE_DESCRIPTION}
|
||||
icon={<EuiIcon size="l" type="magnifyWithPlus" />}
|
||||
selectable={newTermsSelectableConfig}
|
||||
layout="horizontal"
|
||||
/>
|
||||
</EuiFlexItem>
|
||||
)}
|
||||
</EuiFlexGrid>
|
||||
</EuiFormRow>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -78,3 +78,17 @@ export const THREAT_MATCH_TYPE_DESCRIPTION = i18n.translate(
|
|||
'Use indicators from intelligence sources to detect matching events and alerts.',
|
||||
}
|
||||
);
|
||||
|
||||
export const NEW_TERMS_TYPE_TITLE = i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.ruleTypeField.newTermsTitle',
|
||||
{
|
||||
defaultMessage: 'New Terms',
|
||||
}
|
||||
);
|
||||
|
||||
export const NEW_TERMS_TYPE_DESCRIPTION = i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.ruleTypeField.newTermsDescription',
|
||||
{
|
||||
defaultMessage: 'Find documents with values appearing for the first time.',
|
||||
}
|
||||
);
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ export const stepDefineStepMLRule: DefineStepRule = {
|
|||
threatMapping: [],
|
||||
timeline: { id: null, title: null },
|
||||
eqlOptions: {},
|
||||
newTermsFields: ['host.ip'],
|
||||
historyWindowSize: '7d',
|
||||
};
|
||||
|
||||
describe('StepAboutRuleComponent', () => {
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ import { schema } from './schema';
|
|||
import * as i18n from './translations';
|
||||
import {
|
||||
isEqlRule,
|
||||
isNewTermsRule,
|
||||
isThreatMatchRule,
|
||||
isThresholdRule,
|
||||
} from '../../../../../common/detection_engine/utils';
|
||||
|
|
@ -73,6 +74,8 @@ import type { BrowserField, BrowserFields } from '../../../../common/containers/
|
|||
import { useFetchIndex } from '../../../../common/containers/source';
|
||||
import { RulePreview } from '../rule_preview';
|
||||
import { getIsRulePreviewDisabled } from '../rule_preview/helpers';
|
||||
import { NewTermsFields } from '../new_terms_fields';
|
||||
import { ScheduleItem } from '../schedule_item_form';
|
||||
import { DocLink } from '../../../../common/components/links_to_docs/doc_link';
|
||||
|
||||
const DATA_VIEW_SELECT_ID = 'dataView';
|
||||
|
|
@ -127,6 +130,8 @@ export const stepDefineDefaultValue: DefineStepRule = {
|
|||
title: DEFAULT_TIMELINE_TITLE,
|
||||
},
|
||||
eqlOptions: {},
|
||||
newTermsFields: [],
|
||||
historyWindowSize: '7d',
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -208,6 +213,8 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
|
|||
threatMapping: formThreatMapping,
|
||||
machineLearningJobId: formMachineLearningJobId,
|
||||
anomalyThreshold: formAnomalyThreshold,
|
||||
newTermsFields: formNewTermsFields,
|
||||
historyWindowSize: formHistoryWindowSize,
|
||||
},
|
||||
] = useFormData<DefineStepRule>({
|
||||
form,
|
||||
|
|
@ -225,6 +232,8 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
|
|||
'threatMapping',
|
||||
'machineLearningJobId',
|
||||
'anomalyThreshold',
|
||||
'newTermsFields',
|
||||
'historyWindowSize',
|
||||
],
|
||||
});
|
||||
|
||||
|
|
@ -235,6 +244,8 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
|
|||
const threatIndex = formThreatIndex || initialState.threatIndex;
|
||||
const machineLearningJobId = formMachineLearningJobId ?? initialState.machineLearningJobId;
|
||||
const anomalyThreshold = formAnomalyThreshold ?? initialState.anomalyThreshold;
|
||||
const newTermsFields = formNewTermsFields ?? initialState.newTermsFields;
|
||||
const historyWindowSize = formHistoryWindowSize ?? initialState.historyWindowSize;
|
||||
const ruleType = formRuleType || initialState.ruleType;
|
||||
|
||||
// if 'index' is selected, use these browser fields
|
||||
|
|
@ -738,6 +749,30 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
|
|||
</UseMultiFields>
|
||||
</>
|
||||
</RuleTypeEuiFormRow>
|
||||
<RuleTypeEuiFormRow
|
||||
$isVisible={isNewTermsRule(ruleType)}
|
||||
data-test-subj="newTermsInput"
|
||||
fullWidth
|
||||
>
|
||||
<>
|
||||
<UseField
|
||||
path="newTermsFields"
|
||||
component={NewTermsFields}
|
||||
componentProps={{
|
||||
browserFields,
|
||||
}}
|
||||
/>
|
||||
<UseField
|
||||
path="historyWindowSize"
|
||||
component={ScheduleItem}
|
||||
componentProps={{
|
||||
idAria: 'detectionEngineStepDefineRuleHistoryWindowSize',
|
||||
dataTestSubj: 'detectionEngineStepDefineRuleHistoryWindowSize',
|
||||
timeTypes: ['m', 'h', 'd'],
|
||||
}}
|
||||
/>
|
||||
</>
|
||||
</RuleTypeEuiFormRow>
|
||||
<UseField
|
||||
path="timeline"
|
||||
component={PickTimeline}
|
||||
|
|
@ -772,6 +807,8 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
|
|||
machineLearningJobId={machineLearningJobId}
|
||||
anomalyThreshold={anomalyThreshold}
|
||||
eqlOptions={optionsSelected}
|
||||
newTermsFields={newTermsFields}
|
||||
historyWindowSize={historyWindowSize}
|
||||
/>
|
||||
</StepContentWrapper>
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import {
|
|||
} from '../../../../common/components/threat_match/helpers';
|
||||
import {
|
||||
isEqlRule,
|
||||
isNewTermsRule,
|
||||
isThreatMatchRule,
|
||||
isThresholdRule,
|
||||
} from '../../../../../common/detection_engine/utils';
|
||||
|
|
@ -548,4 +549,75 @@ export const schema: FormSchema<DefineStepRule> = {
|
|||
},
|
||||
],
|
||||
},
|
||||
newTermsFields: {
|
||||
type: FIELD_TYPES.COMBO_BOX,
|
||||
label: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.newTermsFieldsLabel',
|
||||
{
|
||||
defaultMessage: 'Fields',
|
||||
}
|
||||
),
|
||||
helpText: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldNewTermsFieldHelpText',
|
||||
{
|
||||
defaultMessage: 'Select a field to check for new terms.',
|
||||
}
|
||||
),
|
||||
validations: [
|
||||
{
|
||||
validator: (
|
||||
...args: Parameters<ValidationFunc>
|
||||
): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
|
||||
const [{ formData }] = args;
|
||||
const needsValidation = isNewTermsRule(formData.ruleType);
|
||||
if (!needsValidation) {
|
||||
return;
|
||||
}
|
||||
|
||||
return fieldValidators.emptyField(
|
||||
i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.newTermsFieldsMin',
|
||||
{
|
||||
defaultMessage: 'Number of fields must be 1.',
|
||||
}
|
||||
)
|
||||
)(...args);
|
||||
},
|
||||
},
|
||||
{
|
||||
validator: (
|
||||
...args: Parameters<ValidationFunc>
|
||||
): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
|
||||
const [{ formData }] = args;
|
||||
const needsValidation = isNewTermsRule(formData.ruleType);
|
||||
if (!needsValidation) {
|
||||
return;
|
||||
}
|
||||
return fieldValidators.maxLengthField({
|
||||
length: 1,
|
||||
message: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.validations.stepDefineRule.newTermsFieldsMax',
|
||||
{
|
||||
defaultMessage: 'Number of fields must be 1.',
|
||||
}
|
||||
),
|
||||
})(...args);
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
historyWindowSize: {
|
||||
label: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.historyWindowSizeLabel',
|
||||
{
|
||||
defaultMessage: 'History Window Size',
|
||||
}
|
||||
),
|
||||
helpText: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.stepScheduleRule.historyWindowSizeHelpText',
|
||||
{
|
||||
defaultMessage: "New terms rules only alert if terms don't appear in historical data.",
|
||||
}
|
||||
),
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -145,6 +145,8 @@ export const RuleSchema = t.intersection([
|
|||
license,
|
||||
meta: MetaRule,
|
||||
machine_learning_job_id: t.array(t.string),
|
||||
new_terms_fields: t.array(t.string),
|
||||
history_window_start: t.string,
|
||||
output_index: t.string,
|
||||
query: t.string,
|
||||
rule_name_override,
|
||||
|
|
|
|||
|
|
@ -161,6 +161,8 @@ export const mockRuleWithEverything = (id: string): Rule => ({
|
|||
timestamp_override_fallback_disabled: false,
|
||||
note: '# this is some markdown documentation',
|
||||
version: 1,
|
||||
new_terms_fields: ['host.name'],
|
||||
history_window_start: 'now-7d',
|
||||
});
|
||||
|
||||
// TODO: update types mapping
|
||||
|
|
@ -214,6 +216,8 @@ export const mockDefineStepRule = (): DefineStepRule => ({
|
|||
},
|
||||
},
|
||||
eqlOptions: {},
|
||||
newTermsFields: ['host.ip'],
|
||||
historyWindowSize: '7d',
|
||||
});
|
||||
|
||||
export const mockScheduleStepRule = (): ScheduleStepRule => ({
|
||||
|
|
|
|||
|
|
@ -97,6 +97,8 @@ export interface RuleFields {
|
|||
threatMapping?: unknown;
|
||||
threatLanguage?: unknown;
|
||||
eqlOptions: unknown;
|
||||
newTermsFields?: unknown;
|
||||
historyWindowSize?: unknown;
|
||||
}
|
||||
|
||||
type QueryRuleFields<T> = Omit<
|
||||
|
|
@ -108,6 +110,8 @@ type QueryRuleFields<T> = Omit<
|
|||
| 'threatQueryBar'
|
||||
| 'threatMapping'
|
||||
| 'eqlOptions'
|
||||
| 'newTermsFields'
|
||||
| 'historyWindowSize'
|
||||
>;
|
||||
type EqlQueryRuleFields<T> = Omit<
|
||||
T,
|
||||
|
|
@ -117,6 +121,8 @@ type EqlQueryRuleFields<T> = Omit<
|
|||
| 'threatIndex'
|
||||
| 'threatQueryBar'
|
||||
| 'threatMapping'
|
||||
| 'newTermsFields'
|
||||
| 'historyWindowSize'
|
||||
>;
|
||||
type ThresholdRuleFields<T> = Omit<
|
||||
T,
|
||||
|
|
@ -126,6 +132,8 @@ type ThresholdRuleFields<T> = Omit<
|
|||
| 'threatQueryBar'
|
||||
| 'threatMapping'
|
||||
| 'eqlOptions'
|
||||
| 'newTermsFields'
|
||||
| 'historyWindowSize'
|
||||
>;
|
||||
type MlRuleFields<T> = Omit<
|
||||
T,
|
||||
|
|
@ -136,10 +144,27 @@ type MlRuleFields<T> = Omit<
|
|||
| 'threatQueryBar'
|
||||
| 'threatMapping'
|
||||
| 'eqlOptions'
|
||||
| 'newTermsFields'
|
||||
| 'historyWindowSize'
|
||||
>;
|
||||
type ThreatMatchRuleFields<T> = Omit<
|
||||
T,
|
||||
'anomalyThreshold' | 'machineLearningJobId' | 'threshold' | 'eqlOptions'
|
||||
| 'anomalyThreshold'
|
||||
| 'machineLearningJobId'
|
||||
| 'threshold'
|
||||
| 'eqlOptions'
|
||||
| 'newTermsFields'
|
||||
| 'historyWindowSize'
|
||||
>;
|
||||
type NewTermsRuleFields<T> = Omit<
|
||||
T,
|
||||
| 'anomalyThreshold'
|
||||
| 'machineLearningJobId'
|
||||
| 'threshold'
|
||||
| 'threatIndex'
|
||||
| 'threatQueryBar'
|
||||
| 'threatMapping'
|
||||
| 'eqlOptions'
|
||||
>;
|
||||
|
||||
const isMlFields = <T>(
|
||||
|
|
@ -149,6 +174,7 @@ const isMlFields = <T>(
|
|||
| MlRuleFields<T>
|
||||
| ThresholdRuleFields<T>
|
||||
| ThreatMatchRuleFields<T>
|
||||
| NewTermsRuleFields<T>
|
||||
): fields is MlRuleFields<T> => has('anomalyThreshold', fields);
|
||||
|
||||
const isThresholdFields = <T>(
|
||||
|
|
@ -158,6 +184,7 @@ const isThresholdFields = <T>(
|
|||
| MlRuleFields<T>
|
||||
| ThresholdRuleFields<T>
|
||||
| ThreatMatchRuleFields<T>
|
||||
| NewTermsRuleFields<T>
|
||||
): fields is ThresholdRuleFields<T> => has('threshold', fields);
|
||||
|
||||
const isThreatMatchFields = <T>(
|
||||
|
|
@ -167,8 +194,19 @@ const isThreatMatchFields = <T>(
|
|||
| MlRuleFields<T>
|
||||
| ThresholdRuleFields<T>
|
||||
| ThreatMatchRuleFields<T>
|
||||
| NewTermsRuleFields<T>
|
||||
): fields is ThreatMatchRuleFields<T> => has('threatIndex', fields);
|
||||
|
||||
const isNewTermsFields = <T>(
|
||||
fields:
|
||||
| QueryRuleFields<T>
|
||||
| EqlQueryRuleFields<T>
|
||||
| MlRuleFields<T>
|
||||
| ThresholdRuleFields<T>
|
||||
| ThreatMatchRuleFields<T>
|
||||
| NewTermsRuleFields<T>
|
||||
): fields is NewTermsRuleFields<T> => has('newTermsFields', fields);
|
||||
|
||||
const isEqlFields = <T>(
|
||||
fields:
|
||||
| QueryRuleFields<T>
|
||||
|
|
@ -176,6 +214,7 @@ const isEqlFields = <T>(
|
|||
| MlRuleFields<T>
|
||||
| ThresholdRuleFields<T>
|
||||
| ThreatMatchRuleFields<T>
|
||||
| NewTermsRuleFields<T>
|
||||
): fields is EqlQueryRuleFields<T> => has('eqlOptions', fields);
|
||||
|
||||
export const filterRuleFieldsForType = <T extends Partial<RuleFields>>(
|
||||
|
|
@ -186,7 +225,8 @@ export const filterRuleFieldsForType = <T extends Partial<RuleFields>>(
|
|||
| EqlQueryRuleFields<T>
|
||||
| MlRuleFields<T>
|
||||
| ThresholdRuleFields<T>
|
||||
| ThreatMatchRuleFields<T> => {
|
||||
| ThreatMatchRuleFields<T>
|
||||
| NewTermsRuleFields<T> => {
|
||||
switch (type) {
|
||||
case 'machine_learning':
|
||||
const {
|
||||
|
|
@ -197,6 +237,8 @@ export const filterRuleFieldsForType = <T extends Partial<RuleFields>>(
|
|||
threatQueryBar,
|
||||
threatMapping,
|
||||
eqlOptions,
|
||||
newTermsFields,
|
||||
historyWindowSize,
|
||||
...mlRuleFields
|
||||
} = fields;
|
||||
return mlRuleFields;
|
||||
|
|
@ -208,6 +250,8 @@ export const filterRuleFieldsForType = <T extends Partial<RuleFields>>(
|
|||
threatQueryBar: _removedThreatQueryBar,
|
||||
threatMapping: _removedThreatMapping,
|
||||
eqlOptions: _eqlOptions,
|
||||
newTermsFields: removedNewTermsFields,
|
||||
historyWindowSize: removedHistoryWindowSize,
|
||||
...thresholdRuleFields
|
||||
} = fields;
|
||||
return thresholdRuleFields;
|
||||
|
|
@ -217,6 +261,8 @@ export const filterRuleFieldsForType = <T extends Partial<RuleFields>>(
|
|||
machineLearningJobId: _removedMachineLearningJobId,
|
||||
threshold: _removedThreshold,
|
||||
eqlOptions: __eqlOptions,
|
||||
newTermsFields: _removedNewTermsFields,
|
||||
historyWindowSize: _removedHistoryWindowSize,
|
||||
...threatMatchRuleFields
|
||||
} = fields;
|
||||
return threatMatchRuleFields;
|
||||
|
|
@ -230,6 +276,8 @@ export const filterRuleFieldsForType = <T extends Partial<RuleFields>>(
|
|||
threatQueryBar: __removedThreatQueryBar,
|
||||
threatMapping: __removedThreatMapping,
|
||||
eqlOptions: ___eqlOptions,
|
||||
newTermsFields: __removedNewTermsFields,
|
||||
historyWindowSize: __removedHistoryWindowSize,
|
||||
...queryRuleFields
|
||||
} = fields;
|
||||
return queryRuleFields;
|
||||
|
|
@ -241,9 +289,23 @@ export const filterRuleFieldsForType = <T extends Partial<RuleFields>>(
|
|||
threatIndex: ___removedThreatIndex,
|
||||
threatQueryBar: ___removedThreatQueryBar,
|
||||
threatMapping: ___removedThreatMapping,
|
||||
newTermsFields: ___removedNewTermsFields,
|
||||
historyWindowSize: ___removedHistoryWindowSize,
|
||||
...eqlRuleFields
|
||||
} = fields;
|
||||
return eqlRuleFields;
|
||||
case 'new_terms':
|
||||
const {
|
||||
anomalyThreshold: ___a,
|
||||
machineLearningJobId: ___m,
|
||||
threshold: ___t,
|
||||
threatIndex: ____removedThreatIndex,
|
||||
threatQueryBar: ____removedThreatQueryBar,
|
||||
threatMapping: ____removedThreatMapping,
|
||||
eqlOptions: ____eqlOptions,
|
||||
...newTermsRuleFields
|
||||
} = fields;
|
||||
return newTermsRuleFields;
|
||||
}
|
||||
assertUnreachable(type);
|
||||
};
|
||||
|
|
@ -339,6 +401,15 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep
|
|||
event_category_override: ruleFields.eqlOptions?.eventCategoryField,
|
||||
tiebreaker_field: ruleFields.eqlOptions?.tiebreakerField,
|
||||
}
|
||||
: isNewTermsFields(ruleFields)
|
||||
? {
|
||||
index: ruleFields.index,
|
||||
filters: ruleFields.queryBar?.filters,
|
||||
language: ruleFields.queryBar?.query?.language,
|
||||
query: ruleFields.queryBar?.query?.query as string,
|
||||
new_terms_fields: ruleFields.newTermsFields,
|
||||
history_window_start: `now-${ruleFields.historyWindowSize}`,
|
||||
}
|
||||
: {
|
||||
index: ruleFields.index,
|
||||
filters: ruleFields.queryBar?.filters,
|
||||
|
|
@ -491,6 +562,8 @@ export const formatPreviewRule = ({
|
|||
machineLearningJobId,
|
||||
anomalyThreshold,
|
||||
eqlOptions,
|
||||
newTermsFields,
|
||||
historyWindowSize,
|
||||
}: {
|
||||
index: string[];
|
||||
dataViewId?: string;
|
||||
|
|
@ -504,6 +577,8 @@ export const formatPreviewRule = ({
|
|||
machineLearningJobId: string[];
|
||||
anomalyThreshold: number;
|
||||
eqlOptions: EqlOptionsSelected;
|
||||
newTermsFields: string[];
|
||||
historyWindowSize: string;
|
||||
}): CreateRulesSchema => {
|
||||
const defineStepData = {
|
||||
...stepDefineDefaultValue,
|
||||
|
|
@ -518,6 +593,8 @@ export const formatPreviewRule = ({
|
|||
machineLearningJobId,
|
||||
anomalyThreshold,
|
||||
eqlOptions,
|
||||
newTermsFields,
|
||||
historyWindowSize,
|
||||
};
|
||||
const aboutStepData = {
|
||||
...stepAboutDefaultValue,
|
||||
|
|
|
|||
|
|
@ -110,6 +110,8 @@ describe('rule helpers', () => {
|
|||
eventCategoryField: undefined,
|
||||
tiebreakerField: undefined,
|
||||
},
|
||||
newTermsFields: ['host.name'],
|
||||
historyWindowSize: '7d',
|
||||
};
|
||||
|
||||
const aboutRuleStepData: AboutStepRule = {
|
||||
|
|
@ -248,6 +250,8 @@ describe('rule helpers', () => {
|
|||
eventCategoryField: undefined,
|
||||
tiebreakerField: undefined,
|
||||
},
|
||||
newTermsFields: [],
|
||||
historyWindowSize: '7d',
|
||||
};
|
||||
|
||||
expect(result).toEqual(expected);
|
||||
|
|
@ -297,6 +301,8 @@ describe('rule helpers', () => {
|
|||
eventCategoryField: undefined,
|
||||
tiebreakerField: undefined,
|
||||
},
|
||||
newTermsFields: [],
|
||||
historyWindowSize: '7d',
|
||||
};
|
||||
|
||||
expect(result).toEqual(expected);
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ export const getActionsStepsData = (
|
|||
};
|
||||
};
|
||||
|
||||
/* eslint-disable complexity */
|
||||
export const getDefineStepsData = (rule: Rule): DefineStepRule => ({
|
||||
ruleType: rule.type,
|
||||
anomalyThreshold: rule.anomaly_threshold ?? 50,
|
||||
|
|
@ -119,8 +120,20 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({
|
|||
eventCategoryField: rule.event_category_override,
|
||||
tiebreakerField: rule.tiebreaker_field,
|
||||
},
|
||||
newTermsFields: rule.new_terms_fields ?? [],
|
||||
historyWindowSize: rule.history_window_start
|
||||
? convertHistoryStartToSize(rule.history_window_start)
|
||||
: '7d',
|
||||
});
|
||||
|
||||
const convertHistoryStartToSize = (relativeTime: string) => {
|
||||
if (relativeTime.startsWith('now-')) {
|
||||
return relativeTime.substring(4);
|
||||
} else {
|
||||
return relativeTime;
|
||||
}
|
||||
};
|
||||
|
||||
export const getScheduleStepsData = (rule: Rule): ScheduleStepRule => {
|
||||
const { interval, from } = rule;
|
||||
const fromHumanizedValue = getHumanizedDuration(from, interval);
|
||||
|
|
@ -354,6 +367,7 @@ const getRuleSpecificRuleParamKeys = (ruleType: Type) => {
|
|||
return ['anomaly_threshold', 'machine_learning_job_id'];
|
||||
case 'threshold':
|
||||
return ['threshold', ...queryRuleParams];
|
||||
case 'new_terms':
|
||||
case 'threat_match':
|
||||
case 'query':
|
||||
case 'saved_query':
|
||||
|
|
|
|||
|
|
@ -153,6 +153,8 @@ export interface DefineStepRule {
|
|||
threatQueryBar: FieldValueQueryBar;
|
||||
threatMapping: ThreatMapping;
|
||||
eqlOptions: EqlOptionsSelected;
|
||||
newTermsFields: string[];
|
||||
historyWindowSize: string;
|
||||
}
|
||||
|
||||
export interface ScheduleStepRule {
|
||||
|
|
|
|||
|
|
@ -48,10 +48,12 @@ import {
|
|||
createQueryAlertType,
|
||||
createSavedQueryAlertType,
|
||||
createThresholdAlertType,
|
||||
createNewTermsAlertType,
|
||||
} from '../../rule_types';
|
||||
import { createSecurityRuleTypeWrapper } from '../../rule_types/create_security_rule_type_wrapper';
|
||||
import { RULE_PREVIEW_INVOCATION_COUNT } from '../../../../../common/detection_engine/constants';
|
||||
import type { RuleExecutionContext, StatusChangeArgs } from '../../rule_execution_log';
|
||||
import { assertUnreachable } from '../../../../../common/utility_types';
|
||||
import { wrapSearchSourceClient } from './utils/wrap_search_source_client';
|
||||
|
||||
const PREVIEW_TIMEOUT_SECONDS = 60;
|
||||
|
|
@ -354,6 +356,19 @@ export const previewRulesRoute = async (
|
|||
{ create: alertInstanceFactoryStub, done: () => ({ getRecoveredAlerts: () => [] }) }
|
||||
);
|
||||
break;
|
||||
case 'new_terms':
|
||||
const newTermsAlertType = previewRuleTypeWrapper(createNewTermsAlertType(ruleOptions));
|
||||
await runExecutors(
|
||||
newTermsAlertType.executor,
|
||||
newTermsAlertType.id,
|
||||
newTermsAlertType.name,
|
||||
previewRuleParams,
|
||||
() => true,
|
||||
{ create: alertInstanceFactoryStub, done: () => ({ getRecoveredAlerts: () => [] }) }
|
||||
);
|
||||
break;
|
||||
default:
|
||||
assertUnreachable(previewRuleParams);
|
||||
}
|
||||
|
||||
// Refreshes alias to ensure index is able to be read before returning
|
||||
|
|
|
|||
|
|
@ -322,6 +322,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
|
|||
wrapHits,
|
||||
wrapSequences,
|
||||
ruleDataReader: ruleDataClient.getReader({ namespace: options.spaceId }),
|
||||
mergeStrategy,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ export interface GenericBulkCreateResponse<T extends BaseFieldsLatest> {
|
|||
createdItemsCount: number;
|
||||
createdItems: Array<AlertWithCommonFieldsLatest<T> & { _id: string; _index: string }>;
|
||||
errors: string[];
|
||||
alertsWereTruncated: boolean;
|
||||
}
|
||||
|
||||
export const bulkCreateFactory =
|
||||
|
|
@ -35,7 +36,8 @@ export const bulkCreateFactory =
|
|||
refreshForBulkCreate: RefreshTypes
|
||||
) =>
|
||||
async <T extends BaseFieldsLatest>(
|
||||
wrappedDocs: Array<WrappedFieldsLatest<T>>
|
||||
wrappedDocs: Array<WrappedFieldsLatest<T>>,
|
||||
maxAlerts?: number
|
||||
): Promise<GenericBulkCreateResponse<T>> => {
|
||||
if (wrappedDocs.length === 0) {
|
||||
return {
|
||||
|
|
@ -44,18 +46,20 @@ export const bulkCreateFactory =
|
|||
bulkCreateDuration: '0',
|
||||
createdItemsCount: 0,
|
||||
createdItems: [],
|
||||
alertsWereTruncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
const start = performance.now();
|
||||
|
||||
const { createdAlerts, errors } = await alertWithPersistence(
|
||||
const { createdAlerts, errors, alertsWereTruncated } = await alertWithPersistence(
|
||||
wrappedDocs.map((doc) => ({
|
||||
_id: doc._id,
|
||||
// `fields` should have already been merged into `doc._source`
|
||||
_source: doc._source,
|
||||
})),
|
||||
refreshForBulkCreate
|
||||
refreshForBulkCreate,
|
||||
maxAlerts
|
||||
);
|
||||
|
||||
const end = performance.now();
|
||||
|
|
@ -76,6 +80,7 @@ export const bulkCreateFactory =
|
|||
bulkCreateDuration: makeFloatString(end - start),
|
||||
createdItemsCount: createdAlerts.length,
|
||||
createdItems: createdAlerts,
|
||||
alertsWereTruncated,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
|
|
@ -84,6 +89,7 @@ export const bulkCreateFactory =
|
|||
bulkCreateDuration: makeFloatString(end - start),
|
||||
createdItemsCount: createdAlerts.length,
|
||||
createdItems: createdAlerts,
|
||||
alertsWereTruncated,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,17 +6,13 @@
|
|||
*/
|
||||
|
||||
import { flattenWithPrefix } from '@kbn/securitysolution-rules';
|
||||
import type * as estypes from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
import type { BaseHit } from '../../../../../../common/detection_engine/types';
|
||||
import type { ConfigType } from '../../../../../config';
|
||||
import type { BuildReasonMessage } from '../../../signals/reason_formatters';
|
||||
import { getMergeStrategy } from '../../../signals/source_fields_merging/strategies';
|
||||
import type {
|
||||
BaseSignalHit,
|
||||
SignalSource,
|
||||
SignalSourceHit,
|
||||
SimpleHit,
|
||||
} from '../../../signals/types';
|
||||
import type { BaseSignalHit, SignalSource, SignalSourceHit } from '../../../signals/types';
|
||||
import { additionalAlertFields, buildAlert } from './build_alert';
|
||||
import { filterSource } from './filter_source';
|
||||
import type { CompleteRule, RuleParams } from '../../../schemas/rule_schemas';
|
||||
|
|
@ -50,7 +46,7 @@ const buildEventTypeAlert = (doc: BaseSignalHit): object => {
|
|||
export const buildBulkBody = (
|
||||
spaceId: string | null | undefined,
|
||||
completeRule: CompleteRule<RuleParams>,
|
||||
doc: SimpleHit,
|
||||
doc: estypes.SearchHit<SignalSource>,
|
||||
mergeStrategy: ConfigType['alertMergeStrategy'],
|
||||
ignoreFields: ConfigType['alertIgnoreFields'],
|
||||
applyOverrides: boolean,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ALERT_UUID } from '@kbn/rule-data-utils';
|
||||
import { ALERT_NEW_TERMS } from '../../../../../../common/field_maps/field_names';
|
||||
import { getCompleteRuleMock, getNewTermsRuleParams } from '../../../schemas/rule_schemas.mock';
|
||||
import { sampleDocNoSortIdWithTimestamp } from '../../../signals/__mocks__/es_results';
|
||||
import { wrapNewTermsAlerts } from './wrap_new_terms_alerts';
|
||||
|
||||
const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71';
|
||||
describe('wrapNewTermsAlerts', () => {
|
||||
test('should create an alert with the correct _id from a document', () => {
|
||||
const doc = sampleDocNoSortIdWithTimestamp(docId);
|
||||
const completeRule = getCompleteRuleMock(getNewTermsRuleParams());
|
||||
const alerts = wrapNewTermsAlerts({
|
||||
eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.1'] }],
|
||||
spaceId: 'default',
|
||||
mergeStrategy: 'missingFields',
|
||||
completeRule,
|
||||
indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
|
||||
});
|
||||
|
||||
expect(alerts[0]._id).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58');
|
||||
expect(alerts[0]._source[ALERT_UUID]).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58');
|
||||
expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1']);
|
||||
});
|
||||
|
||||
test('should create an alert with a different _id if the space is different', () => {
|
||||
const doc = sampleDocNoSortIdWithTimestamp(docId);
|
||||
const completeRule = getCompleteRuleMock(getNewTermsRuleParams());
|
||||
const alerts = wrapNewTermsAlerts({
|
||||
eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.1'] }],
|
||||
spaceId: 'otherSpace',
|
||||
mergeStrategy: 'missingFields',
|
||||
completeRule,
|
||||
indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
|
||||
});
|
||||
|
||||
expect(alerts[0]._id).toEqual('f7877a31b1cc83373dbc9ba5939ebfab1db66545');
|
||||
expect(alerts[0]._source[ALERT_UUID]).toEqual('f7877a31b1cc83373dbc9ba5939ebfab1db66545');
|
||||
expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1']);
|
||||
});
|
||||
|
||||
test('should create an alert with a different _id if the newTerms array is different', () => {
|
||||
const doc = sampleDocNoSortIdWithTimestamp(docId);
|
||||
const completeRule = getCompleteRuleMock(getNewTermsRuleParams());
|
||||
const alerts = wrapNewTermsAlerts({
|
||||
eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.2'] }],
|
||||
spaceId: 'otherSpace',
|
||||
mergeStrategy: 'missingFields',
|
||||
completeRule,
|
||||
indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
|
||||
});
|
||||
|
||||
expect(alerts[0]._id).toEqual('75e5a507a4bc48bcd983820c7fd2d9621ff4e2ea');
|
||||
expect(alerts[0]._source[ALERT_UUID]).toEqual('75e5a507a4bc48bcd983820c7fd2d9621ff4e2ea');
|
||||
expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.2']);
|
||||
});
|
||||
|
||||
test('should create an alert with a different _id if the newTerms array contains multiple terms', () => {
|
||||
const doc = sampleDocNoSortIdWithTimestamp(docId);
|
||||
const completeRule = getCompleteRuleMock(getNewTermsRuleParams());
|
||||
const alerts = wrapNewTermsAlerts({
|
||||
eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.1', '127.0.0.2'] }],
|
||||
spaceId: 'otherSpace',
|
||||
mergeStrategy: 'missingFields',
|
||||
completeRule,
|
||||
indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
|
||||
});
|
||||
|
||||
expect(alerts[0]._id).toEqual('86a216cfa4884767d9bb26d2b8db911cb4aa85ce');
|
||||
expect(alerts[0]._source[ALERT_UUID]).toEqual('86a216cfa4884767d9bb26d2b8db911cb4aa85ce');
|
||||
expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1', '127.0.0.2']);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type * as estypes from '@elastic/elasticsearch/lib/api/types';
|
||||
import objectHash from 'object-hash';
|
||||
import { ALERT_UUID } from '@kbn/rule-data-utils';
|
||||
import type {
|
||||
BaseFieldsLatest,
|
||||
NewTermsFieldsLatest,
|
||||
WrappedFieldsLatest,
|
||||
} from '../../../../../../common/detection_engine/schemas/alerts';
|
||||
import { ALERT_NEW_TERMS } from '../../../../../../common/field_maps/field_names';
|
||||
import type { ConfigType } from '../../../../../config';
|
||||
import type { CompleteRule, RuleParams } from '../../../schemas/rule_schemas';
|
||||
import { buildReasonMessageForNewTermsAlert } from '../../../signals/reason_formatters';
|
||||
import type { SignalSource } from '../../../signals/types';
|
||||
import { buildBulkBody } from './build_bulk_body';
|
||||
|
||||
export interface EventsAndTerms {
|
||||
event: estypes.SearchHit<SignalSource>;
|
||||
newTerms: Array<string | number | null>;
|
||||
}
|
||||
|
||||
export const wrapNewTermsAlerts = ({
|
||||
eventsAndTerms,
|
||||
spaceId,
|
||||
completeRule,
|
||||
mergeStrategy,
|
||||
indicesToQuery,
|
||||
}: {
|
||||
eventsAndTerms: EventsAndTerms[];
|
||||
spaceId: string | null | undefined;
|
||||
completeRule: CompleteRule<RuleParams>;
|
||||
mergeStrategy: ConfigType['alertMergeStrategy'];
|
||||
indicesToQuery: string[];
|
||||
}): Array<WrappedFieldsLatest<NewTermsFieldsLatest>> => {
|
||||
return eventsAndTerms.map((eventAndTerms) => {
|
||||
const id = objectHash([
|
||||
eventAndTerms.event._index,
|
||||
eventAndTerms.event._id,
|
||||
String(eventAndTerms.event._version),
|
||||
`${spaceId}:${completeRule.alertId}`,
|
||||
eventAndTerms.newTerms,
|
||||
]);
|
||||
const baseAlert: BaseFieldsLatest = buildBulkBody(
|
||||
spaceId,
|
||||
completeRule,
|
||||
eventAndTerms.event,
|
||||
mergeStrategy,
|
||||
[],
|
||||
true,
|
||||
buildReasonMessageForNewTermsAlert,
|
||||
indicesToQuery
|
||||
);
|
||||
return {
|
||||
_id: id,
|
||||
_index: '',
|
||||
_source: {
|
||||
...baseAlert,
|
||||
[ALERT_NEW_TERMS]: eventAndTerms.newTerms,
|
||||
[ALERT_UUID]: id,
|
||||
},
|
||||
};
|
||||
});
|
||||
};
|
||||
|
|
@ -11,3 +11,4 @@ export { createMlAlertType } from './ml/create_ml_alert_type';
|
|||
export { createQueryAlertType } from './query/create_query_alert_type';
|
||||
export { createSavedQueryAlertType } from './saved_query/create_saved_query_alert_type';
|
||||
export { createThresholdAlertType } from './threshold/create_threshold_alert_type';
|
||||
export { createNewTermsAlertType } from './new_terms/create_new_terms_alert_type';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
## Design
|
||||
|
||||
The rule accepts 2 new parameters that are unique to the new_terms rule type, in addition to common Security rule parameters such as query, index, and filters, to, from, etc. The new parameters are:
|
||||
|
||||
- `new_terms_fields`: an array of field names, currently limited to an array of size 1. In the future we will likely allow multiple field names to be specified here.
|
||||
Example: ['host.ip']
|
||||
- `history_window_start`: defines the additional time range to search over when determining if a term is "new". If a term is found between the times `history_window_start` and from then it will not be classified as a new term.
|
||||
Example: now-30d
|
||||
The rule pages through all terms that have appeared in the last rule interval and checks each term to determine if it's new. It pages through terms 10000 at a time.
|
||||
|
||||
Each page is evaluated in 3 phases.
|
||||
Phase 1: Collect "recent" terms - terms that have appeared in the last rule interval, without regard to whether or not they have appeared in historical data. This is done using a composite aggregation to ensure we can iterate over every term.
|
||||
|
||||
Phase 2: Check if the page of terms contains any new terms. This uses a regular terms agg with the include parameter - every term is added to the array of include values, so the terms agg is limited to only aggregating on the terms of interest from phase 1. This avoids issues with the terms agg providing approximate results due to getting different terms from different shards.
|
||||
|
||||
Phase 3: Any new terms from phase 2 are processed and the first document to contain that term is retrieved. The document becomes the basis of the generated alert. This is done with an aggregation query that is very similar to the agg used in phase 2, except it also includes a top_hits agg. top_hits is moved to a separate, later phase for efficiency - top_hits is slow and most terms will not be new in phase 2. This means we only execute the top_hits agg on the terms that are actually new which is faster.
|
||||
|
||||
## Alert schema
|
||||
|
||||
New terms alerts have one special field at the moment: `kibana.alert.new_terms`. This field contains the detected term that caused the alert. A single source document may have multiple new terms if the source document contains an array of values in the specified field. In that case, multiple alerts will be generated from the single source document - one for each new value.
|
||||
|
||||
## Timestamp override and fallback
|
||||
|
||||
The new terms rule type reuses the singleSearchAfter function which implements timestamp fallback for queries automatically. However, the min aggregation by timestamp necessitates a slightly more complex fallback strategy since min aggs only accept a single field. If a timestamp override is defined, the new terms rule type defines a query-time runtime field `kibana.combined_timestamp` which is defined as the timestamp override value if it exists, otherwise `@timestamp`, for each document. We can then use the min aggregation on this runtime field to calculate the earliest time a term was found.
|
||||
|
||||
## Limitations and future enhancements
|
||||
|
||||
- Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions.
|
||||
- In the future we may want to support searching for new sets of terms, e.g. a pair of `host.ip` and `host.id` that has never been seen together before.
|
||||
|
|
@ -0,0 +1,137 @@
|
|||
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||
|
||||
exports[`aggregations buildDocFetchAgg builds a correct top hits aggregation 1`] = `
|
||||
Object {
|
||||
"new_terms": Object {
|
||||
"aggs": Object {
|
||||
"docs": Object {
|
||||
"top_hits": Object {
|
||||
"size": 1,
|
||||
"sort": Array [
|
||||
Object {
|
||||
"@timestamp": "asc",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
"terms": Object {
|
||||
"field": "host.name",
|
||||
"include": Array [
|
||||
"myHost",
|
||||
],
|
||||
"size": 10000,
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`aggregations buildNewTermsAggregation builds a correct aggregation with @timestamp 1`] = `
|
||||
Object {
|
||||
"new_terms": Object {
|
||||
"aggs": Object {
|
||||
"filtering_agg": Object {
|
||||
"bucket_selector": Object {
|
||||
"buckets_path": Object {
|
||||
"first_seen_value": "first_seen",
|
||||
},
|
||||
"script": Object {
|
||||
"params": Object {
|
||||
"start_time": 1650000000,
|
||||
},
|
||||
"source": "params.first_seen_value > params.start_time",
|
||||
},
|
||||
},
|
||||
},
|
||||
"first_seen": Object {
|
||||
"min": Object {
|
||||
"field": "@timestamp",
|
||||
},
|
||||
},
|
||||
},
|
||||
"terms": Object {
|
||||
"field": "host.ip",
|
||||
"include": Array [
|
||||
"myHost",
|
||||
],
|
||||
"size": 10000,
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`aggregations buildNewTermsAggregation builds a correct aggregation with event.ingested 1`] = `
|
||||
Object {
|
||||
"new_terms": Object {
|
||||
"aggs": Object {
|
||||
"filtering_agg": Object {
|
||||
"bucket_selector": Object {
|
||||
"buckets_path": Object {
|
||||
"first_seen_value": "first_seen",
|
||||
},
|
||||
"script": Object {
|
||||
"params": Object {
|
||||
"start_time": 1650935705,
|
||||
},
|
||||
"source": "params.first_seen_value > params.start_time",
|
||||
},
|
||||
},
|
||||
},
|
||||
"first_seen": Object {
|
||||
"min": Object {
|
||||
"field": "event.ingested",
|
||||
},
|
||||
},
|
||||
},
|
||||
"terms": Object {
|
||||
"field": "host.name",
|
||||
"include": Array [
|
||||
"myHost",
|
||||
],
|
||||
"size": 10000,
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`aggregations buildRecentTermsAgg builds a correct composite agg without \`after\` 1`] = `
|
||||
Object {
|
||||
"new_terms": Object {
|
||||
"composite": Object {
|
||||
"after": undefined,
|
||||
"size": 10000,
|
||||
"sources": Array [
|
||||
Object {
|
||||
"host.name": Object {
|
||||
"terms": Object {
|
||||
"field": "host.name",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`aggregations buildRecentTermsAgg builds a correct composite aggregation with \`after\` 1`] = `
|
||||
Object {
|
||||
"new_terms": Object {
|
||||
"composite": Object {
|
||||
"after": Object {
|
||||
"host.name": "myHost",
|
||||
},
|
||||
"size": 10000,
|
||||
"sources": Array [
|
||||
Object {
|
||||
"host.name": Object {
|
||||
"terms": Object {
|
||||
"field": "host.name",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||
|
||||
exports[`buildTimestampRuntimeMapping builds a correct timestamp fallback runtime mapping 1`] = `
|
||||
Object {
|
||||
"kibana.combined_timestamp": Object {
|
||||
"script": Object {
|
||||
"params": Object {
|
||||
"timestampOverride": "event.ingested",
|
||||
},
|
||||
"source": "
|
||||
if (doc.containsKey(params.timestampOverride) && doc[params.timestampOverride].size()!=0) {
|
||||
emit(doc[params.timestampOverride].value.millis);
|
||||
} else {
|
||||
emit(doc['@timestamp'].value.millis);
|
||||
}
|
||||
",
|
||||
},
|
||||
"type": "date",
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import moment from 'moment';
|
||||
import {
|
||||
buildDocFetchAgg,
|
||||
buildNewTermsAgg,
|
||||
buildRecentTermsAgg,
|
||||
} from './build_new_terms_aggregation';
|
||||
|
||||
describe('aggregations', () => {
|
||||
describe('buildRecentTermsAgg', () => {
|
||||
test('builds a correct composite agg without `after`', () => {
|
||||
const aggregation = buildRecentTermsAgg({
|
||||
field: 'host.name',
|
||||
after: undefined,
|
||||
});
|
||||
|
||||
expect(aggregation).toMatchSnapshot();
|
||||
});
|
||||
|
||||
test('builds a correct composite aggregation with `after`', () => {
|
||||
const aggregation = buildRecentTermsAgg({
|
||||
field: 'host.name',
|
||||
after: { 'host.name': 'myHost' },
|
||||
});
|
||||
|
||||
expect(aggregation).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildNewTermsAggregation', () => {
|
||||
test('builds a correct aggregation with event.ingested', () => {
|
||||
const newValueWindowStart = moment(1650935705);
|
||||
const aggregation = buildNewTermsAgg({
|
||||
newValueWindowStart,
|
||||
field: 'host.name',
|
||||
timestampField: 'event.ingested',
|
||||
include: ['myHost'],
|
||||
});
|
||||
|
||||
expect(aggregation).toMatchSnapshot();
|
||||
});
|
||||
|
||||
test('builds a correct aggregation with @timestamp', () => {
|
||||
const newValueWindowStart = moment(1650000000);
|
||||
const aggregation = buildNewTermsAgg({
|
||||
newValueWindowStart,
|
||||
field: 'host.ip',
|
||||
timestampField: '@timestamp',
|
||||
include: ['myHost'],
|
||||
});
|
||||
|
||||
expect(aggregation).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildDocFetchAgg', () => {
|
||||
test('builds a correct top hits aggregation', () => {
|
||||
const aggregation = buildDocFetchAgg({
|
||||
field: 'host.name',
|
||||
timestampField: '@timestamp',
|
||||
include: ['myHost'],
|
||||
});
|
||||
|
||||
expect(aggregation).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Moment } from 'moment';
|
||||
import type { ESSearchResponse } from '@kbn/core/types/elasticsearch';
|
||||
import type { SignalSource } from '../../signals/types';
|
||||
|
||||
export type RecentTermsAggResult = ESSearchResponse<
|
||||
SignalSource,
|
||||
{ body: { aggregations: ReturnType<typeof buildRecentTermsAgg> } }
|
||||
>;
|
||||
|
||||
export type NewTermsAggResult = ESSearchResponse<
|
||||
SignalSource,
|
||||
{ body: { aggregations: ReturnType<typeof buildNewTermsAgg> } }
|
||||
>;
|
||||
|
||||
export type DocFetchAggResult = ESSearchResponse<
|
||||
SignalSource,
|
||||
{ body: { aggregations: ReturnType<typeof buildDocFetchAgg> } }
|
||||
>;
|
||||
|
||||
const PAGE_SIZE = 10000;
|
||||
|
||||
/**
|
||||
* Creates an aggregation that pages through all terms. Used to find the terms that have appeared recently,
|
||||
* without regard to whether or not they're actually new.
|
||||
*/
|
||||
export const buildRecentTermsAgg = ({
|
||||
field,
|
||||
after,
|
||||
}: {
|
||||
field: string;
|
||||
after: Record<string, string | number | null> | undefined;
|
||||
}) => {
|
||||
return {
|
||||
new_terms: {
|
||||
composite: {
|
||||
sources: [
|
||||
{
|
||||
[field]: {
|
||||
terms: {
|
||||
field,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
size: PAGE_SIZE,
|
||||
after,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates an aggregation that returns a bucket for each term in the `include` array
|
||||
* that only appears after the time `newValueWindowStart`.
|
||||
*/
|
||||
export const buildNewTermsAgg = ({
|
||||
newValueWindowStart,
|
||||
field,
|
||||
timestampField,
|
||||
include,
|
||||
}: {
|
||||
newValueWindowStart: Moment;
|
||||
field: string;
|
||||
timestampField: string;
|
||||
include: Array<string | number>;
|
||||
}) => {
|
||||
return {
|
||||
new_terms: {
|
||||
terms: {
|
||||
field,
|
||||
size: PAGE_SIZE,
|
||||
// include actually accepts strings or numbers, so we cast to string[] to make TS happy
|
||||
include: include as string[],
|
||||
},
|
||||
aggs: {
|
||||
first_seen: {
|
||||
min: {
|
||||
field: timestampField,
|
||||
},
|
||||
},
|
||||
filtering_agg: {
|
||||
bucket_selector: {
|
||||
buckets_path: {
|
||||
first_seen_value: 'first_seen',
|
||||
},
|
||||
script: {
|
||||
params: {
|
||||
start_time: newValueWindowStart.valueOf(),
|
||||
},
|
||||
source: 'params.first_seen_value > params.start_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates an aggregation that fetches the oldest document for each value in the `include` array.
|
||||
*/
|
||||
export const buildDocFetchAgg = ({
|
||||
field,
|
||||
timestampField,
|
||||
include,
|
||||
}: {
|
||||
field: string;
|
||||
timestampField: string;
|
||||
include: Array<string | number>;
|
||||
}) => {
|
||||
return {
|
||||
new_terms: {
|
||||
terms: {
|
||||
field,
|
||||
size: PAGE_SIZE,
|
||||
// include actually accepts strings or numbers, so we cast to string[] to make TS happy
|
||||
include: include as string[],
|
||||
},
|
||||
aggs: {
|
||||
docs: {
|
||||
top_hits: {
|
||||
size: 1,
|
||||
sort: [
|
||||
{
|
||||
[timestampField]: 'asc' as const,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { buildTimestampRuntimeMapping } from './build_timestamp_runtime_mapping';
|
||||
|
||||
describe('buildTimestampRuntimeMapping', () => {
|
||||
test('builds a correct timestamp fallback runtime mapping', () => {
|
||||
const runtimeMapping = buildTimestampRuntimeMapping({
|
||||
timestampOverride: 'event.ingested',
|
||||
});
|
||||
|
||||
expect(runtimeMapping).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type * as estypes from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
export const TIMESTAMP_RUNTIME_FIELD = 'kibana.combined_timestamp' as const;
|
||||
|
||||
export const buildTimestampRuntimeMapping = ({
|
||||
timestampOverride,
|
||||
}: {
|
||||
timestampOverride: string;
|
||||
}): estypes.MappingRuntimeFields => {
|
||||
return {
|
||||
[TIMESTAMP_RUNTIME_FIELD]: {
|
||||
type: 'date',
|
||||
script: {
|
||||
source: `
|
||||
if (doc.containsKey(params.timestampOverride) && doc[params.timestampOverride].size()!=0) {
|
||||
emit(doc[params.timestampOverride].value.millis);
|
||||
} else {
|
||||
emit(doc['@timestamp'].value.millis);
|
||||
}
|
||||
`,
|
||||
params: {
|
||||
timestampOverride,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
|
@ -0,0 +1,345 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import dateMath from '@elastic/datemath';
|
||||
import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
|
||||
import { NEW_TERMS_RULE_TYPE_ID } from '@kbn/securitysolution-rules';
|
||||
import { SERVER_APP_ID } from '../../../../../common/constants';
|
||||
|
||||
import type { NewTermsRuleParams } from '../../schemas/rule_schemas';
|
||||
import { newTermsRuleParams } from '../../schemas/rule_schemas';
|
||||
import type { CreateRuleOptions, SecurityAlertType } from '../types';
|
||||
import { singleSearchAfter } from '../../signals/single_search_after';
|
||||
import { getFilter } from '../../signals/get_filter';
|
||||
import type { GenericBulkCreateResponse } from '../factories';
|
||||
import type { BaseFieldsLatest } from '../../../../../common/detection_engine/schemas/alerts';
|
||||
import { wrapNewTermsAlerts } from '../factories/utils/wrap_new_terms_alerts';
|
||||
import type {
|
||||
DocFetchAggResult,
|
||||
RecentTermsAggResult,
|
||||
NewTermsAggResult,
|
||||
} from './build_new_terms_aggregation';
|
||||
import {
|
||||
buildDocFetchAgg,
|
||||
buildRecentTermsAgg,
|
||||
buildNewTermsAgg,
|
||||
} from './build_new_terms_aggregation';
|
||||
import {
|
||||
buildTimestampRuntimeMapping,
|
||||
TIMESTAMP_RUNTIME_FIELD,
|
||||
} from './build_timestamp_runtime_mapping';
|
||||
import type { SignalSource } from '../../signals/types';
|
||||
|
||||
interface BulkCreateResults {
|
||||
bulkCreateTimes: string[];
|
||||
createdSignalsCount: number;
|
||||
createdSignals: unknown[];
|
||||
success: boolean;
|
||||
errors: string[];
|
||||
alertsWereTruncated: boolean;
|
||||
}
|
||||
|
||||
interface SearchAfterResults {
|
||||
searchDurations: string[];
|
||||
searchErrors: string[];
|
||||
}
|
||||
|
||||
const addBulkCreateResults = (
|
||||
results: BulkCreateResults,
|
||||
newResults: GenericBulkCreateResponse<BaseFieldsLatest>
|
||||
): BulkCreateResults => {
|
||||
return {
|
||||
bulkCreateTimes: [...results.bulkCreateTimes, newResults.bulkCreateDuration],
|
||||
createdSignalsCount: results.createdSignalsCount + newResults.createdItemsCount,
|
||||
createdSignals: [...results.createdSignals, ...newResults.createdItems],
|
||||
success: results.success && newResults.success,
|
||||
errors: [...results.errors, ...newResults.errors],
|
||||
alertsWereTruncated: results.alertsWereTruncated || newResults.alertsWereTruncated,
|
||||
};
|
||||
};
|
||||
|
||||
export const createNewTermsAlertType = (
|
||||
createOptions: CreateRuleOptions
|
||||
): SecurityAlertType<NewTermsRuleParams, {}, {}, 'default'> => {
|
||||
const { logger } = createOptions;
|
||||
return {
|
||||
id: NEW_TERMS_RULE_TYPE_ID,
|
||||
name: 'New Terms Rule',
|
||||
validate: {
|
||||
params: {
|
||||
validate: (object: unknown) => {
|
||||
const [validated, errors] = validateNonExact(object, newTermsRuleParams);
|
||||
if (errors != null) {
|
||||
throw new Error(errors);
|
||||
}
|
||||
if (validated == null) {
|
||||
throw new Error('Validation of rule params failed');
|
||||
}
|
||||
return validated;
|
||||
},
|
||||
},
|
||||
},
|
||||
actionGroups: [
|
||||
{
|
||||
id: 'default',
|
||||
name: 'Default',
|
||||
},
|
||||
],
|
||||
defaultActionGroupId: 'default',
|
||||
actionVariables: {
|
||||
context: [{ name: 'server', description: 'the server' }],
|
||||
},
|
||||
minimumLicenseRequired: 'basic',
|
||||
isExportable: false,
|
||||
producer: SERVER_APP_ID,
|
||||
async executor(execOptions) {
|
||||
const {
|
||||
runOpts: {
|
||||
buildRuleMessage,
|
||||
bulkCreate,
|
||||
completeRule,
|
||||
exceptionItems,
|
||||
tuple,
|
||||
mergeStrategy,
|
||||
inputIndex,
|
||||
runtimeMappings,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
},
|
||||
services,
|
||||
params,
|
||||
spaceId,
|
||||
} = execOptions;
|
||||
|
||||
const filter = await getFilter({
|
||||
filters: params.filters,
|
||||
index: inputIndex,
|
||||
language: params.language,
|
||||
savedId: undefined,
|
||||
services,
|
||||
type: params.type,
|
||||
query: params.query,
|
||||
lists: exceptionItems,
|
||||
});
|
||||
|
||||
const parsedHistoryWindowSize = dateMath.parse(params.historyWindowStart, {
|
||||
forceNow: tuple.to.toDate(),
|
||||
});
|
||||
if (parsedHistoryWindowSize == null) {
|
||||
throw Error(`Failed to parse 'historyWindowStart'`);
|
||||
}
|
||||
|
||||
let afterKey;
|
||||
let bulkCreateResults: BulkCreateResults = {
|
||||
bulkCreateTimes: [],
|
||||
createdSignalsCount: 0,
|
||||
createdSignals: [],
|
||||
success: true,
|
||||
errors: [],
|
||||
alertsWereTruncated: false,
|
||||
};
|
||||
|
||||
const searchAfterResults: SearchAfterResults = {
|
||||
searchDurations: [],
|
||||
searchErrors: [],
|
||||
};
|
||||
|
||||
// If we have a timestampOverride, we'll compute a runtime field that emits the override for each document if it exists,
|
||||
// otherwise it emits @timestamp. If we don't have a timestamp override we don't want to pay the cost of using a
|
||||
// runtime field, so we just use @timestamp directly.
|
||||
const { timestampField, timestampRuntimeMappings } = params.timestampOverride
|
||||
? {
|
||||
timestampField: TIMESTAMP_RUNTIME_FIELD,
|
||||
timestampRuntimeMappings: buildTimestampRuntimeMapping({
|
||||
timestampOverride: params.timestampOverride,
|
||||
}),
|
||||
}
|
||||
: { timestampField: '@timestamp', timestampRuntimeMappings: undefined };
|
||||
|
||||
// There are 2 conditions that mean we're finished: either there were still too many alerts to create
|
||||
// after deduplication and the array of alerts was truncated before being submitted to ES, or there were
|
||||
// exactly enough new alerts to hit maxSignals without truncating the array of alerts. We check both because
|
||||
// it's possible for the array to be truncated but alert documents could fail to be created for other reasons,
|
||||
// in which case createdSignalsCount would still be less than maxSignals. Since valid alerts were truncated from
|
||||
// the array in that case, we stop and report the errors.
|
||||
while (
|
||||
!bulkCreateResults.alertsWereTruncated &&
|
||||
bulkCreateResults.createdSignalsCount < params.maxSignals
|
||||
) {
|
||||
// PHASE 1: Fetch a page of terms using a composite aggregation. This will collect a page from
|
||||
// all of the terms seen over the last rule interval. In the next phase we'll determine which
|
||||
// ones are new.
|
||||
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
|
||||
aggregations: buildRecentTermsAgg({
|
||||
field: params.newTermsFields[0],
|
||||
after: afterKey,
|
||||
}),
|
||||
searchAfterSortIds: undefined,
|
||||
index: inputIndex,
|
||||
// The time range for the initial composite aggregation is the rule interval, `from` and `to`
|
||||
from: tuple.from.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
services,
|
||||
filter,
|
||||
logger,
|
||||
pageSize: 0,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
buildRuleMessage,
|
||||
runtimeMappings,
|
||||
});
|
||||
const searchResultWithAggs = searchResult as RecentTermsAggResult;
|
||||
if (!searchResultWithAggs.aggregations) {
|
||||
throw new Error('expected to find aggregations on search result');
|
||||
}
|
||||
logger.debug(`Time spent on composite agg: ${searchDuration}`);
|
||||
|
||||
searchAfterResults.searchDurations.push(searchDuration);
|
||||
searchAfterResults.searchErrors.push(...searchErrors);
|
||||
|
||||
afterKey = searchResultWithAggs.aggregations.new_terms.after_key;
|
||||
|
||||
// If the aggregation returns no after_key it signals that we've paged through all results
|
||||
// and the current page is empty so we can immediately break.
|
||||
if (afterKey == null) {
|
||||
break;
|
||||
}
|
||||
const bucketsForField = searchResultWithAggs.aggregations.new_terms.buckets;
|
||||
const includeValues = bucketsForField
|
||||
.map((bucket) => Object.values(bucket.key)[0])
|
||||
.filter((value): value is string | number => value != null);
|
||||
|
||||
// PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window.
|
||||
// The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the
|
||||
// response correspond to each new term.
|
||||
const {
|
||||
searchResult: pageSearchResult,
|
||||
searchDuration: pageSearchDuration,
|
||||
searchErrors: pageSearchErrors,
|
||||
} = await singleSearchAfter({
|
||||
aggregations: buildNewTermsAgg({
|
||||
newValueWindowStart: tuple.from,
|
||||
timestampField,
|
||||
field: params.newTermsFields[0],
|
||||
include: includeValues,
|
||||
}),
|
||||
runtimeMappings: {
|
||||
...runtimeMappings,
|
||||
...timestampRuntimeMappings,
|
||||
},
|
||||
searchAfterSortIds: undefined,
|
||||
index: inputIndex,
|
||||
// For Phase 2, we expand the time range to aggregate over the history window
|
||||
// in addition to the rule interval
|
||||
from: parsedHistoryWindowSize.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
services,
|
||||
filter,
|
||||
logger,
|
||||
pageSize: 0,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
buildRuleMessage,
|
||||
});
|
||||
searchAfterResults.searchDurations.push(pageSearchDuration);
|
||||
searchAfterResults.searchErrors.push(...pageSearchErrors);
|
||||
|
||||
logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`);
|
||||
|
||||
const pageSearchResultWithAggs = pageSearchResult as NewTermsAggResult;
|
||||
if (!pageSearchResultWithAggs.aggregations) {
|
||||
throw new Error('expected to find aggregations on page search result');
|
||||
}
|
||||
|
||||
// PHASE 3: For each term that is not in the history window, fetch the oldest document in
|
||||
// the rule interval for that term. This is the first document to contain the new term, and will
|
||||
// become the basis of the resulting alert.
|
||||
// One document could become multiple alerts if the document contains an array with multiple new terms.
|
||||
if (pageSearchResultWithAggs.aggregations.new_terms.buckets.length > 0) {
|
||||
const actualNewTerms = pageSearchResultWithAggs.aggregations.new_terms.buckets.map(
|
||||
(bucket) => bucket.key
|
||||
);
|
||||
|
||||
const {
|
||||
searchResult: docFetchSearchResult,
|
||||
searchDuration: docFetchSearchDuration,
|
||||
searchErrors: docFetchSearchErrors,
|
||||
} = await singleSearchAfter({
|
||||
aggregations: buildDocFetchAgg({
|
||||
timestampField,
|
||||
field: params.newTermsFields[0],
|
||||
include: actualNewTerms,
|
||||
}),
|
||||
runtimeMappings: {
|
||||
...runtimeMappings,
|
||||
...timestampRuntimeMappings,
|
||||
},
|
||||
searchAfterSortIds: undefined,
|
||||
index: inputIndex,
|
||||
// For phase 3, we go back to aggregating only over the rule interval - excluding the history window
|
||||
from: tuple.from.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
services,
|
||||
filter,
|
||||
logger,
|
||||
pageSize: 0,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
buildRuleMessage,
|
||||
});
|
||||
searchAfterResults.searchDurations.push(docFetchSearchDuration);
|
||||
searchAfterResults.searchErrors.push(...docFetchSearchErrors);
|
||||
|
||||
const docFetchResultWithAggs = docFetchSearchResult as DocFetchAggResult;
|
||||
|
||||
if (!docFetchResultWithAggs.aggregations) {
|
||||
throw new Error('expected to find aggregations on page search result');
|
||||
}
|
||||
|
||||
const eventsAndTerms: Array<{
|
||||
event: estypes.SearchHit<SignalSource>;
|
||||
newTerms: Array<string | number | null>;
|
||||
}> = docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => ({
|
||||
event: bucket.docs.hits.hits[0],
|
||||
newTerms: [bucket.key],
|
||||
}));
|
||||
|
||||
const wrappedAlerts = wrapNewTermsAlerts({
|
||||
eventsAndTerms,
|
||||
spaceId,
|
||||
completeRule,
|
||||
mergeStrategy,
|
||||
indicesToQuery: inputIndex,
|
||||
});
|
||||
|
||||
const bulkCreateResult = await bulkCreate(
|
||||
wrappedAlerts,
|
||||
params.maxSignals - bulkCreateResults.createdSignalsCount
|
||||
);
|
||||
|
||||
bulkCreateResults = addBulkCreateResults(bulkCreateResults, bulkCreateResult);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
// If an error occurs but doesn't cause us to throw then we still count the execution as a success.
|
||||
// Should be refactored for better clarity, but that's how it is for now.
|
||||
success: true,
|
||||
warning: false,
|
||||
searchAfterTimes: searchAfterResults.searchDurations,
|
||||
bulkCreateTimes: bulkCreateResults.bulkCreateTimes,
|
||||
lastLookBackDate: undefined,
|
||||
createdSignalsCount: bulkCreateResults.createdSignalsCount,
|
||||
createdSignals: bulkCreateResults.createdSignals,
|
||||
errors: [...searchAfterResults.searchErrors, ...bulkCreateResults.errors],
|
||||
warningMessages: [],
|
||||
state: {},
|
||||
};
|
||||
},
|
||||
};
|
||||
};
|
||||
|
|
@ -69,6 +69,7 @@ export interface RunOpts<TParams extends RuleParams> {
|
|||
ruleDataReader: IRuleDataReader;
|
||||
inputIndex: string[];
|
||||
runtimeMappings: estypes.MappingRuntimeFields | undefined;
|
||||
mergeStrategy: ConfigType['alertMergeStrategy'];
|
||||
primaryTimestamp: string;
|
||||
secondaryTimestamp?: string;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
EQL_RULE_TYPE_ID,
|
||||
INDICATOR_RULE_TYPE_ID,
|
||||
ML_RULE_TYPE_ID,
|
||||
NEW_TERMS_RULE_TYPE_ID,
|
||||
QUERY_RULE_TYPE_ID,
|
||||
SAVED_QUERY_RULE_TYPE_ID,
|
||||
THRESHOLD_RULE_TYPE_ID,
|
||||
|
|
@ -21,7 +22,8 @@ const allAlertTypeIds = `(alert.attributes.alertTypeId: ${EQL_RULE_TYPE_ID}
|
|||
OR alert.attributes.alertTypeId: ${QUERY_RULE_TYPE_ID}
|
||||
OR alert.attributes.alertTypeId: ${SAVED_QUERY_RULE_TYPE_ID}
|
||||
OR alert.attributes.alertTypeId: ${INDICATOR_RULE_TYPE_ID}
|
||||
OR alert.attributes.alertTypeId: ${THRESHOLD_RULE_TYPE_ID})`.replace(/[\n\r]/g, '');
|
||||
OR alert.attributes.alertTypeId: ${THRESHOLD_RULE_TYPE_ID}
|
||||
OR alert.attributes.alertTypeId: ${NEW_TERMS_RULE_TYPE_ID})`.replace(/[\n\r]/g, '');
|
||||
|
||||
describe('enrichFilterWithRuleTypeMapping', () => {
|
||||
test('it returns a full filter with an AND if sent down', () => {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { convertPatchAPIToInternalSchema, patchTypeSpecificSnakeToCamel } from '
|
|||
import {
|
||||
getEqlRuleParams,
|
||||
getMlRuleParams,
|
||||
getNewTermsRuleParams,
|
||||
getQueryRuleParams,
|
||||
getSavedQueryRuleParams,
|
||||
getThreatRuleParams,
|
||||
|
|
@ -179,6 +180,29 @@ describe('rule_converters', () => {
|
|||
'Invalid value "invalid" supplied to "anomaly_threshold"'
|
||||
);
|
||||
});
|
||||
|
||||
test('should accept new terms params when existing rule type is new terms', () => {
|
||||
const patchParams = {
|
||||
new_terms_fields: ['event.new_field'],
|
||||
};
|
||||
const rule = getNewTermsRuleParams();
|
||||
const patchedParams = patchTypeSpecificSnakeToCamel(patchParams, rule);
|
||||
expect(patchedParams).toEqual(
|
||||
expect.objectContaining({
|
||||
newTermsFields: ['event.new_field'],
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
test('should reject invalid new terms params when existing rule type is new terms', () => {
|
||||
const patchParams = {
|
||||
new_terms_fields: 'invalid',
|
||||
};
|
||||
const rule = getNewTermsRuleParams();
|
||||
expect(() => patchTypeSpecificSnakeToCamel(patchParams, rule)).toThrowError(
|
||||
'Invalid value "invalid" supplied to "new_terms_fields"'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('convertPatchAPIToInternalSchema', () => {
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ import type {
|
|||
MachineLearningRuleParams,
|
||||
MachineLearningSpecificRuleParams,
|
||||
InternalRuleUpdate,
|
||||
NewTermsRuleParams,
|
||||
NewTermsSpecificRuleParams,
|
||||
} from './rule_schemas';
|
||||
import { assertUnreachable } from '../../../../common/utility_types';
|
||||
import type {
|
||||
|
|
@ -45,6 +47,7 @@ import type {
|
|||
import {
|
||||
eqlPatchParams,
|
||||
machineLearningPatchParams,
|
||||
newTermsPatchParams,
|
||||
queryPatchParams,
|
||||
savedQueryPatchParams,
|
||||
threatMatchPatchParams,
|
||||
|
|
@ -56,6 +59,7 @@ import type {
|
|||
EqlPatchParams,
|
||||
FullResponseSchema,
|
||||
MachineLearningPatchParams,
|
||||
NewTermsPatchParams,
|
||||
QueryPatchParams,
|
||||
ResponseTypeSpecific,
|
||||
SavedQueryPatchParams,
|
||||
|
|
@ -162,6 +166,18 @@ export const typeSpecificSnakeToCamel = (params: CreateTypeSpecific): TypeSpecif
|
|||
machineLearningJobId: normalizeMachineLearningJobIds(params.machine_learning_job_id),
|
||||
};
|
||||
}
|
||||
case 'new_terms': {
|
||||
return {
|
||||
type: params.type,
|
||||
query: params.query,
|
||||
newTermsFields: params.new_terms_fields,
|
||||
historyWindowStart: params.history_window_start,
|
||||
index: params.index,
|
||||
filters: params.filters,
|
||||
language: params.language ?? 'kuery',
|
||||
dataViewId: params.data_view_id,
|
||||
};
|
||||
}
|
||||
default: {
|
||||
return assertUnreachable(params);
|
||||
}
|
||||
|
|
@ -269,6 +285,22 @@ const patchMachineLearningParams = (
|
|||
};
|
||||
};
|
||||
|
||||
const patchNewTermsParams = (
|
||||
params: NewTermsPatchParams,
|
||||
existingRule: NewTermsRuleParams
|
||||
): NewTermsSpecificRuleParams => {
|
||||
return {
|
||||
type: existingRule.type,
|
||||
language: params.language ?? existingRule.language,
|
||||
index: params.index ?? existingRule.index,
|
||||
dataViewId: params.data_view_id ?? existingRule.dataViewId,
|
||||
query: params.query ?? existingRule.query,
|
||||
filters: params.filters ?? existingRule.filters,
|
||||
newTermsFields: params.new_terms_fields ?? existingRule.newTermsFields,
|
||||
historyWindowStart: params.history_window_start ?? existingRule.historyWindowStart,
|
||||
};
|
||||
};
|
||||
|
||||
const parseValidationError = (error: string | null): BadRequestError => {
|
||||
if (error != null) {
|
||||
return new BadRequestError(error);
|
||||
|
|
@ -329,6 +361,13 @@ export const patchTypeSpecificSnakeToCamel = (
|
|||
}
|
||||
return patchMachineLearningParams(validated, existingRule);
|
||||
}
|
||||
case 'new_terms': {
|
||||
const [validated, error] = validateNonExact(params, newTermsPatchParams);
|
||||
if (validated == null) {
|
||||
throw parseValidationError(error);
|
||||
}
|
||||
return patchNewTermsParams(validated, existingRule);
|
||||
}
|
||||
default: {
|
||||
return assertUnreachable(existingRule);
|
||||
}
|
||||
|
|
@ -542,6 +581,18 @@ export const typeSpecificCamelToSnake = (params: TypeSpecificRuleParams): Respon
|
|||
machine_learning_job_id: params.machineLearningJobId,
|
||||
};
|
||||
}
|
||||
case 'new_terms': {
|
||||
return {
|
||||
type: params.type,
|
||||
query: params.query,
|
||||
new_terms_fields: params.newTermsFields,
|
||||
history_window_start: params.historyWindowStart,
|
||||
index: params.index,
|
||||
filters: params.filters,
|
||||
language: params.language,
|
||||
data_view_id: params.dataViewId,
|
||||
};
|
||||
}
|
||||
default: {
|
||||
return assertUnreachable(params);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import type {
|
|||
CompleteRule,
|
||||
EqlRuleParams,
|
||||
MachineLearningRuleParams,
|
||||
NewTermsRuleParams,
|
||||
QueryRuleParams,
|
||||
RuleParams,
|
||||
SavedQueryRuleParams,
|
||||
|
|
@ -148,6 +149,28 @@ export const getSavedQueryRuleParams = (): SavedQueryRuleParams => {
|
|||
};
|
||||
};
|
||||
|
||||
export const getNewTermsRuleParams = (): NewTermsRuleParams => {
|
||||
return {
|
||||
...getBaseRuleParams(),
|
||||
type: 'new_terms',
|
||||
language: 'kuery',
|
||||
query: 'user.name: root or user.name: admin',
|
||||
index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
|
||||
dataViewId: undefined,
|
||||
filters: [
|
||||
{
|
||||
query: {
|
||||
match_phrase: {
|
||||
'host.name': 'some-host',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
newTermsFields: ['host.name'],
|
||||
historyWindowStart: 'now-30d',
|
||||
};
|
||||
};
|
||||
|
||||
export const getThreatRuleParams = (): ThreatRuleParams => {
|
||||
return {
|
||||
...getBaseRuleParams(),
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import {
|
|||
QUERY_RULE_TYPE_ID,
|
||||
THRESHOLD_RULE_TYPE_ID,
|
||||
SAVED_QUERY_RULE_TYPE_ID,
|
||||
NEW_TERMS_RULE_TYPE_ID,
|
||||
} from '@kbn/securitysolution-rules';
|
||||
|
||||
import type { SanitizedRuleConfig } from '@kbn/alerting-plugin/common';
|
||||
|
|
@ -74,6 +75,8 @@ import {
|
|||
RelatedIntegrationArray,
|
||||
RequiredFieldArray,
|
||||
SetupGuide,
|
||||
newTermsFields,
|
||||
historyWindowStart,
|
||||
timestampOverrideFallbackDisabledOrUndefined,
|
||||
} from '../../../../common/detection_engine/schemas/common';
|
||||
import { SERVER_APP_ID } from '../../../../common/constants';
|
||||
|
|
@ -209,6 +212,20 @@ export const machineLearningRuleParams = t.intersection([
|
|||
export type MachineLearningSpecificRuleParams = t.TypeOf<typeof machineLearningSpecificRuleParams>;
|
||||
export type MachineLearningRuleParams = t.TypeOf<typeof machineLearningRuleParams>;
|
||||
|
||||
const newTermsSpecificRuleParams = t.type({
|
||||
type: t.literal('new_terms'),
|
||||
query,
|
||||
newTermsFields,
|
||||
historyWindowStart,
|
||||
index: indexOrUndefined,
|
||||
filters: filtersOrUndefined,
|
||||
language: nonEqlLanguages,
|
||||
dataViewId: dataViewIdOrUndefined,
|
||||
});
|
||||
export const newTermsRuleParams = t.intersection([baseRuleParams, newTermsSpecificRuleParams]);
|
||||
export type NewTermsSpecificRuleParams = t.TypeOf<typeof newTermsSpecificRuleParams>;
|
||||
export type NewTermsRuleParams = t.TypeOf<typeof newTermsRuleParams>;
|
||||
|
||||
export const typeSpecificRuleParams = t.union([
|
||||
eqlSpecificRuleParams,
|
||||
threatSpecificRuleParams,
|
||||
|
|
@ -216,6 +233,7 @@ export const typeSpecificRuleParams = t.union([
|
|||
savedQuerySpecificRuleParams,
|
||||
thresholdSpecificRuleParams,
|
||||
machineLearningSpecificRuleParams,
|
||||
newTermsSpecificRuleParams,
|
||||
]);
|
||||
export type TypeSpecificRuleParams = t.TypeOf<typeof typeSpecificRuleParams>;
|
||||
|
||||
|
|
@ -243,6 +261,7 @@ export const allRuleTypes = t.union([
|
|||
t.literal(QUERY_RULE_TYPE_ID),
|
||||
t.literal(SAVED_QUERY_RULE_TYPE_ID),
|
||||
t.literal(THRESHOLD_RULE_TYPE_ID),
|
||||
t.literal(NEW_TERMS_RULE_TYPE_ID),
|
||||
]);
|
||||
|
||||
export const internalRuleCreate = t.type({
|
||||
|
|
|
|||
|
|
@ -191,6 +191,7 @@ export const buildEventsSearchQuery = ({
|
|||
...docFields,
|
||||
],
|
||||
...(aggregations ? { aggregations } : {}),
|
||||
runtime_mappings: runtimeMappings,
|
||||
sort,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -99,6 +99,7 @@ export const getFilter = async ({
|
|||
switch (type) {
|
||||
case 'threat_match':
|
||||
case 'threshold':
|
||||
case 'new_terms':
|
||||
case 'query': {
|
||||
return queryFilter();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ export interface BuildReasonMessageArgs {
|
|||
}
|
||||
|
||||
export interface BuildReasonMessageUtilArgs extends BuildReasonMessageArgs {
|
||||
type?: 'eql' | 'ml' | 'query' | 'threatMatch' | 'threshold';
|
||||
type?: 'eql' | 'ml' | 'query' | 'threatMatch' | 'threshold' | 'new_terms';
|
||||
}
|
||||
|
||||
export type BuildReasonMessage = (args: BuildReasonMessageArgs) => string;
|
||||
|
|
@ -134,3 +134,6 @@ export const buildReasonMessageForThreatMatchAlert = (args: BuildReasonMessageAr
|
|||
|
||||
export const buildReasonMessageForThresholdAlert = (args: BuildReasonMessageArgs) =>
|
||||
buildReasonMessageUtil({ ...args, type: 'threshold' });
|
||||
|
||||
export const buildReasonMessageForNewTermsAlert = (args: BuildReasonMessageArgs) =>
|
||||
buildReasonMessageUtil({ ...args, type: 'new_terms' });
|
||||
|
|
|
|||
|
|
@ -127,6 +127,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -144,6 +145,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -161,6 +163,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -178,6 +181,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -239,6 +243,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -256,6 +261,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -273,6 +279,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -349,6 +356,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -480,6 +488,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
mockService.scopedClusterClient.asCurrentUser.search
|
||||
.mockResolvedValueOnce(
|
||||
|
|
@ -613,6 +622,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
const exceptionItem = getExceptionListItemSchemaMock();
|
||||
|
|
@ -683,6 +693,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -861,6 +872,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
statusCode: 500,
|
||||
},
|
||||
},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.bulk.mockResponseOnce(bulkItem); // adds the response with errors we are testing
|
||||
|
|
@ -880,6 +892,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -897,6 +910,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -914,6 +928,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -964,6 +979,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -981,6 +997,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
@ -998,6 +1015,7 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
},
|
||||
],
|
||||
errors: {},
|
||||
alertsWereTruncated: false,
|
||||
});
|
||||
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
|
|
|
|||
|
|
@ -241,7 +241,8 @@ export type BulkResponseErrorAggregation = Record<string, { count: number; statu
|
|||
export type SignalsEnrichment = (signals: SignalSourceHit[]) => Promise<SignalSourceHit[]>;
|
||||
|
||||
export type BulkCreate = <T extends BaseFieldsLatest>(
|
||||
docs: Array<WrappedFieldsLatest<T>>
|
||||
docs: Array<WrappedFieldsLatest<T>>,
|
||||
maxAlerts?: number
|
||||
) => Promise<GenericBulkCreateResponse<T>>;
|
||||
|
||||
export type SimpleHit = BaseHit<{ '@timestamp'?: string }>;
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import {
|
|||
createEqlAlertType,
|
||||
createIndicatorMatchAlertType,
|
||||
createMlAlertType,
|
||||
createNewTermsAlertType,
|
||||
createQueryAlertType,
|
||||
createSavedQueryAlertType,
|
||||
createThresholdAlertType,
|
||||
|
|
@ -256,6 +257,7 @@ export class Plugin implements ISecuritySolutionPlugin {
|
|||
plugins.alerting.registerType(securityRuleTypeWrapper(createMlAlertType(ruleOptions)));
|
||||
plugins.alerting.registerType(securityRuleTypeWrapper(createQueryAlertType(ruleOptions)));
|
||||
plugins.alerting.registerType(securityRuleTypeWrapper(createThresholdAlertType(ruleOptions)));
|
||||
plugins.alerting.registerType(securityRuleTypeWrapper(createNewTermsAlertType(ruleOptions)));
|
||||
|
||||
// TODO We need to get the endpoint routes inside of initRoutes
|
||||
initRoutes(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,422 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { orderBy } from 'lodash';
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { RuleExecutionStatus } from '@kbn/security-solution-plugin/common/detection_engine/schemas/common';
|
||||
import { NewTermsCreateSchema } from '@kbn/security-solution-plugin/common/detection_engine/schemas/request';
|
||||
import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants';
|
||||
import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/detection_engine/schemas/request/rule_schemas.mock';
|
||||
import { DetectionAlert } from '@kbn/security-solution-plugin/common/detection_engine/schemas/alerts';
|
||||
import { FtrProviderContext } from '../../common/ftr_provider_context';
|
||||
import {
|
||||
createRule,
|
||||
createRuleWithExceptionEntries,
|
||||
createSignalsIndex,
|
||||
deleteAllAlerts,
|
||||
deleteSignalsIndex,
|
||||
getOpenSignals,
|
||||
getSignalsByIds,
|
||||
waitForRuleSuccessOrStatus,
|
||||
waitForSignalsToBePresent,
|
||||
} from '../../utils';
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default ({ getService }: FtrProviderContext) => {
|
||||
const esArchiver = getService('esArchiver');
|
||||
const supertest = getService('supertest');
|
||||
const log = getService('log');
|
||||
const es = getService('es');
|
||||
|
||||
/**
|
||||
* Specific api integration tests for threat matching rule type
|
||||
*/
|
||||
describe('create_new_terms', () => {
|
||||
before(async () => {
|
||||
await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts');
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts');
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await createSignalsIndex(supertest, log);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await deleteSignalsIndex(supertest, log);
|
||||
await deleteAllAlerts(supertest, log);
|
||||
});
|
||||
|
||||
it('should create a single rule with a rule_id and validate it ran successfully', async () => {
|
||||
const ruleResponse = await createRule(
|
||||
supertest,
|
||||
log,
|
||||
getCreateNewTermsRulesSchemaMock('rule-1', true)
|
||||
);
|
||||
|
||||
await waitForRuleSuccessOrStatus(
|
||||
supertest,
|
||||
log,
|
||||
ruleResponse.id,
|
||||
RuleExecutionStatus.succeeded
|
||||
);
|
||||
|
||||
const { body: rule } = await supertest
|
||||
.get(DETECTION_ENGINE_RULES_URL)
|
||||
.set('kbn-xsrf', 'true')
|
||||
.query({ id: ruleResponse.id })
|
||||
.expect(200);
|
||||
|
||||
expect(rule?.execution_summary?.last_execution.status).to.eql('succeeded');
|
||||
});
|
||||
|
||||
const removeRandomValuedProperties = (alert: DetectionAlert | undefined) => {
|
||||
if (!alert) {
|
||||
return undefined;
|
||||
}
|
||||
const {
|
||||
'kibana.version': version,
|
||||
'kibana.alert.rule.execution.uuid': execUuid,
|
||||
'kibana.alert.rule.uuid': uuid,
|
||||
'@timestamp': timestamp,
|
||||
'kibana.alert.rule.created_at': createdAt,
|
||||
'kibana.alert.rule.updated_at': updatedAt,
|
||||
'kibana.alert.uuid': alertUuid,
|
||||
...restOfAlert
|
||||
} = alert;
|
||||
return restOfAlert;
|
||||
};
|
||||
|
||||
// This test also tests that alerts are NOT created for terms that are not new: the host name
|
||||
// suricata-sensor-san-francisco appears in a document at 2019-02-19T20:42:08.230Z, but also appears
|
||||
// in earlier documents so is not new. An alert should not be generated for that term.
|
||||
it('should generate 1 alert with 1 selected field', async () => {
|
||||
const rule: NewTermsCreateSchema = {
|
||||
...getCreateNewTermsRulesSchemaMock('rule-1', true),
|
||||
new_terms_fields: ['host.name'],
|
||||
from: '2019-02-19T20:42:00.000Z',
|
||||
history_window_start: '2019-01-19T20:42:00.000Z',
|
||||
};
|
||||
|
||||
const createdRule = await createRule(supertest, log, rule);
|
||||
|
||||
await waitForRuleSuccessOrStatus(
|
||||
supertest,
|
||||
log,
|
||||
createdRule.id,
|
||||
RuleExecutionStatus.succeeded
|
||||
);
|
||||
|
||||
const signalsOpen = await getOpenSignals(supertest, log, es, createdRule);
|
||||
expect(signalsOpen.hits.hits.length).eql(1);
|
||||
expect(removeRandomValuedProperties(signalsOpen.hits.hits[0]._source)).eql({
|
||||
'kibana.alert.new_terms': ['zeek-newyork-sha-aa8df15'],
|
||||
'kibana.alert.rule.category': 'New Terms Rule',
|
||||
'kibana.alert.rule.consumer': 'siem',
|
||||
'kibana.alert.rule.name': 'Query with a rule id',
|
||||
'kibana.alert.rule.producer': 'siem',
|
||||
'kibana.alert.rule.rule_type_id': 'siem.newTermsRule',
|
||||
'kibana.space_ids': ['default'],
|
||||
'kibana.alert.rule.tags': [],
|
||||
agent: {
|
||||
ephemeral_id: '7cc2091a-72f1-4c63-843b-fdeb622f9c69',
|
||||
hostname: 'zeek-newyork-sha-aa8df15',
|
||||
id: '4b4462ef-93d2-409c-87a6-299d942e5047',
|
||||
type: 'auditbeat',
|
||||
version: '8.0.0',
|
||||
},
|
||||
cloud: { instance: { id: '139865230' }, provider: 'digitalocean', region: 'nyc1' },
|
||||
ecs: { version: '1.0.0-beta2' },
|
||||
host: {
|
||||
architecture: 'x86_64',
|
||||
hostname: 'zeek-newyork-sha-aa8df15',
|
||||
id: '3729d06ce9964aa98549f41cbd99334d',
|
||||
ip: ['157.230.208.30', '10.10.0.6', 'fe80::24ce:f7ff:fede:a571'],
|
||||
mac: ['26:ce:f7:de:a5:71'],
|
||||
name: 'zeek-newyork-sha-aa8df15',
|
||||
os: {
|
||||
codename: 'cosmic',
|
||||
family: 'debian',
|
||||
kernel: '4.18.0-10-generic',
|
||||
name: 'Ubuntu',
|
||||
platform: 'ubuntu',
|
||||
version: '18.10 (Cosmic Cuttlefish)',
|
||||
},
|
||||
},
|
||||
message:
|
||||
'Login by user root (UID: 0) on pts/0 (PID: 20638) from 8.42.77.171 (IP: 8.42.77.171)',
|
||||
process: { pid: 20638 },
|
||||
service: { type: 'system' },
|
||||
source: { ip: '8.42.77.171' },
|
||||
user: { id: 0, name: 'root', terminal: 'pts/0' },
|
||||
'event.action': 'user_login',
|
||||
'event.category': 'authentication',
|
||||
'event.dataset': 'login',
|
||||
'event.kind': 'signal',
|
||||
'event.module': 'system',
|
||||
'event.origin': '/var/log/wtmp',
|
||||
'event.outcome': 'success',
|
||||
'event.type': 'authentication_success',
|
||||
'kibana.alert.original_time': '2019-02-19T20:42:08.230Z',
|
||||
'kibana.alert.ancestors': [
|
||||
{
|
||||
id: 'x07wJ2oB9v5HJNSHhyxi',
|
||||
type: 'event',
|
||||
index: 'auditbeat-8.0.0-2019.02.19-000001',
|
||||
depth: 0,
|
||||
},
|
||||
],
|
||||
'kibana.alert.status': 'active',
|
||||
'kibana.alert.workflow_status': 'open',
|
||||
'kibana.alert.depth': 1,
|
||||
'kibana.alert.reason':
|
||||
'authentication event by root on zeek-newyork-sha-aa8df15 created high alert Query with a rule id.',
|
||||
'kibana.alert.severity': 'high',
|
||||
'kibana.alert.risk_score': 55,
|
||||
'kibana.alert.rule.parameters': {
|
||||
description: 'Detecting root and admin users',
|
||||
risk_score: 55,
|
||||
severity: 'high',
|
||||
author: [],
|
||||
false_positives: [],
|
||||
from: '2019-02-19T20:42:00.000Z',
|
||||
rule_id: 'rule-1',
|
||||
max_signals: 100,
|
||||
risk_score_mapping: [],
|
||||
severity_mapping: [],
|
||||
threat: [],
|
||||
to: 'now',
|
||||
references: [],
|
||||
version: 1,
|
||||
exceptions_list: [],
|
||||
immutable: false,
|
||||
related_integrations: [],
|
||||
required_fields: [],
|
||||
setup: '',
|
||||
type: 'new_terms',
|
||||
query: '*',
|
||||
new_terms_fields: ['host.name'],
|
||||
history_window_start: '2019-01-19T20:42:00.000Z',
|
||||
index: ['auditbeat-*'],
|
||||
language: 'kuery',
|
||||
},
|
||||
'kibana.alert.rule.actions': [],
|
||||
'kibana.alert.rule.author': [],
|
||||
'kibana.alert.rule.created_by': 'elastic',
|
||||
'kibana.alert.rule.description': 'Detecting root and admin users',
|
||||
'kibana.alert.rule.enabled': true,
|
||||
'kibana.alert.rule.exceptions_list': [],
|
||||
'kibana.alert.rule.false_positives': [],
|
||||
'kibana.alert.rule.from': '2019-02-19T20:42:00.000Z',
|
||||
'kibana.alert.rule.immutable': false,
|
||||
'kibana.alert.rule.indices': ['auditbeat-*'],
|
||||
'kibana.alert.rule.interval': '5m',
|
||||
'kibana.alert.rule.max_signals': 100,
|
||||
'kibana.alert.rule.references': [],
|
||||
'kibana.alert.rule.risk_score_mapping': [],
|
||||
'kibana.alert.rule.rule_id': 'rule-1',
|
||||
'kibana.alert.rule.severity_mapping': [],
|
||||
'kibana.alert.rule.threat': [],
|
||||
'kibana.alert.rule.to': 'now',
|
||||
'kibana.alert.rule.type': 'new_terms',
|
||||
'kibana.alert.rule.updated_by': 'elastic',
|
||||
'kibana.alert.rule.version': 1,
|
||||
'kibana.alert.rule.risk_score': 55,
|
||||
'kibana.alert.rule.severity': 'high',
|
||||
'kibana.alert.original_event.action': 'user_login',
|
||||
'kibana.alert.original_event.category': 'authentication',
|
||||
'kibana.alert.original_event.dataset': 'login',
|
||||
'kibana.alert.original_event.kind': 'event',
|
||||
'kibana.alert.original_event.module': 'system',
|
||||
'kibana.alert.original_event.origin': '/var/log/wtmp',
|
||||
'kibana.alert.original_event.outcome': 'success',
|
||||
'kibana.alert.original_event.type': 'authentication_success',
|
||||
});
|
||||
});
|
||||
|
||||
it('should generate 3 alerts when 1 document has 3 new values', async () => {
|
||||
const rule: NewTermsCreateSchema = {
|
||||
...getCreateNewTermsRulesSchemaMock('rule-1', true),
|
||||
new_terms_fields: ['host.ip'],
|
||||
from: '2019-02-19T20:42:00.000Z',
|
||||
history_window_start: '2019-01-19T20:42:00.000Z',
|
||||
};
|
||||
|
||||
const createdRule = await createRule(supertest, log, rule);
|
||||
|
||||
await waitForRuleSuccessOrStatus(
|
||||
supertest,
|
||||
log,
|
||||
createdRule.id,
|
||||
RuleExecutionStatus.succeeded
|
||||
);
|
||||
|
||||
const signalsOpen = await getOpenSignals(supertest, log, es, createdRule);
|
||||
expect(signalsOpen.hits.hits.length).eql(3);
|
||||
const signalsOrderedByHostIp = orderBy(
|
||||
signalsOpen.hits.hits,
|
||||
'_source.kibana.alert.new_terms',
|
||||
'asc'
|
||||
);
|
||||
expect(signalsOrderedByHostIp[0]._source?.['kibana.alert.new_terms']).eql(['10.10.0.6']);
|
||||
expect(signalsOrderedByHostIp[1]._source?.['kibana.alert.new_terms']).eql(['157.230.208.30']);
|
||||
expect(signalsOrderedByHostIp[2]._source?.['kibana.alert.new_terms']).eql([
|
||||
'fe80::24ce:f7ff:fede:a571',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should generate alerts for every term when history window is small', async () => {
|
||||
const rule: NewTermsCreateSchema = {
|
||||
...getCreateNewTermsRulesSchemaMock('rule-1', true),
|
||||
new_terms_fields: ['host.name'],
|
||||
from: '2019-02-19T20:42:00.000Z',
|
||||
// Set the history_window_start equal to 'from' so we should alert on all terms in the time range
|
||||
history_window_start: '2019-02-19T20:42:00.000Z',
|
||||
};
|
||||
|
||||
const createdRule = await createRule(supertest, log, rule);
|
||||
|
||||
await waitForRuleSuccessOrStatus(
|
||||
supertest,
|
||||
log,
|
||||
createdRule.id,
|
||||
RuleExecutionStatus.succeeded
|
||||
);
|
||||
|
||||
const signalsOpen = await getOpenSignals(supertest, log, es, createdRule);
|
||||
expect(signalsOpen.hits.hits.length).eql(5);
|
||||
const hostNames = signalsOpen.hits.hits
|
||||
.map((signal) => signal._source?.['kibana.alert.new_terms'])
|
||||
.sort();
|
||||
expect(hostNames[0]).eql(['suricata-sensor-amsterdam']);
|
||||
expect(hostNames[1]).eql(['suricata-sensor-san-francisco']);
|
||||
expect(hostNames[2]).eql(['zeek-newyork-sha-aa8df15']);
|
||||
expect(hostNames[3]).eql(['zeek-sensor-amsterdam']);
|
||||
expect(hostNames[4]).eql(['zeek-sensor-san-francisco']);
|
||||
});
|
||||
|
||||
describe('timestamp override and fallback', () => {
|
||||
before(async () => {
|
||||
await esArchiver.load(
|
||||
'x-pack/test/functional/es_archives/security_solution/timestamp_fallback'
|
||||
);
|
||||
await esArchiver.load(
|
||||
'x-pack/test/functional/es_archives/security_solution/timestamp_override_3'
|
||||
);
|
||||
});
|
||||
after(async () => {
|
||||
await esArchiver.unload(
|
||||
'x-pack/test/functional/es_archives/security_solution/timestamp_fallback'
|
||||
);
|
||||
await esArchiver.unload(
|
||||
'x-pack/test/functional/es_archives/security_solution/timestamp_override_3'
|
||||
);
|
||||
});
|
||||
|
||||
it('should generate the correct alerts', async () => {
|
||||
const rule: NewTermsCreateSchema = {
|
||||
...getCreateNewTermsRulesSchemaMock('rule-1', true),
|
||||
// myfakeindex-3 does not have event.ingested mapped so we can test if the runtime field
|
||||
// 'kibana.combined_timestamp' handles unmapped fields properly
|
||||
index: ['timestamp-fallback-test', 'myfakeindex-3'],
|
||||
new_terms_fields: ['host.name'],
|
||||
from: '2020-12-16T16:00:00.000Z',
|
||||
// Set the history_window_start equal to 'from' so we should alert on all terms in the time range
|
||||
history_window_start: '2020-12-16T16:00:00.000Z',
|
||||
timestamp_override: 'event.ingested',
|
||||
};
|
||||
|
||||
const createdRule = await createRule(supertest, log, rule);
|
||||
|
||||
await waitForSignalsToBePresent(supertest, log, 2, [createdRule.id]);
|
||||
|
||||
const signalsOpen = await getSignalsByIds(supertest, log, [createdRule.id]);
|
||||
expect(signalsOpen.hits.hits.length).eql(2);
|
||||
const hostNames = signalsOpen.hits.hits
|
||||
.map((signal) => signal._source?.['kibana.alert.new_terms'])
|
||||
.sort();
|
||||
expect(hostNames[0]).eql(['host-3']);
|
||||
expect(hostNames[1]).eql(['host-4']);
|
||||
});
|
||||
});
|
||||
|
||||
it('should apply exceptions', async () => {
|
||||
const rule: NewTermsCreateSchema = {
|
||||
...getCreateNewTermsRulesSchemaMock('rule-1', true),
|
||||
new_terms_fields: ['host.name'],
|
||||
from: '2019-02-19T20:42:00.000Z',
|
||||
// Set the history_window_start equal to 'from' so we should alert on all terms in the time range
|
||||
history_window_start: '2019-02-19T20:42:00.000Z',
|
||||
};
|
||||
const createdRule = await createRuleWithExceptionEntries(supertest, log, rule, [
|
||||
[
|
||||
{
|
||||
field: 'host.name',
|
||||
operator: 'included',
|
||||
type: 'match',
|
||||
value: 'zeek-sensor-san-francisco',
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
await waitForRuleSuccessOrStatus(
|
||||
supertest,
|
||||
log,
|
||||
createdRule.id,
|
||||
RuleExecutionStatus.succeeded
|
||||
);
|
||||
|
||||
const signalsOpen = await getOpenSignals(supertest, log, es, createdRule);
|
||||
expect(signalsOpen.hits.hits.length).eql(4);
|
||||
const hostNames = signalsOpen.hits.hits
|
||||
.map((signal) => signal._source?.['kibana.alert.new_terms'])
|
||||
.sort();
|
||||
expect(hostNames[0]).eql(['suricata-sensor-amsterdam']);
|
||||
expect(hostNames[1]).eql(['suricata-sensor-san-francisco']);
|
||||
expect(hostNames[2]).eql(['zeek-newyork-sha-aa8df15']);
|
||||
expect(hostNames[3]).eql(['zeek-sensor-amsterdam']);
|
||||
});
|
||||
|
||||
it('should work for max signals > 100', async () => {
|
||||
const maxSignals = 200;
|
||||
const rule: NewTermsCreateSchema = {
|
||||
...getCreateNewTermsRulesSchemaMock('rule-1', true),
|
||||
new_terms_fields: ['process.pid'],
|
||||
from: '2018-02-19T20:42:00.000Z',
|
||||
// Set the history_window_start equal to 'from' so we should alert on all terms in the time range
|
||||
history_window_start: '2018-02-19T20:42:00.000Z',
|
||||
max_signals: maxSignals,
|
||||
};
|
||||
|
||||
const createdRule = await createRule(supertest, log, rule);
|
||||
|
||||
await waitForRuleSuccessOrStatus(
|
||||
supertest,
|
||||
log,
|
||||
createdRule.id,
|
||||
RuleExecutionStatus.succeeded
|
||||
);
|
||||
|
||||
const signalsOpen = await getOpenSignals(
|
||||
supertest,
|
||||
log,
|
||||
es,
|
||||
createdRule,
|
||||
RuleExecutionStatus.succeeded,
|
||||
maxSignals
|
||||
);
|
||||
expect(signalsOpen.hits.hits.length).eql(maxSignals);
|
||||
const processPids = signalsOpen.hits.hits
|
||||
.map((signal) => signal._source?.['kibana.alert.new_terms'])
|
||||
.sort();
|
||||
expect(processPids[0]).eql([1]);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
|
@ -24,6 +24,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
|
|||
loadTestFile(require.resolve('./preview_rules'));
|
||||
loadTestFile(require.resolve('./create_rules_bulk'));
|
||||
loadTestFile(require.resolve('./create_ml'));
|
||||
loadTestFile(require.resolve('./create_new_terms'));
|
||||
loadTestFile(require.resolve('./create_threat_matching'));
|
||||
loadTestFile(require.resolve('./delete_rules'));
|
||||
loadTestFile(require.resolve('./delete_rules_bulk'));
|
||||
|
|
|
|||
|
|
@ -20,12 +20,13 @@ export const getOpenSignals = async (
|
|||
log: ToolingLog,
|
||||
es: Client,
|
||||
rule: FullResponseSchema,
|
||||
status: RuleExecutionStatus = RuleExecutionStatus.succeeded
|
||||
status: RuleExecutionStatus = RuleExecutionStatus.succeeded,
|
||||
size?: number
|
||||
) => {
|
||||
await waitForRuleSuccessOrStatus(supertest, log, rule.id, status);
|
||||
// Critically important that we wait for rule success AND refresh the write index in that order before we
|
||||
// assert that no signals were created. Otherwise, signals could be written but not available to query yet
|
||||
// when we search, causing tests that check that signals are NOT created to pass when they should fail.
|
||||
await refreshIndex(es, '.alerts-security.alerts-default*');
|
||||
return getSignalsByIds(supertest, log, [rule.id]);
|
||||
return getSignalsByIds(supertest, log, [rule.id], size);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ export const removeServerGeneratedProperties = (
|
|||
created_at,
|
||||
updated_at,
|
||||
execution_summary,
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
...removedProperties
|
||||
} = rule;
|
||||
return removedProperties;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
{
|
||||
"type": "doc",
|
||||
"value": {
|
||||
"index": "timestamp-fallback-test",
|
||||
"source": {
|
||||
"message": "hello world",
|
||||
"@timestamp": "2020-12-16T15:15:18.570Z",
|
||||
"host": {
|
||||
"name": "host-1"
|
||||
}
|
||||
},
|
||||
"type": "_doc"
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
"type": "doc",
|
||||
"value": {
|
||||
"index": "timestamp-fallback-test",
|
||||
"source": {
|
||||
"message": "hello world",
|
||||
"@timestamp": "2020-12-16T15:15:18.570Z",
|
||||
"event": {
|
||||
"ingested": "2020-12-16T15:16:18.570Z"
|
||||
},
|
||||
"host": {
|
||||
"name": "host-2"
|
||||
}
|
||||
},
|
||||
"type": "_doc"
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
"type": "doc",
|
||||
"value": {
|
||||
"index": "timestamp-fallback-test",
|
||||
"source": {
|
||||
"message": "hello world",
|
||||
"@timestamp": "2020-12-16T16:15:18.570Z",
|
||||
"host": {
|
||||
"name": "host-3"
|
||||
}
|
||||
},
|
||||
"type": "_doc"
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
"type": "doc",
|
||||
"value": {
|
||||
"index": "timestamp-fallback-test",
|
||||
"source": {
|
||||
"message": "hello world",
|
||||
"@timestamp": "2020-12-16T16:15:18.570Z",
|
||||
"event": {
|
||||
"ingested": "2020-12-16T16:16:18.570Z"
|
||||
},
|
||||
"host": {
|
||||
"name": "host-4"
|
||||
}
|
||||
},
|
||||
"type": "_doc"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"type": "index",
|
||||
"value": {
|
||||
"index": "timestamp-fallback-test",
|
||||
"mappings": {
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"message": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"event": {
|
||||
"properties": {
|
||||
"ingested": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
},
|
||||
"host": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue