github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution] Extract common shared security params into object (#212694)

Browse source 
## Summary

Replaces many long lists of parameters with `sharedParams` - a list of
commonly used inputs from the shared security rule wrapper.
`sharedParams` should be treated as immutable throughout the entire rule
execution to eliminate confusion about which params are specific to
certain code paths and which ones are simply passed through from the
shared wrapper.

More refactoring will follow to further reduce the pass through param
passing. I attempted to limit the scope of changes in this PR by
destructuring `sharedParams` into the expected param format for some
functions. This also sets us up to remove function passing of
`wrapHits`, `bulkCreate`, etc, which would have required passing more of
these individual shared params deep into rule execution logic.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Marshall Main 2025-03-07 12:30:35 -05:00 • committed by GitHub
parent e7c71937d5
commit a78f9c2efe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
53 changed files with 740 additions and 1775 deletions
Download patch file Download diff file Expand all files Collapse all files

50
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/shared_params.ts Normal file
View file

@ -0,0 +1,50 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import dateMath from '@kbn/datemath';
import { DEFAULT_INDEX_PATTERN } from '../../../../../common/constants';
import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
import type { QueryRuleParams, RuleParams } from '../../rule_schema';
import type { SecuritySharedParams } from '../types';
import { getListClientMock } from '@kbn/lists-plugin/server/services/lists/list_client.mock';
import { createRuleDataClientMock } from '@kbn/rule-registry-plugin/server/rule_data_client/rule_data_client.mock';
import { getCompleteRuleMock } from '../../rule_schema/mocks';
export const getSharedParamsMock = <T extends RuleParams = QueryRuleParams>({
ruleParams,
rewrites,
}: {
ruleParams: T;
rewrites?: Partial<SecuritySharedParams<T>>;
}): SecuritySharedParams<T> => ({
ruleExecutionLogger: ruleExecutionLogMock.forExecutors.create(),
completeRule: getCompleteRuleMock(ruleParams),
mergeStrategy: 'allFields',
spaceId: 'default',
inputIndex: DEFAULT_INDEX_PATTERN,
alertTimestampOverride: undefined,
publicBaseUrl: 'http://testkibanabaseurl.com',
intendedTimestamp: undefined,
primaryTimestamp: '@timestamp',
listClient: getListClientMock(),
tuple: {
from: dateMath.parse(ruleParams.from) as moment.Moment,
to: dateMath.parse(ruleParams.to) as moment.Moment,
maxSignals: ruleParams.maxSignals,
},
searchAfterSize: 100,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
ruleDataClient: createRuleDataClientMock(),
runtimeMappings: undefined,
aggregatableTimestampField: '@timestamp',
unprocessedExceptions: [],
exceptionFilter: undefined,
alertWithSuppression: jest.fn(),
refreshOnIndexingAlerts: false,
...rewrites,
});

18
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts
View file

@ -36,7 +36,7 @@ import { getNotificationResultsLink } from '../rule_actions_legacy';
// eslint-disable-next-line no-restricted-imports
import { formatAlertForNotificationActions } from '../rule_actions_legacy/logic/notifications/schedule_notification_actions';
import { createResultObject } from './utils';
import { bulkCreateFactory, wrapHitsFactory, wrapSequencesFactory } from './factories';
import { bulkCreateFactory, wrapHitsFactory } from './factories';
import { RuleExecutionStatusEnum } from '../../../../common/api/detection_engine/rule_monitoring';
import { truncateList } from '../rule_monitoring';
import aadFieldConversion from '../routes/index/signal_aad_mapping.json';
@ -387,18 +387,6 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
intendedTimestamp,
});
const wrapSequences = wrapSequencesFactory({
ruleExecutionLogger,
ignoreFields: [...ignoreFields, ...legacySignalFields],
mergeStrategy,
completeRule,
spaceId,
publicBaseUrl,
indicesToQuery: inputIndex,
alertTimestampOverride,
intendedTimestamp,
});
const { filter: exceptionFilter, unprocessedExceptions } = await buildExceptionFilter({
startedAt,
alias: null,
@ -414,7 +402,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
...options,
services,
state: runState,
runOpts: {
sharedParams: {
completeRule,
inputIndex,
exceptionFilter,
@ -427,7 +415,6 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
tuple,
bulkCreate,
wrapHits,
wrapSequences,
listClient,
ruleDataClient,
mergeStrategy,
@ -441,6 +428,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
publicBaseUrl,
experimentalFeatures,
intendedTimestamp,
spaceId,
},
});

22
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts
View file

@ -14,9 +14,7 @@ import {
objectPairIntersection,
} from './build_alert_group_from_sequence';
import { SERVER_APP_ID } from '../../../../../common/constants';
import { getCompleteRuleMock, getQueryRuleParams } from '../../rule_schema/mocks';
import type { QueryRuleParams } from '../../rule_schema';
import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
import { getQueryRuleParams } from '../../rule_schema/mocks';
import {
ALERT_ANCESTORS,
ALERT_DEPTH,
@ -24,11 +22,12 @@ import {
ALERT_GROUP_ID,
} from '../../../../../common/field_maps/field_names';
import { buildReasonMessageForEqlAlert } from '../utils/reason_formatters';
import { getSharedParamsMock } from '../__mocks__/shared_params';
const SPACE_ID = 'space';
const PUBLIC_BASE_URL = 'http://testkibanabaseurl.com';
const ruleExecutionLoggerMock = ruleExecutionLogMock.forExecutors.create();
const sharedParams = getSharedParamsMock({
ruleParams: getQueryRuleParams(),
rewrites: { spaceId: 'space' },
});
describe('buildAlert', () => {
beforeEach(() => {
@ -36,7 +35,6 @@ describe('buildAlert', () => {
});
test('it builds an alert as expected without original_event if event does not exist', () => {
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const eqlSequence = {
join_keys: [],
events: [
@ -45,15 +43,9 @@ describe('buildAlert', () => {
],
};
const { shellAlert, buildingBlocks } = buildAlertGroupFromSequence({
ruleExecutionLogger: ruleExecutionLoggerMock,
sharedParams,
sequence: eqlSequence,
completeRule,
mergeStrategy: 'allFields',
spaceId: SPACE_ID,
buildReasonMessage: buildReasonMessageForEqlAlert,
indicesToQuery: completeRule.ruleParams.index as string[],
alertTimestampOverride: undefined,
publicBaseUrl: PUBLIC_BASE_URL,
});
expect(buildingBlocks.length).toEqual(2);
expect(buildingBlocks[0]).toEqual(

33
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts
View file

@ -17,15 +17,13 @@ import { intersection as lodashIntersection, isArray } from 'lodash';
import { getAlertDetailsUrl } from '../../../../../common/utils/alert_detail_path';
import { DEFAULT_ALERTS_INDEX } from '../../../../../common/constants';
import type { ConfigType } from '../../../../config';
import type { Ancestor, SignalSource, SignalSourceHit } from '../types';
import type { Ancestor, SecuritySharedParams, SignalSource, SignalSourceHit } from '../types';
import { buildAlertFields, buildAncestors, generateAlertId } from '../factories/utils/build_alert';
import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert';
import type { EqlSequence } from '../../../../../common/detection_engine/types';
import { generateBuildingBlockIds } from '../factories/utils/generate_building_block_ids';
import type { BuildReasonMessage } from '../utils/reason_formatters';
import type { CompleteRule, RuleParams } from '../../rule_schema';
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
import {
ALERT_BUILDING_BLOCK_TYPE,
ALERT_GROUP_ID,
@ -48,17 +46,10 @@ export interface ExtraFieldsForShellAlert {
}
export interface BuildAlertGroupFromSequence {
ruleExecutionLogger: IRuleExecutionLogForExecutors;
sharedParams: SecuritySharedParams;
sequence: EqlSequence<SignalSource>;
completeRule: CompleteRule<RuleParams>;
mergeStrategy: ConfigType['alertMergeStrategy'];
spaceId: string | null | undefined;
buildReasonMessage: BuildReasonMessage;
indicesToQuery: string[];
alertTimestampOverride: Date | undefined;
applyOverrides?: boolean;
publicBaseUrl?: string;
intendedTimestamp?: Date;
}
// eql shell alerts can have a subAlerts property
@ -75,20 +66,23 @@ export type WrappedEqlShellOptionalSubAlertsType = WrappedFieldsLatest<EqlShellF
* @param completeRule object representing the rule that found the sequence
*/
export const buildAlertGroupFromSequence = ({
ruleExecutionLogger,
sharedParams,
sequence,
completeRule,
mergeStrategy,
spaceId,
buildReasonMessage,
indicesToQuery,
alertTimestampOverride,
publicBaseUrl,
intendedTimestamp,
}: BuildAlertGroupFromSequence): {
shellAlert: WrappedFieldsLatest<EqlShellFieldsLatest> | undefined;
buildingBlocks: Array<WrappedFieldsLatest<EqlBuildingBlockFieldsLatest>>;
} => {
const {
alertTimestampOverride,
intendedTimestamp,
mergeStrategy,
completeRule,
spaceId,
inputIndex: indicesToQuery,
ruleExecutionLogger,
publicBaseUrl,
} = sharedParams;
const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event));
if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) {
return { shellAlert: undefined, buildingBlocks: [] };
@ -106,6 +100,7 @@ export const buildAlertGroupFromSequence = ({
completeRule,
doc: event,
mergeStrategy,
// TODO: respect ignore fields
ignoreFields: {},
ignoreFieldsRegexes: [],
applyOverrides: false,

62
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_eql_search_request.test.ts
View file

@ -9,6 +9,8 @@ import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas
import { getListClientMock } from '@kbn/lists-plugin/server/services/lists/list_client.mock';
import { buildExceptionFilter } from '@kbn/lists-plugin/server/services/exception_lists';
import { buildEqlSearchRequest } from './build_eql_search_request';
import { getEqlRuleParams } from '../../rule_schema/mocks';
import { getSharedParamsMock } from '../__mocks__/shared_params';
const emptyFilter = {
bool: {
@ -18,21 +20,23 @@ const emptyFilter = {
must_not: [],
},
};
const index = ['testindex1', 'testindex2'];
const ruleParams = getEqlRuleParams({ index });
const sharedParams = getSharedParamsMock({
ruleParams,
rewrites: { inputIndex: ['testindex1', 'testindex2'] },
});
describe('buildEqlSearchRequest', () => {
test('should build a basic request with time range', () => {
const request = buildEqlSearchRequest({
sharedParams,
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
eventCategoryOverride: undefined,
exceptionFilter: undefined,
});
expect(request).toEqual({
allow_no_indices: true,
@ -72,18 +76,18 @@ describe('buildEqlSearchRequest', () => {
test('should build a request with timestamp and event category overrides', () => {
const request = buildEqlSearchRequest({
sharedParams: {
...sharedParams,
primaryTimestamp: 'event.ingested',
secondaryTimestamp: '@timestamp',
},
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters: undefined,
primaryTimestamp: 'event.ingested',
secondaryTimestamp: '@timestamp',
runtimeMappings: undefined,
eventCategoryOverride: 'event.other_category',
timestampField: undefined,
exceptionFilter: undefined,
});
expect(request).toEqual({
allow_no_indices: true,
@ -159,18 +163,17 @@ describe('buildEqlSearchRequest', () => {
test('should build a request without @timestamp fallback if secondaryTimestamp is not specified', () => {
const request = buildEqlSearchRequest({
sharedParams: {
...sharedParams,
primaryTimestamp: 'event.ingested',
},
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters: undefined,
primaryTimestamp: 'event.ingested',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
eventCategoryOverride: 'event.other_category',
timestampField: undefined,
exceptionFilter: undefined,
});
expect(request).toEqual({
allow_no_indices: true,
@ -219,17 +222,16 @@ describe('buildEqlSearchRequest', () => {
startedAt: new Date(),
});
const request = buildEqlSearchRequest({
sharedParams: {
...sharedParams,
exceptionFilter: filter,
},
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
eventCategoryOverride: undefined,
exceptionFilter: filter,
});
expect(request).toMatchInlineSnapshot(`
Object {
@ -360,16 +362,12 @@ describe('buildEqlSearchRequest', () => {
},
];
const request = buildEqlSearchRequest({
sharedParams,
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
exceptionFilter: undefined,
});
expect(request).toEqual({
allow_no_indices: true,
@ -427,36 +425,28 @@ describe('buildEqlSearchRequest', () => {
describe('handles the tiebreaker field', () => {
test('should pass a tiebreaker field with a valid value', async () => {
const request = buildEqlSearchRequest({
sharedParams,
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
tiebreakerField: 'host.name',
eventCategoryOverride: undefined,
exceptionFilter: undefined,
});
expect(request?.tiebreaker_field).toEqual(`host.name`);
});
test('should not pass a tiebreaker field with a valid value', async () => {
test('should not pass a tiebreaker field with an invalid value', async () => {
const request = buildEqlSearchRequest({
sharedParams,
query: 'process where true',
index: ['testindex1', 'testindex2'],
from: 'now-5m',
to: 'now',
size: 100,
filters: undefined,
primaryTimestamp: '@timestamp',
secondaryTimestamp: undefined,
runtimeMappings: undefined,
tiebreakerField: '',
eventCategoryOverride: undefined,
exceptionFilter: undefined,
});
expect(request?.tiebreaker_field).toEqual(undefined);
});

27
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_eql_search_request.ts
View file

@ -6,46 +6,43 @@
*/
import type { estypes } from '@elastic/elasticsearch';
import type { Filter } from '@kbn/es-query';
import { isEmpty } from 'lodash/fp';
import type {
RuleFilterArray,
TimestampOverride,
} from '../../../../../common/api/detection_engine/model/rule_schema';
import type { RuleFilterArray } from '../../../../../common/api/detection_engine/model/rule_schema';
import { buildTimeRangeFilter } from '../utils/build_events_query';
import { getQueryFilter } from '../utils/get_query_filter';
import type { SecuritySharedParams } from '../types';
import type { EqlRuleParams } from '../../rule_schema';
interface BuildEqlSearchRequestParams {
sharedParams: SecuritySharedParams<EqlRuleParams>;
query: string;
index: string[];
from: string;
to: string;
size: number;
filters: RuleFilterArray | undefined;
primaryTimestamp: TimestampOverride;
secondaryTimestamp: TimestampOverride | undefined;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
eventCategoryOverride?: string;
timestampField?: string;
tiebreakerField?: string;
exceptionFilter: Filter | undefined;
}
export const buildEqlSearchRequest = ({
sharedParams,
query,
index,
from,
to,
size,
filters,
primaryTimestamp,
secondaryTimestamp,
runtimeMappings,
eventCategoryOverride,
timestampField,
tiebreakerField,
exceptionFilter,
}: BuildEqlSearchRequestParams): estypes.EqlSearchRequest => {
const {
inputIndex: index,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
runtimeMappings,
} = sharedParams;
const timestamps = secondaryTimestamp
? [primaryTimestamp, secondaryTimestamp]
: [primaryTimestamp];

73
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts
View file

@ -14,14 +14,13 @@ import { eqlExecutor } from './eql';
import type { CreateRuleOptions, SecurityAlertType, SignalSourceHit } from '../types';
import { validateIndexPatterns } from '../utils';
import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active';
import type { SharedParams } from '../utils/utils';
import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts';
import type { BuildReasonMessage } from '../utils/reason_formatters';
export const createEqlAlertType = (
createOptions: CreateRuleOptions
): SecurityAlertType<EqlRuleParams, {}, {}, 'default'> => {
const { experimentalFeatures, version, licensing, scheduleNotificationResponseActionsService } =
const { experimentalFeatures, licensing, scheduleNotificationResponseActionsService } =
createOptions;
return {
id: EQL_RULE_TYPE_ID,
@ -62,87 +61,27 @@ export const createEqlAlertType = (
category: DEFAULT_APP_CATEGORIES.security.id,
producer: SERVER_APP_ID,
async executor(execOptions) {
const {
runOpts: {
completeRule,
tuple,
inputIndex,
runtimeMappings,
ruleExecutionLogger,
bulkCreate,
wrapHits,
wrapSequences,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
mergeStrategy,
alertTimestampOverride,
publicBaseUrl,
alertWithSuppression,
intendedTimestamp,
},
services,
state,
spaceId,
} = execOptions;
const { sharedParams, services, state } = execOptions;
const isAlertSuppressionActive = await getIsAlertSuppressionActive({
alertSuppression: completeRule.ruleParams.alertSuppression,
alertSuppression: sharedParams.completeRule.ruleParams.alertSuppression,
licensing,
});
const sharedParams: SharedParams = {
spaceId,
completeRule,
mergeStrategy,
indicesToQuery: inputIndex,
alertTimestampOverride,
ruleExecutionLogger,
publicBaseUrl,
primaryTimestamp,
secondaryTimestamp,
intendedTimestamp,
};
const wrapSuppressedHits = (
events: SignalSourceHit[],
buildReasonMessage: BuildReasonMessage
) =>
wrapSuppressedAlerts({
events,
spaceId,
completeRule,
mergeStrategy,
indicesToQuery: inputIndex,
buildReasonMessage,
alertTimestampOverride,
ruleExecutionLogger,
publicBaseUrl,
primaryTimestamp,
secondaryTimestamp,
intendedTimestamp,
sharedParams,
});
const { result, loggedRequests } = await eqlExecutor({
completeRule,
tuple,
inputIndex,
runtimeMappings,
ruleExecutionLogger,
services,
version,
bulkCreate,
wrapHits,
wrapSequences,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
wrapSuppressedHits,
sharedParams,
alertTimestampOverride,
alertWithSuppression,
services,
wrapSuppressedHits,
isAlertSuppressionActive,
experimentalFeatures,
state,

106
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts
View file

@ -5,20 +5,16 @@
* 2.0.
*/
import dateMath from '@kbn/datemath';
import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks';
import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock';
import { DEFAULT_INDEX_PATTERN } from '../../../../../common/constants';
import type { ExperimentalFeatures } from '../../../../../common';
import { getIndexVersion } from '../../routes/index/get_index_version';
import { SIGNALS_TEMPLATE_VERSION } from '../../routes/index/get_signals_template';
import type { EqlRuleParams } from '../../rule_schema';
import { getCompleteRuleMock, getEqlRuleParams } from '../../rule_schema/mocks';
import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
import { getEqlRuleParams } from '../../rule_schema/mocks';
import { eqlExecutor } from './eql';
import { getDataTierFilter } from '../utils/get_data_tier_filter';
import type { SharedParams } from '../utils/utils';
import { getSharedParamsMock } from '../__mocks__/shared_params';
jest.mock('../../routes/index/get_index_version');
jest.mock('../utils/get_data_tier_filter', () => ({ getDataTierFilter: jest.fn() }));
@ -26,34 +22,13 @@ jest.mock('../utils/get_data_tier_filter', () => ({ getDataTierFilter: jest.fn()
const getDataTierFilterMock = getDataTierFilter as jest.Mock;
describe('eql_executor', () => {
const version = '8.0.0';
const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create();
let alertServices: RuleExecutorServicesMock;
(getIndexVersion as jest.Mock).mockReturnValue(SIGNALS_TEMPLATE_VERSION);
const params = getEqlRuleParams();
const eqlCompleteRule = getCompleteRuleMock<EqlRuleParams>(params);
const tuple = {
from: dateMath.parse(params.from)!,
to: dateMath.parse(params.to)!,
maxSignals: params.maxSignals,
};
const mockExperimentalFeatures = {} as ExperimentalFeatures;
const mockScheduleNotificationResponseActionsService = jest.fn();
const ruleExecutionLoggerMock = ruleExecutionLogMock.forExecutors.create();
const SPACE_ID = 'space';
const PUBLIC_BASE_URL = 'http://testkibanabaseurl.com';
const sharedParams: SharedParams = {
ruleExecutionLogger: ruleExecutionLoggerMock,
completeRule: eqlCompleteRule,
mergeStrategy: 'allFields',
spaceId: SPACE_ID,
indicesToQuery: eqlCompleteRule.ruleParams.index as string[],
alertTimestampOverride: undefined,
publicBaseUrl: PUBLIC_BASE_URL,
intendedTimestamp: undefined,
primaryTimestamp: new Date().toISOString(),
};
const sharedParams = getSharedParamsMock({ ruleParams: params });
beforeEach(() => {
jest.clearAllMocks();
@ -71,23 +46,12 @@ describe('eql_executor', () => {
describe('warning scenarios', () => {
it('warns when exception list for eql rule contains value list exceptions', async () => {
const { result } = await eqlExecutor({
inputIndex: DEFAULT_INDEX_PATTERN,
runtimeMappings: {},
completeRule: eqlCompleteRule,
tuple,
ruleExecutionLogger,
sharedParams: getSharedParamsMock({
ruleParams: params,
rewrites: { unprocessedExceptions: [getExceptionListItemSchemaMock()] },
}),
services: alertServices,
version,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
wrapSequences: jest.fn(),
primaryTimestamp: '@timestamp',
exceptionFilter: undefined,
unprocessedExceptions: [getExceptionListItemSchemaMock()],
wrapSuppressedHits: jest.fn(),
sharedParams,
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
isAlertSuppressionActive: false,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService:
@ -108,23 +72,9 @@ describe('eql_executor', () => {
'verification_exception\n\tRoot causes:\n\t\tverification_exception: Found 1 problem\nline 1:1: Unknown column [event.category]',
});
const { result } = await eqlExecutor({
inputIndex: DEFAULT_INDEX_PATTERN,
runtimeMappings: {},
completeRule: eqlCompleteRule,
tuple,
ruleExecutionLogger,
services: alertServices,
version,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
wrapSequences: jest.fn(),
primaryTimestamp: '@timestamp',
exceptionFilter: undefined,
unprocessedExceptions: [],
wrapSuppressedHits: jest.fn(),
sharedParams,
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
services: alertServices,
wrapSuppressedHits: jest.fn(),
isAlertSuppressionActive: true,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduleNotificationResponseActionsService,
@ -134,23 +84,9 @@ describe('eql_executor', () => {
it('should handle scheduleNotificationResponseActionsService call', async () => {
const { result } = await eqlExecutor({
inputIndex: DEFAULT_INDEX_PATTERN,
runtimeMappings: {},
completeRule: eqlCompleteRule,
tuple,
ruleExecutionLogger,
services: alertServices,
version,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
wrapSequences: jest.fn(),
primaryTimestamp: '@timestamp',
exceptionFilter: undefined,
unprocessedExceptions: [],
wrapSuppressedHits: jest.fn(),
sharedParams,
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
services: alertServices,
wrapSuppressedHits: jest.fn(),
isAlertSuppressionActive: false,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduleNotificationResponseActionsService,
@ -158,7 +94,7 @@ describe('eql_executor', () => {
expect(mockScheduleNotificationResponseActionsService).toBeCalledWith({
signals: result.createdSignals,
signalsCount: result.createdSignalsCount,
responseActions: eqlCompleteRule.ruleParams.responseActions,
responseActions: params.responseActions,
});
});
@ -175,23 +111,9 @@ describe('eql_executor', () => {
]);
await eqlExecutor({
inputIndex: DEFAULT_INDEX_PATTERN,
runtimeMappings: {},
completeRule: eqlCompleteRule,
tuple,
ruleExecutionLogger,
services: alertServices,
version,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
wrapSequences: jest.fn(),
primaryTimestamp: '@timestamp',
exceptionFilter: undefined,
unprocessedExceptions: [],
wrapSuppressedHits: jest.fn(),
sharedParams,
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
services: alertServices,
wrapSuppressedHits: jest.fn(),
isAlertSuppressionActive: true,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduleNotificationResponseActionsService,

79
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts
View file

@ -6,15 +6,11 @@
*/
import { performance } from 'perf_hooks';
import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import type {
AlertInstanceContext,
AlertInstanceState,
RuleExecutorServices,
} from '@kbn/alerting-plugin/server';
import type { estypes } from '@elastic/elasticsearch';
import type { Filter } from '@kbn/es-query';
import isEmpty from 'lodash/isEmpty';
import type { ShardFailure } from '@elastic/elasticsearch/lib/api/types';
@ -23,16 +19,12 @@ import { createEnrichEventsFunction } from '../utils/enrichments';
import type { ExperimentalFeatures } from '../../../../../common';
import type {
BulkCreate,
WrapHits,
WrapSequences,
RuleRangeTuple,
SearchAfterAndBulkCreateReturnType,
SignalSource,
CreateRuleOptions,
WrapSuppressedHits,
SecuritySharedParams,
} from '../types';
import type { SharedParams } from '../utils/utils';
import {
addToSearchAfterReturn,
createSearchAfterReturnType,
@ -42,13 +34,12 @@ import {
getSuppressionMaxSignalsWarning,
} from '../utils/utils';
import { buildReasonMessageForEqlAlert } from '../utils/reason_formatters';
import type { CompleteRule, EqlRuleParams } from '../../rule_schema';
import type { EqlRuleParams } from '../../rule_schema';
import { withSecuritySpan } from '../../../../utils/with_security_span';
import type {
BaseFieldsLatest,
WrappedFieldsLatest,
} from '../../../../../common/api/detection_engine/model/alerts';
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
import {
bulkCreateSuppressedAlertsInMemory,
bulkCreateSuppressedSequencesInMemory,
@ -60,26 +51,12 @@ import * as i18n from '../translations';
import { alertSuppressionTypeGuard } from '../utils/get_is_alert_suppression_active';
import { isEqlSequenceQuery } from '../../../../../common/detection_engine/utils';
import { logShardFailures } from '../utils/log_shard_failure';
import { wrapSequences } from './wrap_sequences';
interface EqlExecutorParams {
inputIndex: string[];
runtimeMappings: estypes.MappingRuntimeFields | undefined;
completeRule: CompleteRule<EqlRuleParams>;
tuple: RuleRangeTuple;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
sharedParams: SecuritySharedParams<EqlRuleParams>;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
version: string;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
sharedParams: SharedParams;
wrapSequences: WrapSequences;
primaryTimestamp: string;
secondaryTimestamp?: string;
exceptionFilter: Filter | undefined;
unprocessedExceptions: ExceptionListItemSchema[];
wrapSuppressedHits: WrapSuppressedHits;
alertTimestampOverride: Date | undefined;
alertWithSuppression: SuppressedAlertService;
isAlertSuppressionActive: boolean;
experimentalFeatures: ExperimentalFeatures;
state?: Record<string, unknown>;
@ -87,24 +64,9 @@ interface EqlExecutorParams {
}
export const eqlExecutor = async ({
inputIndex,
runtimeMappings,
completeRule,
tuple,
ruleExecutionLogger,
services,
version,
bulkCreate,
wrapHits,
wrapSequences,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
wrapSuppressedHits,
sharedParams,
alertTimestampOverride,
alertWithSuppression,
services,
wrapSuppressedHits,
isAlertSuppressionActive,
experimentalFeatures,
state,
@ -113,6 +75,7 @@ export const eqlExecutor = async ({
result: SearchAfterAndBulkCreateReturnType;
loggedRequests?: RulePreviewLoggedRequest[];
}> => {
const { completeRule, tuple, ruleExecutionLogger, bulkCreate, wrapHits } = sharedParams;
const ruleParams = completeRule.ruleParams;
const isLoggedRequestsEnabled = state?.isLoggedRequestsEnabled ?? false;
const loggedRequests: RulePreviewLoggedRequest[] = [];
@ -128,23 +91,19 @@ export const eqlExecutor = async ({
const isSequenceQuery = isEqlSequenceQuery(ruleParams.query);
const request = buildEqlSearchRequest({
sharedParams,
query: ruleParams.query,
index: inputIndex,
from: tuple.from.toISOString(),
to: tuple.to.toISOString(),
size: ruleParams.maxSignals,
filters: [...(ruleParams.filters || []), ...dataTiersFilters],
primaryTimestamp,
secondaryTimestamp,
runtimeMappings,
eventCategoryOverride: ruleParams.eventCategoryOverride,
timestampField: ruleParams.timestampField,
tiebreakerField: ruleParams.tiebreakerField,
exceptionFilter,
});
ruleExecutionLogger.debug(`EQL query request: ${JSON.stringify(request)}`);
const exceptionsWarning = getUnprocessedExceptionsWarnings(unprocessedExceptions);
const exceptionsWarning = getUnprocessedExceptionsWarnings(sharedParams.unprocessedExceptions);
if (exceptionsWarning) {
result.warningMessages.push(exceptionsWarning);
}
@ -187,18 +146,13 @@ export const eqlExecutor = async ({
if (events) {
if (isAlertSuppressionActive) {
await bulkCreateSuppressedAlertsInMemory({
sharedParams,
enrichedEvents: events,
toReturn: result,
wrapHits,
bulkCreate,
services,
buildReasonMessage: buildReasonMessageForEqlAlert,
ruleExecutionLogger,
tuple,
alertSuppression: completeRule.ruleParams.alertSuppression,
wrapSuppressedHits,
alertTimestampOverride,
alertWithSuppression,
experimentalFeatures,
});
} else {
@ -211,21 +165,20 @@ export const eqlExecutor = async ({
alertSuppressionTypeGuard(completeRule.ruleParams.alertSuppression)
) {
await bulkCreateSuppressedSequencesInMemory({
sharedParams,
sequences,
toReturn: result,
bulkCreate,
services,
buildReasonMessage: buildReasonMessageForEqlAlert,
ruleExecutionLogger,
tuple,
alertSuppression: completeRule.ruleParams.alertSuppression,
sharedParams,
alertTimestampOverride,
alertWithSuppression,
experimentalFeatures,
});
} else {
newSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert);
newSignals = wrapSequences({
sharedParams,
sequences,
buildReasonMessage: buildReasonMessageForEqlAlert,
});
}
} else {
throw new Error(

43
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences.ts Normal file
View file

@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { EqlSequence } from '../../../../../common/detection_engine/types';
import type { SecuritySharedParams, SignalSource } from '../types';
import { buildAlertGroupFromSequence } from './build_alert_group_from_sequence';
import type { EqlRuleParams } from '../../rule_schema';
import type {
EqlBuildingBlockFieldsLatest,
EqlShellFieldsLatest,
WrappedFieldsLatest,
} from '../../../../../common/api/detection_engine/model/alerts';
import type { BuildReasonMessage } from '../utils/reason_formatters';
export const wrapSequences = ({
sharedParams,
sequences,
buildReasonMessage,
}: {
sharedParams: SecuritySharedParams<EqlRuleParams>;
sequences: Array<EqlSequence<SignalSource>>;
buildReasonMessage: BuildReasonMessage;
}) =>
sequences.reduce<
Array<
WrappedFieldsLatest<EqlShellFieldsLatest> | WrappedFieldsLatest<EqlBuildingBlockFieldsLatest>
>
>((acc, sequence) => {
const { shellAlert, buildingBlocks } = buildAlertGroupFromSequence({
sharedParams,
sequence,
buildReasonMessage,
});
if (shellAlert) {
acc.push(shellAlert, ...buildingBlocks);
return acc;
}
return acc;
}, []);

65
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts
View file

@ -1,65 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { WrapSequences } from '../types';
import { buildAlertGroupFromSequence } from './build_alert_group_from_sequence';
import type { ConfigType } from '../../../../config';
import type { CompleteRule, RuleParams } from '../../rule_schema';
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
import type {
EqlBuildingBlockFieldsLatest,
EqlShellFieldsLatest,
WrappedFieldsLatest,
} from '../../../../../common/api/detection_engine/model/alerts';
export const wrapSequencesFactory =
({
ruleExecutionLogger,
completeRule,
ignoreFields,
mergeStrategy,
publicBaseUrl,
spaceId,
indicesToQuery,
alertTimestampOverride,
intendedTimestamp,
}: {
ruleExecutionLogger: IRuleExecutionLogForExecutors;
completeRule: CompleteRule<RuleParams>;
ignoreFields: ConfigType['alertIgnoreFields'];
mergeStrategy: ConfigType['alertMergeStrategy'];
spaceId: string | null | undefined;
indicesToQuery: string[];
alertTimestampOverride: Date | undefined;
publicBaseUrl: string | undefined;
intendedTimestamp: Date | undefined;
}): WrapSequences =>
(sequences, buildReasonMessage) =>
sequences.reduce<
Array<
| WrappedFieldsLatest<EqlShellFieldsLatest>
| WrappedFieldsLatest<EqlBuildingBlockFieldsLatest>
>
>((acc, sequence) => {
const { shellAlert, buildingBlocks } = buildAlertGroupFromSequence({
ruleExecutionLogger,
sequence,
completeRule,
mergeStrategy,
spaceId,
buildReasonMessage,
indicesToQuery,
alertTimestampOverride,
publicBaseUrl,
intendedTimestamp,
});
if (shellAlert) {
acc.push(shellAlert, ...buildingBlocks);
return acc;
}
return acc;
}, []);

3
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts
View file

@ -16,7 +16,7 @@ import type { CreateRuleOptions, SecurityAlertType } from '../types';
export const createEsqlAlertType = (
createOptions: CreateRuleOptions
): SecurityAlertType<EsqlRuleParams, {}, {}, 'default'> => {
const { version, experimentalFeatures, licensing, scheduleNotificationResponseActionsService } =
const { experimentalFeatures, licensing, scheduleNotificationResponseActionsService } =
createOptions;
return {
id: ESQL_RULE_TYPE_ID,
@ -49,7 +49,6 @@ export const createEsqlAlertType = (
esqlExecutor({
...params,
experimentalFeatures,
version,
licensing,
scheduleNotificationResponseActionsService,
}),

40
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.test.ts
View file

@ -5,20 +5,18 @@
* 2.0.
*/
import dateMath from '@kbn/datemath';
import { KbnServerError } from '@kbn/kibana-utils-plugin/server';
import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks';
import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock';
import { licensingMock } from '@kbn/licensing-plugin/server/mocks';
import type { ExperimentalFeatures } from '../../../../../common';
import { getIndexVersion } from '../../routes/index/get_index_version';
import { SIGNALS_TEMPLATE_VERSION } from '../../routes/index/get_signals_template';
import type { EsqlRuleParams } from '../../rule_schema';
import { getCompleteRuleMock, getEsqlRuleParams } from '../../rule_schema/mocks';
import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
import { getEsqlRuleParams } from '../../rule_schema/mocks';
import { esqlExecutor } from './esql';
import { getDataTierFilter } from '../utils/get_data_tier_filter';
import { getSharedParamsMock } from '../__mocks__/shared_params';
jest.mock('../../routes/index/get_index_version');
jest.mock('../utils/get_data_tier_filter', () => ({ getDataTierFilter: jest.fn() }));
@ -26,48 +24,30 @@ jest.mock('../utils/get_data_tier_filter', () => ({ getDataTierFilter: jest.fn()
const getDataTierFilterMock = getDataTierFilter as jest.Mock;
describe('esqlExecutor', () => {
const version = '9.1.0';
const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create();
let alertServices: RuleExecutorServicesMock;
(getIndexVersion as jest.Mock).mockReturnValue(SIGNALS_TEMPLATE_VERSION);
const params = getEsqlRuleParams();
const esqlCompleteRule = getCompleteRuleMock<EsqlRuleParams>(params);
const tuple = {
from: dateMath.parse(params.from)!,
to: dateMath.parse(params.to)!,
maxSignals: params.maxSignals,
};
const mockExperimentalFeatures = {} as ExperimentalFeatures;
const mockScheduleNotificationResponseActionsService = jest.fn();
const SPACE_ID = 'space';
const PUBLIC_BASE_URL = 'http://testkibanabaseurl.com';
const licensing = licensingMock.createSetup();
let mockedArguments: Parameters<typeof esqlExecutor>[0];
const sharedParams = getSharedParamsMock({ ruleParams: params });
beforeEach(() => {
jest.clearAllMocks();
alertServices = alertsMock.createRuleExecutorServices();
getDataTierFilterMock.mockResolvedValue([]);
mockedArguments = {
runOpts: {
completeRule: esqlCompleteRule,
tuple,
ruleExecutionLogger,
bulkCreate: jest.fn(),
mergeStrategy: 'allFields',
primaryTimestamp: '@timestamp',
alertWithSuppression: jest.fn(),
unprocessedExceptions: [getExceptionListItemSchemaMock()],
publicBaseUrl: PUBLIC_BASE_URL,
},
sharedParams,
services: alertServices,
version,
licensing: {},
spaceId: SPACE_ID,
licensing,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduleNotificationResponseActionsService,
} as unknown as Parameters<typeof esqlExecutor>[0];
state: {},
};
});
describe('errors', () => {

45
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts
View file

@ -28,7 +28,7 @@ import { rowToDocument, mergeEsqlResultInSource } from './utils';
import { fetchSourceDocuments } from './fetch_source_documents';
import { buildReasonMessageForEsqlAlert } from '../utils/reason_formatters';
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
import type { CreateRuleOptions, RunOpts, SignalSource } from '../types';
import type { CreateRuleOptions, SecuritySharedParams, SignalSource } from '../types';
import { logEsqlRequest } from '../utils/logged_requests';
import { getDataTierFilter } from '../utils/get_data_tier_filter';
import { checkErrorDetails } from './utils/check_error_details';
@ -48,37 +48,35 @@ import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_a
import type { ExperimentalFeatures } from '../../../../../common';
export const esqlExecutor = async ({
runOpts: {
completeRule,
tuple,
ruleExecutionLogger,
bulkCreate,
mergeStrategy,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
alertTimestampOverride,
publicBaseUrl,
alertWithSuppression,
intendedTimestamp,
},
sharedParams,
services,
state,
spaceId,
experimentalFeatures,
licensing,
scheduleNotificationResponseActionsService,
}: {
runOpts: RunOpts<EsqlRuleParams>;
sharedParams: SecuritySharedParams<EsqlRuleParams>;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
state: Record<string, unknown>;
spaceId: string;
version: string;
experimentalFeatures: ExperimentalFeatures;
licensing: LicensingPluginSetup;
scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService'];
}) => {
const {
completeRule,
tuple,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
ruleExecutionLogger,
spaceId,
mergeStrategy,
alertTimestampOverride,
publicBaseUrl,
intendedTimestamp,
bulkCreate,
} = sharedParams;
const loggedRequests: RulePreviewLoggedRequest[] = [];
const ruleParams = completeRule.ruleParams;
/**
@ -210,17 +208,12 @@ export const esqlExecutor = async ({
});
const bulkCreateResult = await bulkCreateSuppressedAlertsInMemory({
sharedParams,
enrichedEvents: syntheticHits,
toReturn: result,
wrapHits,
bulkCreate,
services,
ruleExecutionLogger,
tuple,
alertSuppression: completeRule.ruleParams.alertSuppression,
wrapSuppressedHits,
alertTimestampOverride,
alertWithSuppression,
experimentalFeatures,
buildReasonMessage: buildReasonMessageForEsqlAlert,
mergeSourceAndFields: true,

2
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/factories/index.ts
View file

@ -7,4 +7,4 @@
export * from './bulk_create_factory';
export * from './wrap_hits_factory';
export * from '../eql/wrap_sequences_factory';
export * from '../eql/wrap_sequences';

51
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts
View file

@ -22,7 +22,6 @@ export const createIndicatorMatchAlertType = (
): SecurityAlertType<ThreatRuleParams, {}, {}, 'default'> => {
const {
eventsTelemetry,
version,
licensing,
experimentalFeatures,
scheduleNotificationResponseActionsService,
@ -67,28 +66,7 @@ export const createIndicatorMatchAlertType = (
category: DEFAULT_APP_CATEGORIES.security.id,
producer: SERVER_APP_ID,
async executor(execOptions) {
const {
runOpts: {
inputIndex,
runtimeMappings,
completeRule,
tuple,
listClient,
ruleExecutionLogger,
searchAfterSize,
bulkCreate,
wrapHits,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
intendedTimestamp,
},
services,
spaceId,
state,
} = execOptions;
const runOpts = execOptions.runOpts;
const { sharedParams, services, state } = execOptions;
const wrapSuppressedHits = (
events: SignalSourceHit[],
@ -96,38 +74,15 @@ export const createIndicatorMatchAlertType = (
) =>
wrapSuppressedAlerts({
events,
spaceId,
completeRule,
mergeStrategy: runOpts.mergeStrategy,
indicesToQuery: runOpts.inputIndex,
buildReasonMessage,
alertTimestampOverride: runOpts.alertTimestampOverride,
ruleExecutionLogger,
publicBaseUrl: runOpts.publicBaseUrl,
primaryTimestamp,
secondaryTimestamp,
intendedTimestamp,
sharedParams,
});
const result = await indicatorMatchExecutor({
inputIndex,
runtimeMappings,
completeRule,
tuple,
listClient,
sharedParams,
services,
version,
searchAfterSize,
ruleExecutionLogger,
eventsTelemetry,
bulkCreate,
wrapHits,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
wrapSuppressedHits,
runOpts,
licensing,
experimentalFeatures,
scheduleNotificationResponseActionsService,

82
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/indicator_match.ts
View file

@ -5,8 +5,6 @@
* 2.0.
*/
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import type { estypes } from '@elastic/elasticsearch';
import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server';
import type {
@ -14,106 +12,36 @@ import type {
AlertInstanceState,
RuleExecutorServices,
} from '@kbn/alerting-plugin/server';
import type { ListClient } from '@kbn/lists-plugin/server';
import type { Filter } from '@kbn/es-query';
import type {
RuleRangeTuple,
BulkCreate,
WrapHits,
WrapSuppressedHits,
RunOpts,
CreateRuleOptions,
} from '../types';
import type { WrapSuppressedHits, SecuritySharedParams, CreateRuleOptions } from '../types';
import type { ITelemetryEventsSender } from '../../../telemetry/sender';
import { createThreatSignals } from './threat_mapping/create_threat_signals';
import type { CompleteRule, ThreatRuleParams } from '../../rule_schema';
import type { ThreatRuleParams } from '../../rule_schema';
import { withSecuritySpan } from '../../../../utils/with_security_span';
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../../../common/constants';
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
import { MAX_PER_PAGE } from './threat_mapping/get_event_count';
import type { ExperimentalFeatures } from '../../../../../common';
export const indicatorMatchExecutor = async ({
inputIndex,
runtimeMappings,
completeRule,
tuple,
listClient,
sharedParams,
services,
version,
searchAfterSize,
ruleExecutionLogger,
eventsTelemetry,
bulkCreate,
wrapHits,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
wrapSuppressedHits,
runOpts,
licensing,
experimentalFeatures,
scheduleNotificationResponseActionsService,
}: {
inputIndex: string[];
runtimeMappings: estypes.MappingRuntimeFields | undefined;
completeRule: CompleteRule<ThreatRuleParams>;
tuple: RuleRangeTuple;
listClient: ListClient;
sharedParams: SecuritySharedParams<ThreatRuleParams>;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
version: string;
searchAfterSize: number;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
eventsTelemetry: ITelemetryEventsSender | undefined;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
primaryTimestamp: string;
secondaryTimestamp?: string;
exceptionFilter: Filter | undefined;
unprocessedExceptions: ExceptionListItemSchema[];
wrapSuppressedHits: WrapSuppressedHits;
runOpts: RunOpts<ThreatRuleParams>;
licensing: LicensingPluginSetup;
scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService'];
experimentalFeatures: ExperimentalFeatures;
}) => {
const ruleParams = completeRule.ruleParams;
return withSecuritySpan('indicatorMatchExecutor', async () => {
return createThreatSignals({
alertId: completeRule.alertId,
bulkCreate,
completeRule,
concurrentSearches: ruleParams.concurrentSearches ?? 1,
sharedParams,
eventsTelemetry,
filters: ruleParams.filters ?? [],
inputIndex,
itemsPerSearch: ruleParams.itemsPerSearch ?? MAX_PER_PAGE,
language: ruleParams.language,
listClient,
outputIndex: ruleParams.outputIndex,
query: ruleParams.query,
ruleExecutionLogger,
savedId: ruleParams.savedId,
searchAfterSize,
services,
threatFilters: ruleParams.threatFilters ?? [],
threatIndex: ruleParams.threatIndex,
threatIndicatorPath: ruleParams.threatIndicatorPath ?? DEFAULT_INDICATOR_SOURCE_PATH,
threatLanguage: ruleParams.threatLanguage,
threatMapping: ruleParams.threatMapping,
threatQuery: ruleParams.threatQuery,
tuple,
type: ruleParams.type,
wrapHits,
wrapSuppressedHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
runOpts,
licensing,
experimentalFeatures,
scheduleNotificationResponseActionsService,

23
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/build_threat_enrichment.ts
View file

@ -6,7 +6,7 @@
*/
import type { SignalsEnrichment } from '../../types';
import type { BuildThreatEnrichmentOptions } from './types';
import type { BuildThreatEnrichmentOptions, GetThreatListOptions } from './types';
import { buildThreatMappingFilter } from './build_threat_mapping_filter';
import { getSignalsQueryMapFromThreatIndex } from './get_signals_map_from_threat_index';
@ -15,24 +15,17 @@ import { threatEnrichmentFactory } from './threat_enrichment_factory';
// we do want to make extra requests to the threat index to get enrichments from all threats
// previously we were enriched alerts only from `currentThreatList` but not all threats
export const buildThreatEnrichment = ({
ruleExecutionLogger,
sharedParams,
services,
threatFilters,
threatIndex,
threatIndicatorPath,
threatLanguage,
threatQuery,
pitId,
reassignPitId,
listClient,
exceptionFilter,
threatMapping,
runtimeMappings,
threatIndexFields,
}: BuildThreatEnrichmentOptions): SignalsEnrichment => {
return async (signals) => {
const threatFiltersFromEvents = buildThreatMappingFilter({
threatMapping,
threatMapping: sharedParams.completeRule.ruleParams.threatMapping,
threatList: signals,
entryKey: 'field',
allowedFieldsForTermsQuery: {
@ -41,22 +34,16 @@ export const buildThreatEnrichment = ({
},
});
const threatSearchParams = {
const threatSearchParams: Omit<GetThreatListOptions, 'searchAfter'> = {
sharedParams,
esClient: services.scopedClusterClient.asCurrentUser,
threatFilters: [...threatFilters, threatFiltersFromEvents],
query: threatQuery,
language: threatLanguage,
index: threatIndex,
ruleExecutionLogger,
threatListConfig: {
_source: [`${threatIndicatorPath}.*`, 'threat.feed.*'],
fields: undefined,
},
pitId,
reassignPitId,
runtimeMappings,
listClient,
exceptionFilter,
indexFields: threatIndexFields,
};

70
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts
View file

@ -5,12 +5,16 @@
* 2.0.
*/
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../../../../common/constants';
import { buildThreatMappingFilter } from './build_threat_mapping_filter';
import { getFilter } from '../../utils/get_filter';
import { searchAfterAndBulkCreate } from '../../utils/search_after_bulk_create';
import { buildReasonMessageForThreatMatchAlert } from '../../utils/reason_formatters';
import type { CreateEventSignalOptions } from './types';
import type { SearchAfterAndBulkCreateReturnType } from '../../types';
import type { CreateEventSignalOptions, GetThreatListOptions } from './types';
import type {
SearchAfterAndBulkCreateParams,
SearchAfterAndBulkCreateReturnType,
} from '../../types';
import { getSignalsQueryMapFromThreatIndex } from './get_signals_map_from_threat_index';
import { searchAfterAndBulkCreateSuppressedAlerts } from '../../utils/search_after_bulk_create_suppressed_alerts';
@ -22,46 +26,34 @@ import {
} from './utils';
export const createEventSignal = async ({
bulkCreate,
sharedParams,
currentResult,
currentEventList,
eventsTelemetry,
filters,
inputIndex,
language,
listClient,
query,
ruleExecutionLogger,
savedId,
searchAfterSize,
services,
threatMapping,
tuple,
type,
wrapHits,
wrapSuppressedHits,
threatQuery,
threatFilters,
threatLanguage,
threatIndex,
threatIndicatorPath,
threatPitId,
reassignThreatPitId,
runtimeMappings,
runOpts,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
allowedFieldsForTermsQuery,
threatMatchedFields,
inputIndexFields,
threatIndexFields,
completeRule,
sortOrder = 'desc',
isAlertSuppressionActive,
experimentalFeatures,
}: CreateEventSignalOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
const {
ruleExecutionLogger,
exceptionFilter,
inputIndex,
completeRule: {
ruleParams: { threatMapping, type, language, query, savedId },
},
} = sharedParams;
const threatIndicatorPath =
sharedParams.completeRule.ruleParams.threatIndicatorPath ?? DEFAULT_INDICATOR_SOURCE_PATH;
const threatFiltersFromEvents = buildThreatMappingFilter({
threatMapping,
threatList: currentEventList,
@ -77,22 +69,16 @@ export const createEventSignal = async ({
);
return currentResult;
} else {
const threatSearchParams = {
const threatSearchParams: Omit<GetThreatListOptions, 'searchAfter'> = {
sharedParams,
esClient: services.scopedClusterClient.asCurrentUser,
threatFilters: [...threatFilters, threatFiltersFromEvents],
query: threatQuery,
language: threatLanguage,
index: threatIndex,
ruleExecutionLogger,
threatListConfig: {
_source: threatMatchedFields.threat,
fields: undefined,
},
pitId: threatPitId,
reassignPitId: reassignThreatPitId,
runtimeMappings,
listClient,
exceptionFilter,
indexFields: threatIndexFields,
};
@ -157,34 +143,22 @@ export const createEventSignal = async ({
});
let createResult: SearchAfterAndBulkCreateReturnType;
const searchAfterBulkCreateParams = {
const searchAfterBulkCreateParams: SearchAfterAndBulkCreateParams = {
sharedParams,
buildReasonMessage: buildReasonMessageForThreatMatchAlert,
bulkCreate,
enrichment,
eventsTelemetry,
exceptionsList: unprocessedExceptions,
filter: esFilter,
inputIndexPattern: inputIndex,
listClient,
pageSize: searchAfterSize,
ruleExecutionLogger,
services,
sortOrder,
trackTotalHits: false,
tuple,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
};
if (isAlertSuppressionActive) {
createResult = await searchAfterAndBulkCreateSuppressedAlerts({
...searchAfterBulkCreateParams,
wrapSuppressedHits,
alertTimestampOverride: runOpts.alertTimestampOverride,
alertWithSuppression: runOpts.alertWithSuppression,
alertSuppression: completeRule.ruleParams.alertSuppression,
alertSuppression: sharedParams.completeRule.ruleParams.alertSuppression,
experimentalFeatures,
});
} else {

69
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts
View file

@ -5,49 +5,29 @@
* 2.0.
*/
import { DEFAULT_INDICATOR_SOURCE_PATH } from '../../../../../../common/constants';
import { buildThreatMappingFilter } from './build_threat_mapping_filter';
import { getFilter } from '../../utils/get_filter';
import { searchAfterAndBulkCreate } from '../../utils/search_after_bulk_create';
import { buildReasonMessageForThreatMatchAlert } from '../../utils/reason_formatters';
import type { CreateThreatSignalOptions } from './types';
import type { SearchAfterAndBulkCreateReturnType } from '../../types';
import type {
SearchAfterAndBulkCreateParams,
SearchAfterAndBulkCreateReturnType,
} from '../../types';
import { searchAfterAndBulkCreateSuppressedAlerts } from '../../utils/search_after_bulk_create_suppressed_alerts';
import { buildThreatEnrichment } from './build_threat_enrichment';
export const createThreatSignal = async ({
alertId,
bulkCreate,
completeRule,
sharedParams,
currentResult,
currentThreatList,
eventsTelemetry,
filters,
inputIndex,
language,
listClient,
outputIndex,
query,
ruleExecutionLogger,
savedId,
searchAfterSize,
services,
threatMapping,
tuple,
type,
wrapHits,
wrapSuppressedHits,
runtimeMappings,
runOpts,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
threatFilters,
threatIndex,
threatIndicatorPath,
threatLanguage,
threatPitId,
threatQuery,
reassignThreatPitId,
allowedFieldsForTermsQuery,
inputIndexFields,
@ -56,6 +36,16 @@ export const createThreatSignal = async ({
isAlertSuppressionActive,
experimentalFeatures,
}: CreateThreatSignalOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
const {
exceptionFilter,
inputIndex,
ruleExecutionLogger,
completeRule: {
ruleParams: { language, query, savedId, threatMapping, type },
},
} = sharedParams;
const threatIndicatorPath =
sharedParams.completeRule.ruleParams.threatIndicatorPath ?? DEFAULT_INDICATOR_SOURCE_PATH;
const threatFilter = buildThreatMappingFilter({
threatMapping,
threatList: currentThreatList,
@ -89,51 +79,32 @@ export const createThreatSignal = async ({
);
const threatEnrichment = buildThreatEnrichment({
ruleExecutionLogger,
sharedParams,
services,
threatFilters,
threatIndex,
threatIndicatorPath,
threatLanguage,
threatQuery,
pitId: threatPitId,
reassignPitId: reassignThreatPitId,
listClient,
exceptionFilter,
threatMapping,
runtimeMappings,
threatIndexFields,
});
let result: SearchAfterAndBulkCreateReturnType;
const searchAfterBulkCreateParams = {
const searchAfterBulkCreateParams: SearchAfterAndBulkCreateParams = {
sharedParams,
buildReasonMessage: buildReasonMessageForThreatMatchAlert,
bulkCreate,
enrichment: threatEnrichment,
eventsTelemetry,
exceptionsList: unprocessedExceptions,
filter: esFilter,
inputIndexPattern: inputIndex,
listClient,
pageSize: searchAfterSize,
ruleExecutionLogger,
services,
sortOrder,
trackTotalHits: false,
tuple,
wrapHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
};
if (isAlertSuppressionActive) {
result = await searchAfterAndBulkCreateSuppressedAlerts({
...searchAfterBulkCreateParams,
wrapSuppressedHits,
alertTimestampOverride: runOpts.alertTimestampOverride,
alertWithSuppression: runOpts.alertWithSuppression,
alertSuppression: completeRule.ruleParams.alertSuppression,
alertSuppression: sharedParams.completeRule.ruleParams.alertSuppression,
experimentalFeatures,
});
} else {

120
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts
View file

@ -32,7 +32,7 @@ import {
} from './utils';
import { getAllowedFieldsForTermQuery } from './get_allowed_fields_for_terms_query';
import { getEventCount, getEventList } from './get_event_count';
import { MAX_PER_PAGE, getEventCount, getEventList } from './get_event_count';
import { getMappingFilters } from './get_mapping_filters';
import { THREAT_PIT_KEEP_ALIVE } from '../../../../../../common/cti/constants';
import { getMaxSignalsWarning, getSafeSortIds } from '../../utils/utils';
@ -40,42 +40,32 @@ import { getDataTierFilter } from '../../utils/get_data_tier_filter';
import { getQueryFields } from '../../utils/get_query_fields';
export const createThreatSignals = async ({
alertId,
bulkCreate,
completeRule,
concurrentSearches,
sharedParams,
eventsTelemetry,
filters,
inputIndex,
itemsPerSearch,
language,
listClient,
outputIndex,
query,
ruleExecutionLogger,
savedId,
searchAfterSize,
services,
threatFilters,
threatIndex,
threatIndicatorPath,
threatLanguage,
threatMapping,
threatQuery,
tuple,
type,
wrapHits,
wrapSuppressedHits,
runOpts,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
licensing,
experimentalFeatures,
scheduleNotificationResponseActionsService,
}: CreateThreatSignalsOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
const {
inputIndex,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
completeRule,
tuple,
ruleExecutionLogger,
} = sharedParams;
const {
alertId,
ruleParams: { language, query, threatIndex, threatLanguage, threatMapping, threatQuery },
} = completeRule;
const itemsPerSearch = completeRule.ruleParams.itemsPerSearch ?? MAX_PER_PAGE;
const concurrentSearches = completeRule.ruleParams.concurrentSearches ?? 1;
const filters = completeRule.ruleParams.filters ?? [];
const threatFilters = completeRule.ruleParams.threatFilters ?? [];
const threatMatchedFields = getMatchedFields(threatMapping);
const threatFieldsLength = threatMatchedFields.threat.length;
const allowedFieldsForTermsQuery = await getAllowedFieldsForTermQuery({
@ -323,19 +313,11 @@ export const createThreatSignals = async ({
totalDocumentCount: eventCount,
getDocumentList: async ({ searchAfter }) =>
getEventList({
sharedParams,
services,
ruleExecutionLogger,
filters: allEventFilters,
query,
language,
index: inputIndex,
searchAfter,
perPage,
tuple,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
eventListConfig,
indexFields: inputIndexFields,
sortOrder,
@ -343,44 +325,20 @@ export const createThreatSignals = async ({
createSignal: (slicedChunk) =>
createEventSignal({
alertId,
bulkCreate,
completeRule,
sharedParams,
currentEventList: slicedChunk,
currentResult: results,
eventsTelemetry,
filters: allEventFilters,
inputIndex,
language,
listClient,
outputIndex,
query,
reassignThreatPitId,
ruleExecutionLogger,
savedId,
searchAfterSize,
services,
threatFilters: allThreatFilters,
threatIndex,
threatIndicatorPath,
threatLanguage,
threatMapping,
threatPitId,
threatQuery,
tuple,
type,
wrapHits,
wrapSuppressedHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
allowedFieldsForTermsQuery,
threatMatchedFields,
inputIndexFields,
threatIndexFields,
runOpts,
sortOrder,
isAlertSuppressionActive,
experimentalFeatures,
@ -391,62 +349,32 @@ export const createThreatSignals = async ({
totalDocumentCount: threatListCount,
getDocumentList: async ({ searchAfter }) =>
getThreatList({
sharedParams,
esClient: services.scopedClusterClient.asCurrentUser,
threatFilters: allThreatFilters,
query: threatQuery,
language: threatLanguage,
index: threatIndex,
searchAfter,
ruleExecutionLogger,
perPage,
threatListConfig,
pitId: threatPitId,
reassignPitId: reassignThreatPitId,
runtimeMappings,
listClient,
exceptionFilter,
indexFields: threatIndexFields,
}),
createSignal: (slicedChunk) =>
createThreatSignal({
alertId,
bulkCreate,
completeRule,
sharedParams,
currentResult: results,
currentThreatList: slicedChunk,
eventsTelemetry,
filters: allEventFilters,
inputIndex,
language,
listClient,
outputIndex,
query,
ruleExecutionLogger,
savedId,
searchAfterSize,
services,
threatMapping,
tuple,
type,
wrapHits,
wrapSuppressedHits,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
exceptionFilter,
unprocessedExceptions,
threatFilters: allThreatFilters,
threatIndex,
threatIndicatorPath,
threatLanguage,
threatPitId,
threatQuery,
reassignThreatPitId,
allowedFieldsForTermsQuery,
inputIndexFields,
threatIndexFields,
runOpts,
sortOrder,
isAlertSuppressionActive,
experimentalFeatures,

31
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_event_count.ts
View file

@ -14,44 +14,48 @@ import { buildEventsSearchQuery } from '../../utils/build_events_query';
export const MAX_PER_PAGE = 9000;
export const getEventList = async ({
sharedParams,
services,
ruleExecutionLogger,
query,
language,
index,
perPage,
searchAfter,
filters,
tuple,
primaryTimestamp,
secondaryTimestamp,
runtimeMappings,
exceptionFilter,
eventListConfig,
indexFields,
sortOrder = 'desc',
}: EventsOptions): Promise<estypes.SearchResponse<EventDoc>> => {
const {
inputIndex,
ruleExecutionLogger,
primaryTimestamp,
secondaryTimestamp,
runtimeMappings,
tuple,
exceptionFilter,
completeRule: {
ruleParams: { query, language },
},
} = sharedParams;
const calculatedPerPage = perPage ?? MAX_PER_PAGE;
if (calculatedPerPage > 10000) {
throw new TypeError('perPage cannot exceed the size of 10000');
}
ruleExecutionLogger.debug(
`Querying the events items from the index: "${index}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items`
`Querying the events items from the index: "${sharedParams.inputIndex}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items`
);
const queryFilter = getQueryFilter({
query,
language: language ?? 'kuery',
filters,
index,
index: inputIndex,
exceptionFilter,
fields: indexFields,
});
const { searchResult } = await singleSearchAfter({
searchAfterSortIds: searchAfter,
index,
index: inputIndex,
from: tuple.from.toISOString(),
to: tuple.to.toISOString(),
services,
@ -70,6 +74,9 @@ export const getEventList = async ({
return searchResult;
};
// TODO: possible bug: event count does not respect large value list exceptions, but searchAfterBulkCreate does.
// could lead to worse performance
export const getEventCount = async ({
esClient,
query,

13
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_signals_map_from_threat_index.mock.ts
View file

@ -7,28 +7,21 @@
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
import type { GetThreatListOptions } from './types';
import { getListClientMock } from '@kbn/lists-plugin/server/services/lists/list_client.mock';
import { ruleExecutionLogMock } from '../../../rule_monitoring/mocks';
import { getSharedParamsMock } from '../../__mocks__/shared_params';
import { getThreatRuleParams } from '../../../rule_schema/mocks';
const esClient = elasticsearchServiceMock.createElasticsearchClient();
const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create();
export const threatSearchParamsMock: GetThreatListOptions = {
sharedParams: getSharedParamsMock({ ruleParams: getThreatRuleParams() }),
esClient,
query: '*:*',
language: 'kuery',
threatFilters: [],
index: ['threats-*'],
ruleExecutionLogger,
threatListConfig: {
_source: false,
fields: undefined,
},
pitId: 'mock',
reassignPitId: jest.fn(),
listClient: getListClientMock(),
searchAfter: undefined,
runtimeMappings: undefined,
exceptionFilter: undefined,
indexFields: [],
};

28
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts
View file

@ -20,37 +20,41 @@ import type {
export const INDICATOR_PER_PAGE = 1000;
export const getThreatList = async ({
sharedParams,
esClient,
index,
language,
perPage,
query,
ruleExecutionLogger,
searchAfter,
threatFilters,
threatListConfig,
pitId,
reassignPitId,
runtimeMappings,
listClient,
exceptionFilter,
indexFields,
}: GetThreatListOptions): Promise<estypes.SearchResponse<ThreatListDoc>> => {
const {
exceptionFilter,
listClient,
runtimeMappings,
ruleExecutionLogger,
completeRule: {
ruleParams: { threatQuery, threatLanguage, threatIndex },
},
} = sharedParams;
const calculatedPerPage = perPage ?? INDICATOR_PER_PAGE;
if (calculatedPerPage > 10000) {
throw new TypeError('perPage cannot exceed the size of 10000');
}
const queryFilter = getQueryFilter({
query,
language: language ?? 'kuery',
query: threatQuery,
language: threatLanguage ?? 'kuery',
filters: threatFilters,
index,
index: threatIndex,
// Exceptions shouldn't apply to threat list??
exceptionFilter,
fields: indexFields,
});
ruleExecutionLogger.debug(
`Querying the indicator items from the index: "${index}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items`
`Querying the indicator items from the index: "${threatIndex}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items`
);
const response = await esClient.search<
@ -62,7 +66,7 @@ export const getThreatList = async ({
search_after: searchAfter,
runtime_mappings: runtimeMappings,
sort: getSortForThreatList({
index,
index: threatIndex,
listItemIndex: listClient.getListItemName(),
}),
track_total_hits: false,

7
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/threat_enrichment_factory.ts
View file

@ -5,16 +5,17 @@
* 2.0.
*/
import type { CreateEventSignalOptions, GetThreatListOptions } from './types';
import type { ThreatIndicatorPath } from '../../../../../../common/api/detection_engine';
import type { GetThreatListOptions } from './types';
import type { SignalSourceHit } from '../../types';
import { getThreatList } from './get_threat_list';
import { enrichSignalThreatMatchesFromSignalsMap } from './enrich_signal_threat_matches';
import { type SignalsQueryMap } from './get_signals_map_from_threat_index';
interface ThreatEnrichmentFactoryOptions {
threatIndicatorPath: CreateEventSignalOptions['threatIndicatorPath'];
threatIndicatorPath: ThreatIndicatorPath;
signalsQueryMap: SignalsQueryMap;
threatFilters: CreateEventSignalOptions['threatFilters'];
threatFilters: unknown[];
threatSearchParams: Omit<GetThreatListOptions, 'searchAfter'>;
}

122
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/types.ts
View file

@ -6,24 +6,17 @@
*/
import type { estypes } from '@elastic/elasticsearch';
import type {
ThreatQuery,
ThreatMapping,
ThreatMappingEntries,
ThreatIndex,
ThreatLanguageOrUndefined,
ConcurrentSearches,
ItemsPerSearch,
ThreatIndicatorPath,
LanguageOrUndefined,
Type,
} from '@kbn/securitysolution-io-ts-alerting-types';
import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server';
import type {
OpenPointInTimeResponse,
QueryDslBoolQuery,
} from '@elastic/elasticsearch/lib/api/types';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import type { ListClient } from '@kbn/lists-plugin/server';
import type {
AlertInstanceContext,
AlertInstanceState,
@ -33,143 +26,65 @@ import type { ElasticsearchClient } from '@kbn/core/server';
import type { Filter, DataViewFieldBase } from '@kbn/es-query';
import type { ITelemetryEventsSender } from '../../../../telemetry/sender';
import type {
BulkCreate,
RuleRangeTuple,
SearchAfterAndBulkCreateReturnType,
WrapHits,
WrapSuppressedHits,
OverrideBodyQuery,
RunOpts,
SecuritySharedParams,
CreateRuleOptions,
} from '../../types';
import type { CompleteRule, ThreatRuleParams } from '../../../rule_schema';
import type { ThreatRuleParams } from '../../../rule_schema';
import type { IRuleExecutionLogForExecutors } from '../../../rule_monitoring';
import type { ExperimentalFeatures } from '../../../../../../common';
export type SortOrderOrUndefined = 'asc' | 'desc' | undefined;
export interface CreateThreatSignalsOptions {
alertId: string;
bulkCreate: BulkCreate;
completeRule: CompleteRule<ThreatRuleParams>;
concurrentSearches: ConcurrentSearches;
sharedParams: SecuritySharedParams<ThreatRuleParams>;
eventsTelemetry: ITelemetryEventsSender | undefined;
filters: unknown[];
inputIndex: string[];
itemsPerSearch: ItemsPerSearch;
language: LanguageOrUndefined;
listClient: ListClient;
outputIndex: string;
query: string;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
savedId: string | undefined;
searchAfterSize: number;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
threatFilters: unknown[];
threatIndex: ThreatIndex;
threatIndicatorPath: ThreatIndicatorPath;
threatLanguage: ThreatLanguageOrUndefined;
threatMapping: ThreatMapping;
threatQuery: ThreatQuery;
tuple: RuleRangeTuple;
type: Type;
wrapHits: WrapHits;
wrapSuppressedHits: WrapSuppressedHits;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
exceptionFilter: Filter | undefined;
unprocessedExceptions: ExceptionListItemSchema[];
runOpts: RunOpts<ThreatRuleParams>;
licensing: LicensingPluginSetup;
experimentalFeatures: ExperimentalFeatures;
scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService'];
}
export interface CreateThreatSignalOptions {
alertId: string;
bulkCreate: BulkCreate;
completeRule: CompleteRule<ThreatRuleParams>;
sharedParams: SecuritySharedParams<ThreatRuleParams>;
currentResult: SearchAfterAndBulkCreateReturnType;
currentThreatList: ThreatListItem[];
eventsTelemetry: ITelemetryEventsSender | undefined;
filters: unknown[];
inputIndex: string[];
language: LanguageOrUndefined;
listClient: ListClient;
outputIndex: string;
query: string;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
savedId: string | undefined;
searchAfterSize: number;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
threatMapping: ThreatMapping;
tuple: RuleRangeTuple;
type: Type;
wrapHits: WrapHits;
wrapSuppressedHits: WrapSuppressedHits;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
exceptionFilter: Filter | undefined;
unprocessedExceptions: ExceptionListItemSchema[];
threatFilters: unknown[];
threatIndex: ThreatIndex;
threatIndicatorPath: ThreatIndicatorPath;
threatLanguage: ThreatLanguageOrUndefined;
threatQuery: ThreatQuery;
perPage?: number;
threatPitId: OpenPointInTimeResponse['id'];
reassignThreatPitId: (newPitId: OpenPointInTimeResponse['id'] | undefined) => void;
allowedFieldsForTermsQuery: AllowedFieldsForTermsQuery;
inputIndexFields: DataViewFieldBase[];
threatIndexFields: DataViewFieldBase[];
runOpts: RunOpts<ThreatRuleParams>;
sortOrder?: SortOrderOrUndefined;
isAlertSuppressionActive: boolean;
experimentalFeatures: ExperimentalFeatures;
}
export interface CreateEventSignalOptions {
alertId: string;
bulkCreate: BulkCreate;
completeRule: CompleteRule<ThreatRuleParams>;
sharedParams: SecuritySharedParams<ThreatRuleParams>;
currentResult: SearchAfterAndBulkCreateReturnType;
currentEventList: EventItem[];
eventsTelemetry: ITelemetryEventsSender | undefined;
filters: unknown[];
inputIndex: string[];
language: LanguageOrUndefined;
listClient: ListClient;
outputIndex: string;
query: string;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
savedId: string | undefined;
searchAfterSize: number;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
tuple: RuleRangeTuple;
type: Type;
wrapHits: WrapHits;
wrapSuppressedHits: WrapSuppressedHits;
threatFilters: unknown[];
threatIndex: ThreatIndex;
threatIndicatorPath: ThreatIndicatorPath;
threatLanguage: ThreatLanguageOrUndefined;
threatMapping: ThreatMapping;
threatQuery: ThreatQuery;
perPage?: number;
threatPitId: OpenPointInTimeResponse['id'];
reassignThreatPitId: (newPitId: OpenPointInTimeResponse['id'] | undefined) => void;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
exceptionFilter: Filter | undefined;
unprocessedExceptions: ExceptionListItemSchema[];
allowedFieldsForTermsQuery: AllowedFieldsForTermsQuery;
threatMatchedFields: ThreatMatchedFields;
inputIndexFields: DataViewFieldBase[];
threatIndexFields: DataViewFieldBase[];
runOpts: RunOpts<ThreatRuleParams>;
sortOrder?: SortOrderOrUndefined;
isAlertSuppressionActive: boolean;
experimentalFeatures: ExperimentalFeatures;
@ -230,20 +145,14 @@ interface ThreatListConfig {
}
export interface GetThreatListOptions {
sharedParams: SecuritySharedParams<ThreatRuleParams>;
esClient: ElasticsearchClient;
index: string[];
language: ThreatLanguageOrUndefined;
perPage?: number;
query: string;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
searchAfter: estypes.SortResults | undefined;
threatFilters: unknown[];
threatListConfig: ThreatListConfig;
pitId: OpenPointInTimeResponse['id'];
reassignPitId: (newPitId: OpenPointInTimeResponse['id'] | undefined) => void;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
listClient: ListClient;
exceptionFilter: Filter | undefined;
indexFields: DataViewFieldBase[];
}
@ -291,36 +200,21 @@ export type DecodedThreatNamedQuery = BaseThreatNamedQuery & { id?: string; inde
export type GetMatchedThreats = (ids: string[]) => Promise<ThreatListItem[]>;
export interface BuildThreatEnrichmentOptions {
ruleExecutionLogger: IRuleExecutionLogForExecutors;
sharedParams: SecuritySharedParams<ThreatRuleParams>;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
threatFilters: unknown[];
threatIndex: ThreatIndex;
threatIndicatorPath: ThreatIndicatorPath;
threatLanguage: ThreatLanguageOrUndefined;
threatQuery: ThreatQuery;
pitId: string;
reassignPitId: (newPitId: OpenPointInTimeResponse['id'] | undefined) => void;
listClient: ListClient;
exceptionFilter: Filter | undefined;
threatMapping: ThreatMapping;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
threatIndexFields: DataViewFieldBase[];
}
export interface EventsOptions {
sharedParams: SecuritySharedParams<ThreatRuleParams>;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
query: string;
language: ThreatLanguageOrUndefined;
index: string[];
searchAfter: estypes.SortResults | undefined;
perPage?: number;
filters: unknown[];
primaryTimestamp: string;
secondaryTimestamp?: string;
tuple: RuleRangeTuple;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
exceptionFilter: Filter | undefined;
eventListConfig?: OverrideBodyQuery;
indexFields: DataViewFieldBase[];
sortOrder?: SortOrderOrUndefined;

47
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts
View file

@ -54,31 +54,10 @@ export const createMlAlertType = (
category: DEFAULT_APP_CATEGORIES.security.id,
producer: SERVER_APP_ID,
async executor(execOptions) {
const {
runOpts: {
bulkCreate,
completeRule,
listClient,
ruleExecutionLogger,
tuple,
wrapHits,
exceptionFilter,
unprocessedExceptions,
mergeStrategy,
alertTimestampOverride,
publicBaseUrl,
alertWithSuppression,
primaryTimestamp,
secondaryTimestamp,
intendedTimestamp,
},
services,
spaceId,
state,
} = execOptions;
const { sharedParams, services, state } = execOptions;
const isAlertSuppressionActive = await getIsAlertSuppressionActive({
alertSuppression: completeRule.ruleParams.alertSuppression,
alertSuppression: sharedParams.completeRule.ruleParams.alertSuppression,
licensing,
});
const isLoggedRequestsEnabled = Boolean(state?.isLoggedRequestsEnabled);
@ -86,33 +65,15 @@ export const createMlAlertType = (
const wrapSuppressedHits: WrapSuppressedHits = (events, buildReasonMessage) =>
wrapSuppressedAlerts({
events,
spaceId,
completeRule,
mergeStrategy,
indicesToQuery: [],
buildReasonMessage,
alertTimestampOverride,
ruleExecutionLogger,
publicBaseUrl,
primaryTimestamp,
secondaryTimestamp,
intendedTimestamp,
sharedParams,
});
const { result, loggedRequests } = await mlExecutor({
completeRule,
tuple,
sharedParams,
ml,
listClient,
services,
ruleExecutionLogger,
bulkCreate,
wrapHits,
exceptionFilter,
unprocessedExceptions,
wrapSuppressedHits,
alertTimestampOverride,
alertWithSuppression,
isAlertSuppressionActive,
experimentalFeatures,
scheduleNotificationResponseActionsService,

86
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts
View file

@ -11,12 +11,12 @@ import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
import { mlExecutor } from './ml';
import type { ExperimentalFeatures } from '../../../../../common';
import { getCompleteRuleMock, getMlRuleParams } from '../../rule_schema/mocks';
import { getListClientMock } from '@kbn/lists-plugin/server/services/lists/list_client.mock';
import { findMlSignals } from './find_ml_signals';
import { bulkCreateMlSignals } from './bulk_create_ml_signals';
import { mlPluginServerMock } from '@kbn/ml-plugin/server/mocks';
import type { MachineLearningRuleParams } from '../../rule_schema';
import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
import { getSharedParamsMock } from '../__mocks__/shared_params';
jest.mock('./find_ml_signals');
jest.mock('./bulk_create_ml_signals');
@ -29,7 +29,7 @@ describe('ml_executor', () => {
let stopDatafeedsMock: jest.Mock;
let mlMock: ReturnType<typeof mlPluginServerMock.createSetupContract>;
let alertServices: RuleExecutorServicesMock;
let ruleExecutionLogger: ReturnType<typeof ruleExecutionLogMock.forExecutors.create>;
const params = getMlRuleParams();
const mlCompleteRule = getCompleteRuleMock<MachineLearningRuleParams>(params);
const tuple = {
@ -37,7 +37,15 @@ describe('ml_executor', () => {
to: dateMath.parse(params.to)!,
maxSignals: params.maxSignals,
};
const listClient = getListClientMock();
const sharedParams = getSharedParamsMock({ ruleParams: params, rewrites: { tuple } });
const ruleExecutionLogger: ReturnType<typeof ruleExecutionLogMock.forExecutors.create> =
ruleExecutionLogMock.forExecutors.create({
ruleId: sharedParams.completeRule.alertId,
ruleUuid: sharedParams.completeRule.ruleParams.ruleId,
ruleName: sharedParams.completeRule.ruleConfig.name,
ruleType: sharedParams.completeRule.ruleConfig.ruleTypeId,
});
sharedParams.ruleExecutionLogger = ruleExecutionLogger;
beforeEach(() => {
mockExperimentalFeatures = {} as jest.Mocked<ExperimentalFeatures>;
@ -50,12 +58,6 @@ describe('ml_executor', () => {
stopDatafeeds: stopDatafeedsMock,
});
alertServices = alertsMock.createRuleExecutorServices();
ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create({
ruleId: mlCompleteRule.alertId,
ruleUuid: mlCompleteRule.ruleParams.ruleId,
ruleName: mlCompleteRule.ruleConfig.name,
ruleType: mlCompleteRule.ruleConfig.ruleTypeId,
});
(findMlSignals as jest.Mock).mockResolvedValue({
anomalyResults: {
_shards: {},
@ -77,19 +79,10 @@ describe('ml_executor', () => {
it('should throw an error if ML plugin was not available', async () => {
await expect(
mlExecutor({
completeRule: mlCompleteRule,
tuple,
sharedParams,
ml: undefined,
services: alertServices,
ruleExecutionLogger,
listClient,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
exceptionFilter: undefined,
unprocessedExceptions: [],
wrapSuppressedHits: jest.fn(),
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
isAlertSuppressionActive: true,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction,
@ -100,19 +93,10 @@ describe('ml_executor', () => {
it('should record a partial failure if Machine learning job summary was null', async () => {
jobsSummaryMock.mockResolvedValue([]);
const { result } = await mlExecutor({
completeRule: mlCompleteRule,
tuple,
sharedParams,
ml: mlMock,
services: alertServices,
ruleExecutionLogger,
listClient,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
exceptionFilter: undefined,
unprocessedExceptions: [],
wrapSuppressedHits: jest.fn(),
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
isAlertSuppressionActive: true,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction,
@ -134,19 +118,10 @@ describe('ml_executor', () => {
]);
const { result } = await mlExecutor({
completeRule: mlCompleteRule,
tuple,
sharedParams,
ml: mlMock,
services: alertServices,
ruleExecutionLogger,
listClient,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
exceptionFilter: undefined,
unprocessedExceptions: [],
wrapSuppressedHits: jest.fn(),
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
isAlertSuppressionActive: true,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction,
@ -164,19 +139,10 @@ describe('ml_executor', () => {
});
const { result } = await mlExecutor({
completeRule: mlCompleteRule,
tuple,
sharedParams,
ml: mlMock,
services: alertServices,
ruleExecutionLogger,
listClient,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
exceptionFilter: undefined,
unprocessedExceptions: [],
wrapSuppressedHits: jest.fn(),
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
isAlertSuppressionActive: true,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction,
@ -197,19 +163,10 @@ describe('ml_executor', () => {
);
const { result } = await mlExecutor({
completeRule: mlCompleteRule,
tuple,
sharedParams,
ml: mlMock,
services: alertServices,
ruleExecutionLogger,
listClient,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
exceptionFilter: undefined,
unprocessedExceptions: [],
wrapSuppressedHits: jest.fn(),
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
isAlertSuppressionActive: true,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction,
@ -223,19 +180,10 @@ describe('ml_executor', () => {
});
it('should call scheduleNotificationResponseActionsService', async () => {
const { result } = await mlExecutor({
completeRule: mlCompleteRule,
tuple,
sharedParams,
ml: mlMock,
services: alertServices,
ruleExecutionLogger,
listClient,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
exceptionFilter: undefined,
unprocessedExceptions: [],
wrapSuppressedHits: jest.fn(),
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
isAlertSuppressionActive: true,
experimentalFeatures: mockExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction,

54
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts
View file

@ -8,29 +8,19 @@
/* eslint require-atomic-updates: ["error", { "allowProperties": true }] */
import type { KibanaRequest } from '@kbn/core/server';
import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import type {
AlertInstanceContext,
AlertInstanceState,
RuleExecutorServices,
} from '@kbn/alerting-plugin/server';
import type { ListClient } from '@kbn/lists-plugin/server';
import type { Filter } from '@kbn/es-query';
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
import { isJobStarted } from '../../../../../common/machine_learning/helpers';
import type { ExperimentalFeatures } from '../../../../../common/experimental_features';
import type { CompleteRule, MachineLearningRuleParams } from '../../rule_schema';
import type { MachineLearningRuleParams } from '../../rule_schema';
import { bulkCreateMlSignals } from './bulk_create_ml_signals';
import { filterEventsAgainstList } from '../utils/large_list_filters/filter_events_against_list';
import { findMlSignals } from './find_ml_signals';
import type {
BulkCreate,
CreateRuleOptions,
RuleRangeTuple,
WrapHits,
WrapSuppressedHits,
} from '../types';
import type { CreateRuleOptions, SecuritySharedParams, WrapSuppressedHits } from '../types';
import {
addToSearchAfterReturn,
createErrorsFromShard,
@ -40,25 +30,15 @@ import {
} from '../utils/utils';
import type { SetupPlugins } from '../../../../plugin';
import { withSecuritySpan } from '../../../../utils/with_security_span';
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
import type { AnomalyResults } from '../../../machine_learning';
import { bulkCreateSuppressedAlertsInMemory } from '../utils/bulk_create_suppressed_alerts_in_memory';
import { buildReasonMessageForMlAlert } from '../utils/reason_formatters';
interface MachineLearningRuleExecutorParams {
completeRule: CompleteRule<MachineLearningRuleParams>;
tuple: RuleRangeTuple;
sharedParams: SecuritySharedParams<MachineLearningRuleParams>;
ml: SetupPlugins['ml'];
listClient: ListClient;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
exceptionFilter: Filter | undefined;
unprocessedExceptions: ExceptionListItemSchema[];
wrapSuppressedHits: WrapSuppressedHits;
alertTimestampOverride: Date | undefined;
alertWithSuppression: SuppressedAlertService;
isAlertSuppressionActive: boolean;
experimentalFeatures: ExperimentalFeatures;
scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService'];
@ -66,24 +46,25 @@ interface MachineLearningRuleExecutorParams {
}
export const mlExecutor = async ({
completeRule,
tuple,
sharedParams,
ml,
listClient,
services,
ruleExecutionLogger,
bulkCreate,
wrapHits,
exceptionFilter,
unprocessedExceptions,
isAlertSuppressionActive,
wrapSuppressedHits,
alertTimestampOverride,
alertWithSuppression,
experimentalFeatures,
scheduleNotificationResponseActionsService,
isLoggedRequestsEnabled = false,
}: MachineLearningRuleExecutorParams) => {
const {
completeRule,
ruleExecutionLogger,
tuple,
exceptionFilter,
listClient,
unprocessedExceptions,
bulkCreate,
wrapHits,
} = sharedParams;
const result = createSearchAfterReturnType();
const ruleParams = completeRule.ruleParams;
const loggedRequests: RulePreviewLoggedRequest[] = [];
@ -174,18 +155,13 @@ export const mlExecutor = async ({
if (anomalyCount && isAlertSuppressionActive) {
await bulkCreateSuppressedAlertsInMemory({
sharedParams,
enrichedEvents: filteredAnomalyHits,
toReturn: result,
wrapHits,
bulkCreate,
services,
buildReasonMessage: buildReasonMessageForMlAlert,
ruleExecutionLogger,
tuple,
alertSuppression: completeRule.ruleParams.alertSuppression,
wrapSuppressedHits,
alertTimestampOverride,
alertWithSuppression,
experimentalFeatures,
});
} else {

23
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/bulk_create_suppressed_alerts_in_memory.ts
View file

@ -4,7 +4,6 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server';
import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas';
import type {
@ -29,20 +28,12 @@ import type { ExperimentalFeatures } from '../../../../../common';
interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams {
wrapSuppressedHits: WrapSuppressedHits;
alertTimestampOverride: Date | undefined;
alertWithSuppression: SuppressedAlertService;
alertSuppression?: AlertSuppressionCamel;
}
export interface BulkCreateSuppressedAlertsParams
extends Pick<
SearchAfterAndBulkCreateSuppressedAlertsParams,
| 'bulkCreate'
| 'services'
| 'ruleExecutionLogger'
| 'tuple'
| 'alertSuppression'
| 'alertWithSuppression'
| 'alertTimestampOverride'
'sharedParams' | 'services' | 'alertSuppression'
> {
wrapHits: (
events: EventsAndTerms[]
@ -63,17 +54,13 @@ export interface BulkCreateSuppressedAlertsParams
* it operates with new terms specific eventsAndTerms{@link EventsAndTerms} parameter property, instead of regular events as common utility
*/
export const bulkCreateSuppressedNewTermsAlertsInMemory = async ({
sharedParams,
eventsAndTerms,
wrapHits,
wrapSuppressedHits,
toReturn,
bulkCreate,
services,
ruleExecutionLogger,
tuple,
alertSuppression,
alertWithSuppression,
alertTimestampOverride,
experimentalFeatures,
}: BulkCreateSuppressedAlertsParams) => {
const suppressOnMissingFields =
@ -98,16 +85,12 @@ export const bulkCreateSuppressedNewTermsAlertsInMemory = async ({
const suppressibleWrappedDocs = wrapSuppressedHits(suppressibleEvents);
return executeBulkCreateAlerts({
sharedParams,
suppressibleWrappedDocs,
unsuppressibleWrappedDocs,
toReturn,
bulkCreate,
services,
ruleExecutionLogger,
tuple,
alertSuppression,
alertWithSuppression,
alertTimestampOverride,
experimentalFeatures,
});
};

49
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts
View file

@ -97,30 +97,25 @@ export const createNewTermsAlertType = (
category: DEFAULT_APP_CATEGORIES.security.id,
producer: SERVER_APP_ID,
async executor(execOptions) {
const { sharedParams, services, params, spaceId, state } = execOptions;
const {
runOpts: {
ruleExecutionLogger,
bulkCreate,
completeRule,
tuple,
mergeStrategy,
inputIndex,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
aggregatableTimestampField,
exceptionFilter,
unprocessedExceptions,
alertTimestampOverride,
publicBaseUrl,
alertWithSuppression,
intendedTimestamp,
},
services,
params,
spaceId,
state,
} = execOptions;
ruleExecutionLogger,
bulkCreate,
completeRule,
tuple,
mergeStrategy,
inputIndex,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
aggregatableTimestampField,
exceptionFilter,
unprocessedExceptions,
alertTimestampOverride,
publicBaseUrl,
intendedTimestamp,
} = sharedParams;
const isLoggedRequestsEnabled = Boolean(state?.isLoggedRequestsEnabled);
const loggedRequests: RulePreviewLoggedRequest[] = [];
@ -287,17 +282,13 @@ export const createNewTermsAlertType = (
if (isAlertSuppressionActive) {
bulkCreateResult = await bulkCreateSuppressedNewTermsAlertsInMemory({
sharedParams,
eventsAndTerms: eventAndTermsChunk,
toReturn: result,
wrapHits,
bulkCreate,
services,
ruleExecutionLogger,
tuple,
alertSuppression: params.alertSuppression,
wrapSuppressedHits,
alertTimestampOverride,
alertWithSuppression,
experimentalFeatures,
});
} else {
@ -327,6 +318,7 @@ export const createNewTermsAlertType = (
// it uses paging through composite aggregation
if (params.newTermsFields.length > 1) {
const bulkCreateResult = await multiTermsComposite({
sharedParams,
filterArgs,
buckets: bucketsForField,
params,
@ -335,7 +327,6 @@ export const createNewTermsAlertType = (
services,
result,
logger,
runOpts: execOptions.runOpts,
afterKey,
createAlertsHook,
isAlertSuppressionActive,

14
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts
View file

@ -29,7 +29,11 @@ import {
} from '../utils/utils';
import type { GenericBulkCreateResponse } from '../utils/bulk_create_with_suppression';
import type { RuleServices, SearchAfterAndBulkCreateReturnType, RunOpts } from '../types';
import type {
RuleServices,
SearchAfterAndBulkCreateReturnType,
SecuritySharedParams,
} from '../types';
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
import * as i18n from '../translations';
@ -40,6 +44,7 @@ import * as i18n from '../translations';
const BATCH_SIZE = 500;
interface MultiTermsCompositeArgsBase {
sharedParams: SecuritySharedParams<NewTermsRuleParams>;
filterArgs: GetFilterArgs;
buckets: Array<{
doc_count: number;
@ -51,7 +56,6 @@ interface MultiTermsCompositeArgsBase {
services: RuleServices;
result: SearchAfterAndBulkCreateReturnType;
logger: Logger;
runOpts: RunOpts<NewTermsRuleParams>;
afterKey: Record<string, string | number | null> | undefined;
createAlertsHook: CreateAlertsHook;
isAlertSuppressionActive: boolean;
@ -79,6 +83,7 @@ type MultiTermsCompositeResult =
* It pages through though all 10,000 results from phase1 until maxSize alerts found
*/
const multiTermsCompositeNonRetryable = async ({
sharedParams,
filterArgs,
buckets,
params,
@ -87,7 +92,6 @@ const multiTermsCompositeNonRetryable = async ({
services,
result,
logger,
runOpts,
afterKey,
createAlertsHook,
batchSize,
@ -101,7 +105,7 @@ const multiTermsCompositeNonRetryable = async ({
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
} = runOpts;
} = sharedParams;
const loggedRequests: RulePreviewLoggedRequest[] = [];
@ -252,7 +256,7 @@ export const multiTermsComposite = async (
args: MultiTermsCompositeArgsBase
): Promise<MultiTermsCompositeResult> => {
let retryBatchSize = BATCH_SIZE;
const ruleExecutionLogger = args.runOpts.ruleExecutionLogger;
const ruleExecutionLogger = args.sharedParams.ruleExecutionLogger;
return pRetry(
async (retryCount) => {
try {

23
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/bulk_create_unsuppressed_alerts.ts
View file

@ -7,7 +7,11 @@
import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
import type { RunOpts, SearchAfterAndBulkCreateReturnType, RuleServices } from '../../types';
import type {
SecuritySharedParams,
SearchAfterAndBulkCreateReturnType,
RuleServices,
} from '../../types';
import type { UnifiedQueryRuleParams } from '../../../rule_schema';
import type { BuildReasonMessage } from '../../utils/reason_formatters';
@ -15,7 +19,7 @@ import { searchAfterAndBulkCreate } from '../../utils/search_after_bulk_create';
import type { ITelemetryEventsSender } from '../../../../telemetry/sender';
type BulkCreateUnsuppressedAlerts = (params: {
runOpts: RunOpts<UnifiedQueryRuleParams>;
sharedParams: SecuritySharedParams<UnifiedQueryRuleParams>;
size: number;
groupByFields: string[];
buildReasonMessage: BuildReasonMessage;
@ -33,28 +37,19 @@ export const bulkCreateUnsuppressedAlerts: BulkCreateUnsuppressedAlerts = async
size,
groupByFields,
buildReasonMessage,
runOpts,
sharedParams,
filter,
services,
eventsTelemetry,
}) => {
const bulkCreatedResult = await searchAfterAndBulkCreate({
tuple: { ...runOpts.tuple, maxSignals: size },
exceptionsList: [],
sharedParams,
services,
listClient: runOpts.listClient,
ruleExecutionLogger: runOpts.ruleExecutionLogger,
eventsTelemetry,
inputIndexPattern: runOpts.inputIndex,
pageSize: runOpts.searchAfterSize,
filter,
buildReasonMessage,
bulkCreate: runOpts.bulkCreate,
wrapHits: runOpts.wrapHits,
runtimeMappings: runOpts.runtimeMappings,
primaryTimestamp: runOpts.primaryTimestamp,
secondaryTimestamp: runOpts.secondaryTimestamp,
additionalFilters: buildMissingFieldsFilter(groupByFields),
maxSignalsOverride: size,
});
return bulkCreatedResult;

70
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts
View file

@ -11,7 +11,11 @@ import type { estypes } from '@elastic/elasticsearch';
import { withSecuritySpan } from '../../../../../utils/with_security_span';
import { buildTimeRangeFilter } from '../../utils/build_events_query';
import type { RuleServices, RunOpts, SearchAfterAndBulkCreateReturnType } from '../../types';
import type {
RuleServices,
SecuritySharedParams,
SearchAfterAndBulkCreateReturnType,
} from '../../types';
import {
addToSearchAfterReturn,
getUnprocessedExceptionsWarnings,
@ -41,9 +45,8 @@ export interface BucketHistory {
}
export interface GroupAndBulkCreateParams {
runOpts: RunOpts<UnifiedQueryRuleParams>;
sharedParams: SecuritySharedParams<UnifiedQueryRuleParams>;
services: RuleServices;
spaceId: string;
filter: estypes.QueryDslQueryContainer;
buildReasonMessage: BuildReasonMessage;
bucketHistory?: BucketHistory[];
@ -122,9 +125,8 @@ export const filterBucketHistory = ({
};
export const groupAndBulkCreate = async ({
runOpts,
sharedParams,
services,
spaceId,
filter,
buildReasonMessage,
bucketHistory,
@ -134,7 +136,7 @@ export const groupAndBulkCreate = async ({
isLoggedRequestsEnabled,
}: GroupAndBulkCreateParams): Promise<GroupAndBulkCreateReturnType> => {
return withSecuritySpan('groupAndBulkCreate', async () => {
const tuple = runOpts.tuple;
const tuple = sharedParams.tuple;
const filteredBucketHistory = filterBucketHistory({
bucketHistory: bucketHistory ?? [],
@ -158,7 +160,7 @@ export const groupAndBulkCreate = async ({
},
};
const exceptionsWarning = getUnprocessedExceptionsWarnings(runOpts.unprocessedExceptions);
const exceptionsWarning = getUnprocessedExceptionsWarnings(sharedParams.unprocessedExceptions);
if (exceptionsWarning) {
toReturn.warningMessages.push(exceptionsWarning);
}
@ -170,37 +172,37 @@ export const groupAndBulkCreate = async ({
const bucketHistoryFilter = buildBucketHistoryFilter({
bucketHistory: filteredBucketHistory,
primaryTimestamp: runOpts.primaryTimestamp,
secondaryTimestamp: runOpts.secondaryTimestamp,
primaryTimestamp: sharedParams.primaryTimestamp,
secondaryTimestamp: sharedParams.secondaryTimestamp,
from: tuple.from,
});
// if we do not suppress alerts for docs with missing values, we will create aggregation for null missing buckets
const suppressOnMissingFields =
(runOpts.completeRule.ruleParams.alertSuppression?.missingFieldsStrategy ??
(sharedParams.completeRule.ruleParams.alertSuppression?.missingFieldsStrategy ??
DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) ===
AlertSuppressionMissingFieldsStrategyEnum.suppress;
const groupingAggregation = buildGroupByFieldAggregation({
groupByFields,
maxSignals: tuple.maxSignals,
aggregatableTimestampField: runOpts.aggregatableTimestampField,
aggregatableTimestampField: sharedParams.aggregatableTimestampField,
missingBucket: suppressOnMissingFields,
});
const eventsSearchParams = {
aggregations: groupingAggregation,
searchAfterSortIds: undefined,
index: runOpts.inputIndex,
index: sharedParams.inputIndex,
from: tuple.from.toISOString(),
to: tuple.to.toISOString(),
services,
ruleExecutionLogger: runOpts.ruleExecutionLogger,
ruleExecutionLogger: sharedParams.ruleExecutionLogger,
filter,
pageSize: 0,
primaryTimestamp: runOpts.primaryTimestamp,
secondaryTimestamp: runOpts.secondaryTimestamp,
runtimeMappings: runOpts.runtimeMappings,
primaryTimestamp: sharedParams.primaryTimestamp,
secondaryTimestamp: sharedParams.secondaryTimestamp,
runtimeMappings: sharedParams.runtimeMappings,
additionalFilters: bucketHistoryFilter,
loggedRequestsConfig: isLoggedRequestsEnabled
? {
@ -232,7 +234,7 @@ export const groupAndBulkCreate = async ({
const unsuppressedResult = await bulkCreateUnsuppressedAlerts({
groupByFields,
size: maxUnsuppressedCount,
runOpts,
sharedParams,
buildReasonMessage,
eventsTelemetry,
filter,
@ -266,40 +268,40 @@ export const groupAndBulkCreate = async ({
const wrappedAlerts = wrapSuppressedAlerts({
suppressionBuckets,
spaceId,
completeRule: runOpts.completeRule,
mergeStrategy: runOpts.mergeStrategy,
indicesToQuery: runOpts.inputIndex,
publicBaseUrl: runOpts.publicBaseUrl,
spaceId: sharedParams.spaceId,
completeRule: sharedParams.completeRule,
mergeStrategy: sharedParams.mergeStrategy,
indicesToQuery: sharedParams.inputIndex,
publicBaseUrl: sharedParams.publicBaseUrl,
buildReasonMessage,
alertTimestampOverride: runOpts.alertTimestampOverride,
ruleExecutionLogger: runOpts.ruleExecutionLogger,
intendedTimestamp: runOpts.intendedTimestamp,
alertTimestampOverride: sharedParams.alertTimestampOverride,
ruleExecutionLogger: sharedParams.ruleExecutionLogger,
intendedTimestamp: sharedParams.intendedTimestamp,
});
const suppressionDuration = runOpts.completeRule.ruleParams.alertSuppression?.duration;
const suppressionDuration = sharedParams.completeRule.ruleParams.alertSuppression?.duration;
if (suppressionDuration) {
const suppressionWindow = `now-${suppressionDuration.value}${suppressionDuration.unit}`;
const bulkCreateResult = await bulkCreateWithSuppression({
alertWithSuppression: runOpts.alertWithSuppression,
ruleExecutionLogger: runOpts.ruleExecutionLogger,
sharedParams,
wrappedDocs: wrappedAlerts,
services,
suppressionWindow,
alertTimestampOverride: runOpts.alertTimestampOverride,
experimentalFeatures,
ruleType: 'query',
});
addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult });
runOpts.ruleExecutionLogger.debug(`created ${bulkCreateResult.createdItemsCount} signals`);
sharedParams.ruleExecutionLogger.debug(
`created ${bulkCreateResult.createdItemsCount} signals`
);
} else {
const bulkCreateResult = await runOpts.bulkCreate(
const bulkCreateResult = await sharedParams.bulkCreate(
wrappedAlerts,
undefined,
createEnrichEventsFunction({
services,
logger: runOpts.ruleExecutionLogger,
logger: sharedParams.ruleExecutionLogger,
})
);
addToSearchAfterReturn({
@ -309,7 +311,9 @@ export const groupAndBulkCreate = async ({
suppressedItemsCount: getNumberOfSuppressedAlerts(bulkCreateResult.createdItems, []),
},
});
runOpts.ruleExecutionLogger.debug(`created ${bulkCreateResult.createdItemsCount} signals`);
sharedParams.ruleExecutionLogger.debug(
`created ${bulkCreateResult.createdItemsCount} signals`
);
}
const newBucketHistory: BucketHistory[] = buckets.map((bucket) => {

4
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts
View file

@ -75,7 +75,6 @@ describe('Custom Query Alerts', () => {
scheduleNotificationResponseActionsService: () => null,
experimentalFeatures: allowedExperimentalValues,
logger,
version: '1.0.0',
id: QUERY_RULE_TYPE_ID,
name: 'Custom Query Rule',
})
@ -124,7 +123,6 @@ describe('Custom Query Alerts', () => {
scheduleNotificationResponseActionsService: () => null,
experimentalFeatures: allowedExperimentalValues,
logger,
version: '1.0.0',
id: QUERY_RULE_TYPE_ID,
name: 'Custom Query Rule',
})
@ -177,7 +175,6 @@ describe('Custom Query Alerts', () => {
scheduleNotificationResponseActionsService: () => null,
experimentalFeatures: allowedExperimentalValues,
logger,
version: '1.0.0',
id: QUERY_RULE_TYPE_ID,
name: 'Custom Query Rule',
})
@ -244,7 +241,6 @@ describe('Custom Query Alerts', () => {
scheduleNotificationResponseActionsService: () => null,
experimentalFeatures: allowedExperimentalValues,
logger,
version: '1.0.0',
id: QUERY_RULE_TYPE_ID,
name: 'Custom Query Rule',
})

7
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts
View file

@ -25,7 +25,6 @@ export const createQueryAlertType = (
const {
eventsTelemetry,
experimentalFeatures,
version,
scheduleNotificationResponseActionsService,
licensing,
id,
@ -70,14 +69,12 @@ export const createQueryAlertType = (
category: DEFAULT_APP_CATEGORIES.security.id,
producer: SERVER_APP_ID,
async executor(execOptions) {
const { runOpts, services, spaceId, state } = execOptions;
const { sharedParams, services, state } = execOptions;
return queryExecutor({
runOpts,
sharedParams,
experimentalFeatures,
eventsTelemetry,
services,
version,
spaceId,
bucketHistory: state.suppressionGroupHistory,
licensing,
scheduleNotificationResponseActionsService,

31
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts
View file

@ -22,32 +22,28 @@ import type { UnifiedQueryRuleParams } from '../../rule_schema';
import type { ExperimentalFeatures } from '../../../../../common/experimental_features';
import { buildReasonMessageForQueryAlert } from '../utils/reason_formatters';
import { withSecuritySpan } from '../../../../utils/with_security_span';
import type { CreateRuleOptions, RunOpts } from '../types';
import type { CreateRuleOptions, SecuritySharedParams } from '../types';
export const queryExecutor = async ({
runOpts,
sharedParams,
experimentalFeatures,
eventsTelemetry,
services,
version,
spaceId,
bucketHistory,
scheduleNotificationResponseActionsService,
licensing,
isLoggedRequestsEnabled,
}: {
runOpts: RunOpts<UnifiedQueryRuleParams>;
sharedParams: SecuritySharedParams<UnifiedQueryRuleParams>;
experimentalFeatures: ExperimentalFeatures;
eventsTelemetry: ITelemetryEventsSender | undefined;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
version: string;
spaceId: string;
bucketHistory?: BucketHistory[];
scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService'];
licensing: LicensingPluginSetup;
isLoggedRequestsEnabled: boolean;
}) => {
const completeRule = runOpts.completeRule;
const { completeRule } = sharedParams;
const ruleParams = completeRule.ruleParams;
return withSecuritySpan('queryExecutor', async () => {
@ -58,8 +54,8 @@ export const queryExecutor = async ({
query: ruleParams.query,
savedId: ruleParams.savedId,
services,
index: runOpts.inputIndex,
exceptionFilter: runOpts.exceptionFilter,
index: sharedParams.inputIndex,
exceptionFilter: sharedParams.exceptionFilter,
loadFields: true,
});
@ -70,9 +66,8 @@ export const queryExecutor = async ({
// TODO: replace this with getIsAlertSuppressionActive function
ruleParams.alertSuppression?.groupBy != null && hasPlatinumLicense
? await groupAndBulkCreate({
runOpts,
sharedParams,
services,
spaceId,
filter: esFilter,
buildReasonMessage: buildReasonMessageForQueryAlert,
bucketHistory,
@ -83,21 +78,11 @@ export const queryExecutor = async ({
})
: {
...(await searchAfterAndBulkCreate({
tuple: runOpts.tuple,
exceptionsList: runOpts.unprocessedExceptions,
sharedParams,
services,
listClient: runOpts.listClient,
ruleExecutionLogger: runOpts.ruleExecutionLogger,
eventsTelemetry,
inputIndexPattern: runOpts.inputIndex,
pageSize: runOpts.searchAfterSize,
filter: esFilter,
buildReasonMessage: buildReasonMessageForQueryAlert,
bulkCreate: runOpts.bulkCreate,
wrapHits: runOpts.wrapHits,
runtimeMappings: runOpts.runtimeMappings,
primaryTimestamp: runOpts.primaryTimestamp,
secondaryTimestamp: runOpts.secondaryTimestamp,
isLoggedRequestsEnabled,
})),
state: { isLoggedRequestsEnabled },

46
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts
View file

@ -15,10 +15,9 @@ import type { SearchHit } from '@elastic/elasticsearch/lib/api/types';
import { buildReasonMessageForThresholdAlert } from '../utils/reason_formatters';
import type { ThresholdBucket } from './types';
import type { RunOpts } from '../types';
import type { CompleteRule, ThresholdRuleParams } from '../../rule_schema';
import type { SecuritySharedParams } from '../types';
import type { ThresholdRuleParams } from '../../rule_schema';
import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts';
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
import { bulkCreateWithSuppression } from '../utils/bulk_create_with_suppression';
import type { GenericBulkCreateResponse } from '../utils/bulk_create_with_suppression';
import { wrapSuppressedThresholdALerts } from './wrap_suppressed_threshold_alerts';
@ -26,16 +25,12 @@ import { transformBulkCreatedItemsToHits } from './utils';
import type { ExperimentalFeatures } from '../../../../../common';
interface BulkCreateSuppressedThresholdAlertsParams {
sharedParams: SecuritySharedParams<ThresholdRuleParams>;
buckets: ThresholdBucket[];
completeRule: CompleteRule<ThresholdRuleParams>;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
inputIndexPattern: string[];
startedAt: Date;
from: Date;
to: Date;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
spaceId: string;
runOpts: RunOpts<ThresholdRuleParams>;
experimentalFeatures: ExperimentalFeatures;
}
@ -45,23 +40,28 @@ interface BulkCreateSuppressedThresholdAlertsParams {
* and unsuppressed alerts, needed to create correct threshold history
*/
export const bulkCreateSuppressedThresholdAlerts = async ({
sharedParams,
buckets,
completeRule,
services,
inputIndexPattern,
startedAt,
from,
to,
ruleExecutionLogger,
spaceId,
runOpts,
experimentalFeatures,
}: BulkCreateSuppressedThresholdAlertsParams): Promise<{
bulkCreateResult: GenericBulkCreateResponse<BaseFieldsLatest & SuppressionFieldsLatest>;
unsuppressedAlerts: Array<SearchHit<unknown>>;
}> => {
const {
completeRule,
mergeStrategy,
inputIndex,
alertTimestampOverride,
ruleExecutionLogger,
publicBaseUrl,
intendedTimestamp,
} = sharedParams;
const ruleParams = completeRule.ruleParams;
const suppressionDuration = runOpts.completeRule.ruleParams.alertSuppression?.duration;
const suppressionDuration = completeRule.ruleParams.alertSuppression?.duration;
if (!suppressionDuration) {
throw Error('Suppression duration can not be empty');
}
@ -70,30 +70,28 @@ export const bulkCreateSuppressedThresholdAlerts = async ({
const wrappedAlerts = wrapSuppressedThresholdALerts({
buckets,
spaceId,
spaceId: sharedParams.spaceId,
completeRule,
mergeStrategy: runOpts.mergeStrategy,
indicesToQuery: runOpts.inputIndex,
mergeStrategy,
indicesToQuery: inputIndex,
buildReasonMessage: buildReasonMessageForThresholdAlert,
alertTimestampOverride: runOpts.alertTimestampOverride,
alertTimestampOverride,
ruleExecutionLogger,
publicBaseUrl: runOpts.publicBaseUrl,
inputIndex: inputIndexPattern.join(','),
publicBaseUrl,
inputIndex: inputIndex.join(','),
startedAt,
from,
to,
suppressionWindow,
threshold: ruleParams.threshold,
intendedTimestamp: runOpts.intendedTimestamp,
intendedTimestamp,
});
const bulkCreateResult = await bulkCreateWithSuppression({
alertWithSuppression: runOpts.alertWithSuppression,
ruleExecutionLogger: runOpts.ruleExecutionLogger,
sharedParams,
wrappedDocs: wrappedAlerts,
services,
suppressionWindow,
alertTimestampOverride: runOpts.alertTimestampOverride,
experimentalFeatures,
});

41
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts
View file

@ -19,7 +19,7 @@ import { validateIndexPatterns } from '../utils';
export const createThresholdAlertType = (
createOptions: CreateRuleOptions
): SecurityAlertType<ThresholdRuleParams, ThresholdAlertState, {}, 'default'> => {
const { version, licensing, experimentalFeatures, scheduleNotificationResponseActionsService } =
const { licensing, experimentalFeatures, scheduleNotificationResponseActionsService } =
createOptions;
return {
id: THRESHOLD_RULE_TYPE_ID,
@ -60,47 +60,12 @@ export const createThresholdAlertType = (
category: DEFAULT_APP_CATEGORIES.security.id,
producer: SERVER_APP_ID,
async executor(execOptions) {
const {
runOpts: {
bulkCreate,
completeRule,
tuple,
wrapHits,
ruleDataClient,
inputIndex,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
ruleExecutionLogger,
aggregatableTimestampField,
exceptionFilter,
unprocessedExceptions,
},
services,
startedAt,
state,
spaceId,
} = execOptions;
const { sharedParams, services, startedAt, state } = execOptions;
const result = await thresholdExecutor({
completeRule,
tuple,
ruleExecutionLogger,
sharedParams,
services,
version,
startedAt,
state,
bulkCreate,
wrapHits,
ruleDataClient,
inputIndex,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
aggregatableTimestampField,
exceptionFilter,
unprocessedExceptions,
spaceId,
runOpts: execOptions.runOpts,
licensing,
experimentalFeatures,
scheduleNotificationResponseActionsService,

112
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.test.ts
View file

@ -12,25 +12,20 @@ import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks
import { licensingMock } from '@kbn/licensing-plugin/server/mocks';
import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
import { thresholdExecutor } from './threshold';
import { getThresholdRuleParams, getCompleteRuleMock } from '../../rule_schema/mocks';
import { getThresholdRuleParams } from '../../rule_schema/mocks';
import { sampleEmptyAggsSearchResults } from '../__mocks__/es_results';
import { getThresholdTermsHash } from './utils';
import type { ThresholdRuleParams } from '../../rule_schema';
import { createRuleDataClientMock } from '@kbn/rule-registry-plugin/server/rule_data_client/rule_data_client.mock';
import { TIMESTAMP } from '@kbn/rule-data-utils';
import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
import type { RunOpts } from '../types';
import type { ExperimentalFeatures } from '../../../../../common';
import { getSharedParamsMock } from '../__mocks__/shared_params';
jest.mock('../utils/get_filter', () => ({ getFilter: jest.fn() }));
describe('threshold_executor', () => {
let alertServices: RuleExecutorServicesMock;
let ruleExecutionLogger: ReturnType<typeof ruleExecutionLogMock.forExecutors.create>;
let mockScheduledNotificationResponseAction: jest.Mock;
const version = '8.0.0';
const params = getThresholdRuleParams();
const thresholdCompleteRule = getCompleteRuleMock<ThresholdRuleParams>(params);
const tuple = {
from: dateMath.parse(params.from)!,
to: dateMath.parse(params.to)!,
@ -38,6 +33,27 @@ describe('threshold_executor', () => {
};
const licensing = licensingMock.createSetup();
const sharedParams = getSharedParamsMock({
ruleParams: params,
rewrites: {
tuple,
bulkCreate: jest.fn().mockImplementation((hits) => ({
errors: [],
success: true,
bulkCreateDuration: '0',
createdItemsCount: 0,
createdItems: [],
})),
},
});
const ruleExecutionLogger: ReturnType<typeof ruleExecutionLogMock.forExecutors.create> =
ruleExecutionLogMock.forExecutors.create({
ruleId: sharedParams.completeRule.alertId,
ruleUuid: sharedParams.completeRule.ruleParams.ruleId,
ruleName: sharedParams.completeRule.ruleConfig.name,
ruleType: sharedParams.completeRule.ruleConfig.ruleTypeId,
});
sharedParams.ruleExecutionLogger = ruleExecutionLogger;
beforeEach(() => {
alertServices = alertsMock.createRuleExecutorServices();
alertServices.scopedClusterClient.asCurrentUser.search.mockResolvedValue(
@ -48,18 +64,11 @@ describe('threshold_executor', () => {
},
})
);
ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create({
ruleId: thresholdCompleteRule.alertId,
ruleUuid: thresholdCompleteRule.ruleParams.ruleId,
ruleName: thresholdCompleteRule.ruleConfig.name,
ruleType: thresholdCompleteRule.ruleConfig.ruleTypeId,
});
mockScheduledNotificationResponseAction = jest.fn();
});
describe('thresholdExecutor', () => {
it('should clean up any signal history that has fallen outside the window when state is initialized', async () => {
const ruleDataClientMock = createRuleDataClientMock();
const terms1 = [
{
field: 'host.name',
@ -88,30 +97,10 @@ describe('threshold_executor', () => {
},
};
const response = await thresholdExecutor({
completeRule: thresholdCompleteRule,
tuple,
sharedParams,
services: alertServices,
state,
version,
ruleExecutionLogger,
startedAt: new Date(),
bulkCreate: jest.fn().mockImplementation((hits) => ({
errors: [],
success: true,
bulkCreateDuration: '0',
createdItemsCount: 0,
createdItems: [],
})),
wrapHits: jest.fn(),
ruleDataClient: ruleDataClientMock,
runtimeMappings: {},
inputIndex: ['auditbeat-*'],
primaryTimestamp: TIMESTAMP,
aggregatableTimestampField: TIMESTAMP,
exceptionFilter: undefined,
unprocessedExceptions: [],
spaceId: 'default',
runOpts: {} as RunOpts<ThresholdRuleParams>,
licensing,
experimentalFeatures: {} as ExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction,
@ -125,7 +114,6 @@ describe('threshold_executor', () => {
});
it('should log a warning if unprocessedExceptions is not empty', async () => {
const ruleDataClientMock = createRuleDataClientMock();
const terms1 = [
{
field: 'host.name',
@ -154,30 +142,13 @@ describe('threshold_executor', () => {
},
};
const result = await thresholdExecutor({
completeRule: thresholdCompleteRule,
tuple,
sharedParams: {
...sharedParams,
unprocessedExceptions: [getExceptionListItemSchemaMock()],
},
services: alertServices,
state,
version,
ruleExecutionLogger,
startedAt: new Date(),
bulkCreate: jest.fn().mockImplementation((hits) => ({
errors: [],
success: true,
bulkCreateDuration: '0',
createdItemsCount: 0,
createdItems: [],
})),
wrapHits: jest.fn(),
ruleDataClient: ruleDataClientMock,
runtimeMappings: {},
inputIndex: ['auditbeat-*'],
primaryTimestamp: TIMESTAMP,
aggregatableTimestampField: TIMESTAMP,
exceptionFilter: undefined,
unprocessedExceptions: [getExceptionListItemSchemaMock()],
spaceId: 'default',
runOpts: {} as RunOpts<ThresholdRuleParams>,
licensing,
experimentalFeatures: {} as ExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction,
@ -189,36 +160,15 @@ describe('threshold_executor', () => {
]);
});
it('should call scheduleNotificationResponseActionsService', async () => {
const ruleDataClientMock = createRuleDataClientMock();
const state = {
initialized: true,
signalHistory: {},
};
const result = await thresholdExecutor({
completeRule: thresholdCompleteRule,
tuple,
sharedParams,
services: alertServices,
state,
version,
ruleExecutionLogger,
startedAt: new Date(),
bulkCreate: jest.fn().mockImplementation((hits) => ({
errors: [],
success: true,
bulkCreateDuration: '0',
createdItemsCount: 0,
createdItems: [],
})),
wrapHits: jest.fn(),
ruleDataClient: ruleDataClientMock,
runtimeMappings: {},
inputIndex: ['auditbeat-*'],
primaryTimestamp: TIMESTAMP,
aggregatableTimestampField: TIMESTAMP,
exceptionFilter: undefined,
unprocessedExceptions: [getExceptionListItemSchemaMock()],
spaceId: 'default',
runOpts: {} as RunOpts<ThresholdRuleParams>,
licensing,
experimentalFeatures: {} as ExperimentalFeatures,
scheduleNotificationResponseActionsService: mockScheduledNotificationResponseAction,
@ -226,7 +176,7 @@ describe('threshold_executor', () => {
expect(mockScheduledNotificationResponseAction).toBeCalledWith({
signals: result.createdSignals,
signalsCount: result.createdSignalsCount,
responseActions: thresholdCompleteRule.ruleParams.responseActions,
responseActions: sharedParams.completeRule.ruleParams.responseActions,
});
});
});

68
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts
View file

@ -8,8 +8,6 @@
import { isEmpty } from 'lodash';
import { firstValueFrom } from 'rxjs';
import type { estypes } from '@elastic/elasticsearch';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server';
import type {
@ -17,9 +15,7 @@ import type {
AlertInstanceState,
RuleExecutorServices,
} from '@kbn/alerting-plugin/server';
import type { IRuleDataClient } from '@kbn/rule-registry-plugin/server';
import type { Filter } from '@kbn/es-query';
import type { CompleteRule, ThresholdRuleParams } from '../../rule_schema';
import type { ThresholdRuleParams } from '../../rule_schema';
import { getFilter } from '../utils/get_filter';
import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals';
import { findThresholdSignals } from './find_threshold_signals';
@ -28,11 +24,8 @@ import { getThresholdSignalHistory } from './get_threshold_signal_history';
import { bulkCreateSuppressedThresholdAlerts } from './bulk_create_suppressed_threshold_alerts';
import type {
BulkCreate,
RuleRangeTuple,
SearchAfterAndBulkCreateReturnType,
WrapHits,
RunOpts,
SecuritySharedParams,
CreateRuleOptions,
} from '../types';
import type { ThresholdAlertState, ThresholdSignalHistory } from './types';
@ -43,57 +36,42 @@ import {
} from '../utils/utils';
import { withSecuritySpan } from '../../../../utils/with_security_span';
import { buildThresholdSignalHistory } from './build_signal_history';
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
import { getSignalHistory, transformBulkCreatedItemsToHits } from './utils';
import type { ExperimentalFeatures } from '../../../../../common';
export const thresholdExecutor = async ({
inputIndex,
runtimeMappings,
completeRule,
tuple,
ruleExecutionLogger,
sharedParams,
services,
version,
startedAt,
state,
bulkCreate,
wrapHits,
ruleDataClient,
primaryTimestamp,
secondaryTimestamp,
aggregatableTimestampField,
exceptionFilter,
unprocessedExceptions,
spaceId,
runOpts,
licensing,
experimentalFeatures,
scheduleNotificationResponseActionsService,
}: {
inputIndex: string[];
runtimeMappings: estypes.MappingRuntimeFields | undefined;
completeRule: CompleteRule<ThresholdRuleParams>;
tuple: RuleRangeTuple;
sharedParams: SecuritySharedParams<ThresholdRuleParams>;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
version: string;
startedAt: Date;
state: ThresholdAlertState;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
ruleDataClient: IRuleDataClient;
primaryTimestamp: string;
secondaryTimestamp?: string;
aggregatableTimestampField: string;
exceptionFilter: Filter | undefined;
unprocessedExceptions: ExceptionListItemSchema[];
spaceId: string;
runOpts: RunOpts<ThresholdRuleParams>;
licensing: LicensingPluginSetup;
experimentalFeatures: ExperimentalFeatures;
scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService'];
}): Promise<SearchAfterAndBulkCreateReturnType & { state: ThresholdAlertState }> => {
const {
completeRule,
unprocessedExceptions,
tuple,
spaceId,
aggregatableTimestampField,
ruleDataClient,
inputIndex,
exceptionFilter,
ruleExecutionLogger,
primaryTimestamp,
secondaryTimestamp,
runtimeMappings,
bulkCreate,
wrapHits,
} = sharedParams;
const result = createSearchAfterReturnType();
const ruleParams = completeRule.ruleParams;
const isLoggedRequestsEnabled = Boolean(state?.isLoggedRequestsEnabled);
@ -164,16 +142,12 @@ export const thresholdExecutor = async ({
if (alertSuppression?.duration && hasPlatinumLicense) {
const suppressedResults = await bulkCreateSuppressedThresholdAlerts({
sharedParams,
buckets,
completeRule,
services,
inputIndexPattern: inputIndex,
startedAt,
from: tuple.from.toDate(),
to: tuple.to.toDate(),
ruleExecutionLogger,
spaceId,
runOpts,
experimentalFeatures,
});
const createResult = suppressedResults.bulkCreateResult;

38
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts
View file

@ -46,14 +46,12 @@ import type { IRuleExecutionLogForExecutors, IRuleMonitoringService } from '../r
import type { RefreshTypes } from '../types';
import type { Status } from '../../../../common/api/detection_engine';
import type { BaseHit, SearchTypes, EqlSequence } from '../../../../common/detection_engine/types';
import type { BaseHit, SearchTypes } from '../../../../common/detection_engine/types';
import type { GenericBulkCreateResponse } from './factories';
import type { BuildReasonMessage } from './utils/reason_formatters';
import type {
BaseFieldsLatest,
DetectionAlert,
EqlBuildingBlockFieldsLatest,
EqlShellFieldsLatest,
WrappedFieldsLatest,
} from '../../../../common/api/detection_engine/model/alerts';
import type {
@ -80,7 +78,7 @@ export interface SecurityAlertTypeReturnValue<TState extends RuleTypeState> {
loggedRequests?: RulePreviewLoggedRequest[];
}
export interface RunOpts<TParams extends RuleParams> {
export interface SecuritySharedParams<TParams extends RuleParams = RuleParams> {
completeRule: CompleteRule<TParams>;
tuple: {
to: Moment;
@ -92,7 +90,6 @@ export interface RunOpts<TParams extends RuleParams> {
searchAfterSize: number;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
wrapSequences: WrapSequences;
ruleDataClient: IRuleDataClient;
inputIndex: string[];
runtimeMappings: estypes.MappingRuntimeFields | undefined;
@ -108,6 +105,7 @@ export interface RunOpts<TParams extends RuleParams> {
publicBaseUrl: string | undefined;
experimentalFeatures?: ExperimentalFeatures;
intendedTimestamp: Date | undefined;
spaceId: string;
}
export type SecurityAlertType<
@ -128,7 +126,7 @@ export type SecurityAlertType<
WithoutReservedActionGroups<TActionGroupIds, never>
> & {
services: PersistenceServices;
runOpts: RunOpts<TParams>;
sharedParams: SecuritySharedParams<TParams>;
}
) => Promise<
SearchAfterAndBulkCreateReturnType & {
@ -164,7 +162,6 @@ export interface CreateRuleOptions {
logger: Logger;
ml?: SetupPlugins['ml'];
eventsTelemetry?: ITelemetryEventsSender | undefined;
version: string;
licensing: LicensingPluginSetup;
scheduleNotificationResponseActionsService: (params: ScheduleNotificationActions) => void;
}
@ -353,13 +350,6 @@ export type WrapSuppressedHits = (
buildReasonMessage: BuildReasonMessage
) => Array<WrappedFieldsLatest<BaseFieldsLatest & SuppressionFieldsLatest>>;
export type WrapSequences = (
sequences: Array<EqlSequence<SignalSource>>,
buildReasonMessage: BuildReasonMessage
) => Array<
WrappedFieldsLatest<EqlShellFieldsLatest> | WrappedFieldsLatest<EqlBuildingBlockFieldsLatest>
>;
export type RuleServices = RuleExecutorServices<
AlertInstanceState,
AlertInstanceContext,
@ -367,30 +357,20 @@ export type RuleServices = RuleExecutorServices<
>;
export interface SearchAfterAndBulkCreateParams {
tuple: {
to: moment.Moment;
from: moment.Moment;
maxSignals: number;
};
sharedParams: SecuritySharedParams;
services: RuleServices;
listClient: ListClient;
exceptionsList: ExceptionListItemSchema[];
ruleExecutionLogger: IRuleExecutionLogForExecutors;
eventsTelemetry: ITelemetryEventsSender | undefined;
inputIndexPattern: string[];
pageSize: number;
filter: estypes.QueryDslQueryContainer;
buildReasonMessage: BuildReasonMessage;
enrichment?: SignalsEnrichment;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
trackTotalHits?: boolean;
sortOrder?: estypes.SortOrder;
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
additionalFilters?: estypes.QueryDslQueryContainer[];
isLoggedRequestsEnabled?: boolean;
/**
* If defined, will override the value of max_signals found in sharedParams.tuple
*/
maxSignalsOverride?: number;
}
export interface SearchAfterAndBulkCreateReturnType {

81
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts
View file

@ -4,7 +4,6 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server';
import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas';
import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types';
@ -15,9 +14,9 @@ import type {
WrapSuppressedHits,
SignalSourceHit,
SignalSource,
SecuritySharedParams,
} from '../types';
import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants';
import type { SharedParams } from './utils';
import { addToSearchAfterReturn, buildShellAlertSuppressionTermsAndFields } from './utils';
import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema';
import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/constants';
@ -35,26 +34,16 @@ import type {
} from '../../../../../common/api/detection_engine/model/alerts';
import { robustGet } from './source_fields_merging/utils/robust_field_access';
import { buildAlertGroupFromSequence } from '../eql/build_alert_group_from_sequence';
import type { EqlRuleParams } from '../../rule_schema';
interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams {
wrapSuppressedHits: WrapSuppressedHits;
alertTimestampOverride: Date | undefined;
alertWithSuppression: SuppressedAlertService;
alertSuppression?: AlertSuppressionCamel;
}
export interface BulkCreateSuppressedAlertsParams
extends Pick<
SearchAfterAndBulkCreateSuppressedAlertsParams,
| 'wrapHits'
| 'bulkCreate'
| 'services'
| 'buildReasonMessage'
| 'ruleExecutionLogger'
| 'tuple'
| 'alertSuppression'
| 'wrapSuppressedHits'
| 'alertWithSuppression'
| 'alertTimestampOverride'
'services' | 'buildReasonMessage' | 'alertSuppression' | 'wrapSuppressedHits' | 'sharedParams'
> {
enrichedEvents: SignalSourceHit[];
toReturn: SearchAfterAndBulkCreateReturnType;
@ -66,21 +55,14 @@ export interface BulkCreateSuppressedAlertsParams
export interface BulkCreateSuppressedSequencesParams
extends Pick<
SearchAfterAndBulkCreateSuppressedAlertsParams,
| 'bulkCreate'
| 'services'
| 'buildReasonMessage'
| 'ruleExecutionLogger'
| 'tuple'
| 'alertSuppression'
| 'alertWithSuppression'
| 'alertTimestampOverride'
'services' | 'buildReasonMessage' | 'alertSuppression'
> {
sharedParams: SecuritySharedParams<EqlRuleParams>;
sequences: Array<EqlHitsSequence<SignalSource>>;
buildingBlockAlerts?: Array<WrappedFieldsLatest<BaseFieldsLatest>>;
toReturn: SearchAfterAndBulkCreateReturnType;
experimentalFeatures: ExperimentalFeatures;
maxNumberOfAlertsMultiplier?: number;
sharedParams: SharedParams;
alertSuppression: AlertSuppressionCamel;
}
/**
@ -89,18 +71,13 @@ export interface BulkCreateSuppressedSequencesParams
* regular alerts will be created for such events without suppression
*/
export const bulkCreateSuppressedAlertsInMemory = async ({
sharedParams,
enrichedEvents,
toReturn,
wrapHits,
bulkCreate,
services,
buildReasonMessage,
ruleExecutionLogger,
tuple,
alertSuppression,
wrapSuppressedHits,
alertWithSuppression,
alertTimestampOverride,
experimentalFeatures,
mergeSourceAndFields = false,
maxNumberOfAlertsMultiplier,
@ -120,23 +97,19 @@ export const bulkCreateSuppressedAlertsInMemory = async ({
mergeSourceAndFields
);
unsuppressibleWrappedDocs = wrapHits(partitionedEvents[1], buildReasonMessage);
unsuppressibleWrappedDocs = sharedParams.wrapHits(partitionedEvents[1], buildReasonMessage);
suppressibleEvents = partitionedEvents[0];
}
const suppressibleWrappedDocs = wrapSuppressedHits(suppressibleEvents, buildReasonMessage);
return executeBulkCreateAlerts({
sharedParams,
suppressibleWrappedDocs,
unsuppressibleWrappedDocs,
toReturn,
bulkCreate,
services,
ruleExecutionLogger,
tuple,
alertSuppression,
alertWithSuppression,
alertTimestampOverride,
experimentalFeatures,
maxNumberOfAlertsMultiplier,
});
@ -148,17 +121,12 @@ export const bulkCreateSuppressedAlertsInMemory = async ({
* regular alerts will be created for such events without suppression
*/
export const bulkCreateSuppressedSequencesInMemory = async ({
sharedParams,
sequences,
toReturn,
bulkCreate,
services,
ruleExecutionLogger,
tuple,
alertSuppression,
buildReasonMessage,
sharedParams,
alertWithSuppression,
alertTimestampOverride,
experimentalFeatures,
maxNumberOfAlertsMultiplier,
}: BulkCreateSuppressedSequencesParams) => {
@ -175,10 +143,10 @@ export const bulkCreateSuppressedSequencesInMemory = async ({
sequences.forEach((sequence) => {
const alertGroupFromSequence = buildAlertGroupFromSequence({
sharedParams,
sequence,
applyOverrides: true,
buildReasonMessage,
...sharedParams,
});
const shellAlert = alertGroupFromSequence.shellAlert;
const buildingBlocks = alertGroupFromSequence.buildingBlocks;
@ -193,17 +161,17 @@ export const bulkCreateSuppressedSequencesInMemory = async ({
unsuppressibleWrappedDocs.push(shellAlert, ...buildingBlocks);
} else {
const wrappedWithSuppressionTerms = buildShellAlertSuppressionTermsAndFields({
sharedParams,
shellAlert,
buildingBlockAlerts: buildingBlocks,
...sharedParams,
});
suppressibleWrappedSequences.push(wrappedWithSuppressionTerms);
}
} else {
const wrappedWithSuppressionTerms = buildShellAlertSuppressionTermsAndFields({
sharedParams,
shellAlert,
buildingBlockAlerts: buildingBlocks,
...sharedParams,
});
suppressibleWrappedSequences.push(wrappedWithSuppressionTerms);
}
@ -211,16 +179,12 @@ export const bulkCreateSuppressedSequencesInMemory = async ({
});
return executeBulkCreateAlerts({
sharedParams,
suppressibleWrappedDocs: suppressibleWrappedSequences,
unsuppressibleWrappedDocs,
toReturn,
bulkCreate,
services,
ruleExecutionLogger,
tuple,
alertSuppression,
alertWithSuppression,
alertTimestampOverride,
experimentalFeatures,
maxNumberOfAlertsMultiplier,
});
@ -229,13 +193,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({
export interface ExecuteBulkCreateAlertsParams<T extends SuppressionFieldsLatest & BaseFieldsLatest>
extends Pick<
SearchAfterAndBulkCreateSuppressedAlertsParams,
| 'bulkCreate'
| 'services'
| 'ruleExecutionLogger'
| 'tuple'
| 'alertSuppression'
| 'alertWithSuppression'
| 'alertTimestampOverride'
'services' | 'sharedParams' | 'alertSuppression'
> {
unsuppressibleWrappedDocs: Array<WrappedFieldsLatest<BaseFieldsLatest>>;
suppressibleWrappedDocs: Array<WrappedFieldsLatest<T>>;
@ -250,19 +208,16 @@ export interface ExecuteBulkCreateAlertsParams<T extends SuppressionFieldsLatest
export const executeBulkCreateAlerts = async <
T extends SuppressionFieldsLatest & BaseFieldsLatest
>({
sharedParams,
unsuppressibleWrappedDocs,
suppressibleWrappedDocs,
toReturn,
bulkCreate,
services,
ruleExecutionLogger,
tuple,
alertSuppression,
alertWithSuppression,
alertTimestampOverride,
experimentalFeatures,
maxNumberOfAlertsMultiplier = MAX_SIGNALS_SUPPRESSION_MULTIPLIER,
}: ExecuteBulkCreateAlertsParams<T>) => {
const { tuple, bulkCreate, ruleExecutionLogger } = sharedParams;
// max signals for suppression includes suppressed and created alerts
// this allows to lift max signals limitation to higher value
// and can detects events beyond default max_signals value
@ -287,12 +242,10 @@ export const executeBulkCreateAlerts = async <
}
const bulkCreateResult = await bulkCreateWithSuppression({
alertWithSuppression,
ruleExecutionLogger,
sharedParams,
wrappedDocs: suppressibleWrappedDocs,
services,
suppressionWindow,
alertTimestampOverride,
isSuppressionPerRuleExecution: !suppressionDuration,
maxAlerts: tuple.maxSignals - toReturn.createdSignalsCount,
experimentalFeatures,

13
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts
View file

@ -8,20 +8,18 @@
import { performance } from 'perf_hooks';
import { isEmpty } from 'lodash';
import type { Type as RuleType } from '@kbn/securitysolution-io-ts-alerting-types';
import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server';
import type {
AlertWithCommonFieldsLatest,
SuppressionFieldsLatest,
} from '@kbn/rule-registry-plugin/common/schemas';
import { isQueryRule } from '../../../../../common/detection_engine/utils';
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
import { makeFloatString } from './utils';
import type {
BaseFieldsLatest,
WrappedFieldsLatest,
} from '../../../../../common/api/detection_engine/model/alerts';
import type { RuleServices } from '../types';
import type { RuleServices, SecuritySharedParams } from '../types';
import { createEnrichEventsFunction } from './enrichments';
import type { ExperimentalFeatures } from '../../../../../common';
import { getNumberOfSuppressedAlerts } from './get_number_of_suppressed_alerts';
@ -40,28 +38,25 @@ export interface GenericBulkCreateResponse<T extends BaseFieldsLatest> {
export const bulkCreateWithSuppression = async <
T extends SuppressionFieldsLatest & BaseFieldsLatest
>({
alertWithSuppression,
ruleExecutionLogger,
sharedParams,
wrappedDocs,
services,
suppressionWindow,
alertTimestampOverride,
isSuppressionPerRuleExecution,
maxAlerts,
experimentalFeatures,
ruleType,
}: {
alertWithSuppression: SuppressedAlertService;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
sharedParams: SecuritySharedParams;
wrappedDocs: Array<WrappedFieldsLatest<T> & { subAlerts?: Array<WrappedFieldsLatest<T>> }>;
services: RuleServices;
suppressionWindow: string;
alertTimestampOverride: Date | undefined;
isSuppressionPerRuleExecution?: boolean;
maxAlerts?: number;
experimentalFeatures: ExperimentalFeatures;
ruleType?: RuleType;
}): Promise<GenericBulkCreateResponse<T>> => {
const { ruleExecutionLogger, alertWithSuppression, alertTimestampOverride } = sharedParams;
if (wrappedDocs.length === 0) {
return {
errors: [],

243
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts
View file

@ -39,8 +39,7 @@ import {
SPACE_IDS,
TIMESTAMP,
} from '@kbn/rule-data-utils';
import type { BulkCreate, BulkResponse, RuleRangeTuple, WrapHits } from '../types';
import { getRuleRangeTuples } from './utils';
import type { BulkCreate, BulkResponse, WrapHits } from '../types';
import { getCompleteRuleMock, getQueryRuleParams } from '../../rule_schema/mocks';
import { bulkCreateFactory } from '../factories/bulk_create_factory';
import { wrapHitsFactory } from '../factories/wrap_hits_factory';
@ -48,21 +47,38 @@ import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
import type { BuildReasonMessage } from './reason_formatters';
import type { QueryRuleParams } from '../../rule_schema';
import { SERVER_APP_ID } from '../../../../../common/constants';
import type { AlertingServerSetup } from '@kbn/alerting-plugin/server';
import { getSharedParamsMock } from '../__mocks__/shared_params';
describe('searchAfterAndBulkCreate', () => {
let mockService: RuleExecutorServicesMock;
let mockPersistenceServices: jest.Mocked<PersistenceServices>;
let buildReasonMessage: BuildReasonMessage;
let bulkCreate: BulkCreate;
let wrapHits: WrapHits;
let inputIndexPattern: string[] = [];
let listClient = listMock.getListClient();
let alerting: AlertingServerSetup;
const mockService: RuleExecutorServicesMock = alertsMock.createRuleExecutorServices();
const mockPersistenceServices: jest.Mocked<PersistenceServices> = createPersistenceServicesMock();
let buildReasonMessage: BuildReasonMessage = jest
.fn()
.mockResolvedValue('some alert reason message');
const listClient = listMock.getListClient();
listClient.searchListItemByValues = jest.fn().mockResolvedValue([]);
const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create();
const someGuids = Array.from({ length: 13 }).map(() => uuidv4());
const sampleParams = getQueryRuleParams();
const queryCompleteRule = getCompleteRuleMock<QueryRuleParams>(sampleParams);
const inputIndex = ['auditbeat-*'];
const wrapHits: WrapHits = wrapHitsFactory({
completeRule: queryCompleteRule,
mergeStrategy: 'missingFields',
ignoreFields: {},
ignoreFieldsRegexes: [],
spaceId: 'default',
indicesToQuery: inputIndex,
alertTimestampOverride: undefined,
ruleExecutionLogger,
publicBaseUrl: 'http://testkibanabaseurl.com',
intendedTimestamp: undefined,
});
const bulkCreate: BulkCreate = bulkCreateFactory(
mockPersistenceServices.alertWithPersistence,
false,
ruleExecutionLogger
);
const defaultFilter = {
match_all: {},
};
@ -81,47 +97,21 @@ describe('searchAfterAndBulkCreate', () => {
[TIMESTAMP]: '2020-04-20T21:27:45+0000',
};
sampleParams.maxSignals = 30;
let tuple: RuleRangeTuple;
const sharedParams = getSharedParamsMock({
ruleParams: sampleParams,
rewrites: {
inputIndex,
bulkCreate,
wrapHits,
searchAfterSize: 1,
listClient,
},
});
beforeEach(async () => {
jest.clearAllMocks();
buildReasonMessage = jest.fn().mockResolvedValue('some alert reason message');
listClient = listMock.getListClient();
listClient.searchListItemByValues = jest.fn().mockResolvedValue([]);
inputIndexPattern = ['auditbeat-*'];
mockService = alertsMock.createRuleExecutorServices();
alerting = alertsMock.createSetup();
alerting.getConfig = jest.fn().mockReturnValue({ run: { alerts: { max: 1000 } } });
tuple = (
await getRuleRangeTuples({
previousStartedAt: new Date(),
startedAt: new Date(),
from: sampleParams.from,
to: sampleParams.to,
interval: '5m',
maxSignals: sampleParams.maxSignals,
ruleExecutionLogger,
alerting,
})
).tuples[0];
mockPersistenceServices = createPersistenceServicesMock();
bulkCreate = bulkCreateFactory(
mockPersistenceServices.alertWithPersistence,
false,
ruleExecutionLogger
);
wrapHits = wrapHitsFactory({
completeRule: queryCompleteRule,
mergeStrategy: 'missingFields',
ignoreFields: {},
ignoreFieldsRegexes: [],
spaceId: 'default',
indicesToQuery: inputIndexPattern,
alertTimestampOverride: undefined,
ruleExecutionLogger,
publicBaseUrl: 'http://testkibanabaseurl.com',
intendedTimestamp: undefined,
});
buildReasonMessage = jest.fn().mockResolvedValue('some alert reason message');
});
test('should return success with number of searches less than max signals', async () => {
@ -217,20 +207,14 @@ describe('searchAfterAndBulkCreate', () => {
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
tuple,
listClient,
exceptionsList: [exceptionItem],
sharedParams: {
...sharedParams,
unprocessedExceptions: [exceptionItem],
},
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(5);
@ -311,20 +295,15 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
tuple,
listClient,
exceptionsList: [exceptionItem],
sharedParams: {
...sharedParams,
unprocessedExceptions: [exceptionItem],
listClient,
},
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(4);
@ -385,20 +364,14 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
tuple,
listClient,
exceptionsList: [exceptionItem],
sharedParams: {
...sharedParams,
unprocessedExceptions: [exceptionItem],
},
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
@ -443,20 +416,14 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
tuple,
listClient,
exceptionsList: [exceptionItem],
sharedParams: {
...sharedParams,
unprocessedExceptions: [exceptionItem],
},
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
@ -511,20 +478,11 @@ describe('searchAfterAndBulkCreate', () => {
);
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
tuple,
listClient,
exceptionsList: [],
sharedParams,
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
@ -565,20 +523,14 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
tuple,
listClient,
exceptionsList: [exceptionItem],
sharedParams: {
...sharedParams,
unprocessedExceptions: [exceptionItem],
},
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(1);
@ -633,20 +585,14 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
tuple,
listClient,
exceptionsList: [exceptionItem],
sharedParams: {
...sharedParams,
unprocessedExceptions: [exceptionItem],
},
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(1);
@ -703,20 +649,11 @@ describe('searchAfterAndBulkCreate', () => {
)
);
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
tuple,
listClient,
exceptionsList: [],
sharedParams,
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
@ -749,20 +686,14 @@ describe('searchAfterAndBulkCreate', () => {
)
);
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
listClient,
exceptionsList: [exceptionItem],
tuple,
sharedParams: {
...sharedParams,
unprocessedExceptions: [exceptionItem],
},
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(true);
expect(createdSignalsCount).toEqual(0);
@ -794,20 +725,14 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
listClient,
exceptionsList: [exceptionItem],
tuple,
sharedParams: {
...sharedParams,
unprocessedExceptions: [exceptionItem],
},
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(false);
expect(createdSignalsCount).toEqual(0); // should not create signals if search threw error
@ -921,20 +846,11 @@ describe('searchAfterAndBulkCreate', () => {
);
const { success, createdSignalsCount, lastLookBackDate, errors } =
await searchAfterAndBulkCreate({
tuple,
listClient,
exceptionsList: [],
sharedParams,
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(success).toEqual(false);
expect(errors).toEqual(['error on creation']);
@ -1006,21 +922,12 @@ describe('searchAfterAndBulkCreate', () => {
const mockEnrichment = jest.fn((a) => a);
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
sharedParams,
enrichment: mockEnrichment,
tuple,
listClient,
exceptionsList: [],
services: mockService,
ruleExecutionLogger,
eventsTelemetry: undefined,
inputIndexPattern,
pageSize: 1,
filter: defaultFilter,
buildReasonMessage,
bulkCreate,
wrapHits,
runtimeMappings: undefined,
primaryTimestamp: '@timestamp',
});
expect(mockEnrichment).toHaveBeenCalledWith(

10
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.ts
View file

@ -15,20 +15,20 @@ import { searchAfterAndBulkCreateFactory } from './search_after_bulk_create_fact
export const searchAfterAndBulkCreate = async (
params: SearchAfterAndBulkCreateParams
): Promise<SearchAfterAndBulkCreateReturnType> => {
const { wrapHits, bulkCreate, services, buildReasonMessage, ruleExecutionLogger, tuple } = params;
const { sharedParams, services, buildReasonMessage } = params;
const bulkCreateExecutor: SearchAfterAndBulkCreateFactoryParams['bulkCreateExecutor'] = async ({
enrichedEvents,
toReturn,
}) => {
const wrappedDocs = wrapHits(enrichedEvents, buildReasonMessage);
const wrappedDocs = sharedParams.wrapHits(enrichedEvents, buildReasonMessage);
const bulkCreateResult = await bulkCreate(
const bulkCreateResult = await sharedParams.bulkCreate(
wrappedDocs,
tuple.maxSignals - toReturn.createdSignalsCount,
sharedParams.tuple.maxSignals - toReturn.createdSignalsCount,
createEnrichEventsFunction({
services,
logger: ruleExecutionLogger,
logger: sharedParams.ruleExecutionLogger,
})
);
addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult });

28
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_factory.ts
View file

@ -60,26 +60,30 @@ export interface SearchAfterAndBulkCreateFactoryParams extends SearchAfterAndBul
}
export const searchAfterAndBulkCreateFactory = async ({
sharedParams,
enrichment = identity,
eventsTelemetry,
exceptionsList,
filter,
inputIndexPattern,
listClient,
pageSize,
ruleExecutionLogger,
services,
sortOrder,
trackTotalHits,
tuple,
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
additionalFilters,
bulkCreateExecutor,
getWarningMessage,
isLoggedRequestsEnabled,
maxSignalsOverride,
}: SearchAfterAndBulkCreateFactoryParams): Promise<SearchAfterAndBulkCreateReturnType> => {
const {
inputIndex: inputIndexPattern,
runtimeMappings,
searchAfterSize: pageSize,
primaryTimestamp,
secondaryTimestamp,
unprocessedExceptions: exceptionsList,
tuple,
ruleExecutionLogger,
listClient,
} = sharedParams;
// eslint-disable-next-line complexity
return withSecuritySpan('searchAfterAndBulkCreate', async () => {
let toReturn = createSearchAfterReturnType();
@ -102,7 +106,9 @@ export const searchAfterAndBulkCreateFactory = async ({
});
}
while (toReturn.createdSignalsCount <= tuple.maxSignals) {
const maxSignals = maxSignalsOverride ?? tuple.maxSignals;
while (toReturn.createdSignalsCount <= maxSignals) {
const cycleNum = `cycle ${searchingIteration++}`;
try {
let mergedSearchResults = createSearchResultReturnType();
@ -127,7 +133,7 @@ export const searchAfterAndBulkCreateFactory = async ({
services,
ruleExecutionLogger,
filter,
pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)),
pageSize: Math.ceil(Math.min(maxSignals, pageSize)),
primaryTimestamp,
secondaryTimestamp,
trackTotalHits,

18
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create_suppressed_alerts.ts
View file

@ -5,8 +5,6 @@
* 2.0.
*/
import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server';
import { getSuppressionMaxSignalsWarning } from './utils';
import type {
SearchAfterAndBulkCreateParams,
@ -18,8 +16,6 @@ import type { ExperimentalFeatures } from '../../../../../common';
interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams {
wrapSuppressedHits: WrapSuppressedHits;
alertTimestampOverride: Date | undefined;
alertWithSuppression: SuppressedAlertService;
alertSuppression?: AlertSuppressionCamel;
experimentalFeatures: ExperimentalFeatures;
}
@ -36,16 +32,11 @@ export const searchAfterAndBulkCreateSuppressedAlerts = async (
params: SearchAfterAndBulkCreateSuppressedAlertsParams
): Promise<SearchAfterAndBulkCreateReturnType> => {
const {
wrapHits,
bulkCreate,
sharedParams,
services,
buildReasonMessage,
ruleExecutionLogger,
tuple,
alertSuppression,
wrapSuppressedHits,
alertWithSuppression,
alertTimestampOverride,
experimentalFeatures,
} = params;
@ -54,16 +45,11 @@ export const searchAfterAndBulkCreateSuppressedAlerts = async (
toReturn,
}) => {
return bulkCreateSuppressedAlertsInMemory({
wrapHits,
bulkCreate,
sharedParams,
services,
buildReasonMessage,
ruleExecutionLogger,
tuple,
alertSuppression,
wrapSuppressedHits,
alertWithSuppression,
alertTimestampOverride,
enrichedEvents,
toReturn,
experimentalFeatures,

31
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts
View file

@ -59,10 +59,10 @@ import type {
SignalSourceHit,
SimpleHit,
WrappedEventHit,
SecuritySharedParams,
} from '../types';
import type { ShardError } from '../../../types';
import type {
CompleteRule,
EqlRuleParams,
EsqlRuleParams,
MachineLearningRuleParams,
@ -83,7 +83,6 @@ import type {
} from '../../../../../common/api/detection_engine/model/alerts';
import { ENABLE_CCS_READ_WARNING_SETTING } from '../../../../../common/constants';
import type { GenericBulkCreateResponse } from '../factories';
import type { ConfigType } from '../../../../config';
import type {
ExtraFieldsForShellAlert,
WrappedEqlShellOptionalSubAlertsType,
@ -1074,33 +1073,15 @@ export const getDisabledActionsWarningText = ({
}
};
export interface SharedParams {
spaceId: string;
completeRule: CompleteRule<RuleWithInMemorySuppression>;
mergeStrategy: ConfigType['alertMergeStrategy'];
indicesToQuery: string[];
alertTimestampOverride: Date | undefined;
ruleExecutionLogger: IRuleExecutionLogForExecutors;
publicBaseUrl: string | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
intendedTimestamp: Date | undefined;
}
export type RuleWithInMemorySuppression =
| ThreatRuleParams
| EqlRuleParams
| MachineLearningRuleParams;
export interface SequenceSuppressionTermsAndFieldsParams {
sharedParams: SecuritySharedParams<EqlRuleParams>;
shellAlert: WrappedFieldsLatest<EqlShellFieldsLatest>;
buildingBlockAlerts: Array<WrappedFieldsLatest<EqlBuildingBlockFieldsLatest>>;
spaceId: string;
completeRule: CompleteRule<RuleWithInMemorySuppression>;
indicesToQuery: string[];
alertTimestampOverride: Date | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
}
export type SequenceSuppressionTermsAndFieldsFactory = (
@ -1126,18 +1107,16 @@ export const stringifyAfterKey = (afterKey: Record<string, string | number | nul
};
export const buildShellAlertSuppressionTermsAndFields = ({
sharedParams,
shellAlert,
buildingBlockAlerts,
spaceId,
completeRule,
alertTimestampOverride,
primaryTimestamp,
secondaryTimestamp,
}: SequenceSuppressionTermsAndFieldsParams): WrappedFieldsLatest<
EqlShellFieldsLatest & SuppressionFieldsLatest
> & {
subAlerts: Array<WrappedFieldsLatest<EqlBuildingBlockFieldsLatest>>;
} => {
const { alertTimestampOverride, primaryTimestamp, secondaryTimestamp, completeRule, spaceId } =
sharedParams;
const suppressionTerms = getSuppressionTerms({
alertSuppression: completeRule?.ruleParams?.alertSuppression,
input: shellAlert._source,

61
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts
View file

@ -19,6 +19,7 @@ import type { CompleteRule, ThreatRuleParams } from '../../rule_schema';
import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert';
import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
import { getSharedParamsMock } from '../__mocks__/shared_params';
jest.mock('../factories/utils/transform_hit_to_alert', () => ({ transformHitToAlert: jest.fn() }));
@ -95,28 +96,35 @@ const completeRuleMock: CompleteRule<ThreatRuleParams> = {
},
alertId: 'c1436b3e-e2a6-412a-92ff-ef7e86b926fe',
};
const wrappedParams = {
spaceId: 'default',
completeRule: {
...completeRuleMock,
ruleParams: {
...completeRuleMock.ruleParams,
alertSuppression: {
groupBy: ['agent.name', 'user.name'],
},
const buildReasonMessage = () => 'mock';
const sharedParams = getSharedParamsMock({
ruleParams: {
...completeRuleMock.ruleParams,
alertSuppression: {
groupBy: ['agent.name', 'user.name'],
},
},
mergeStrategy: 'missingFields' as const,
indicesToQuery: ['test*'],
buildReasonMessage: () => 'mock',
alertTimestampOverride: undefined,
ruleExecutionLogger,
publicBaseUrl: 'public-url-mock',
primaryTimestamp: '@timestamp',
secondaryTimestamp: 'event.ingested',
intendedTimestamp: undefined,
};
rewrites: {
spaceId: 'default',
completeRule: {
...completeRuleMock,
ruleParams: {
...completeRuleMock.ruleParams,
alertSuppression: {
groupBy: ['agent.name', 'user.name'],
},
},
},
mergeStrategy: 'missingFields' as const,
inputIndex: ['test*'],
alertTimestampOverride: undefined,
ruleExecutionLogger,
publicBaseUrl: 'public-url-mock',
primaryTimestamp: '@timestamp',
secondaryTimestamp: 'event.ingested',
intendedTimestamp: undefined,
},
});
describe('wrapSuppressedAlerts', () => {
transformHitToAlertMock.mockReturnValue({ 'mock-props': true });
@ -135,12 +143,13 @@ describe('wrapSuppressedAlerts', () => {
_index: 'test*',
},
],
...wrappedParams,
sharedParams,
buildReasonMessage,
});
expect(transformHitToAlertMock).toHaveBeenCalledWith({
spaceId: 'default',
completeRule: wrappedParams.completeRule,
completeRule: sharedParams.completeRule,
doc: {
fields: {
'@timestamp': [expectedTimestamp],
@ -154,7 +163,7 @@ describe('wrapSuppressedAlerts', () => {
ignoreFields: {},
ignoreFieldsRegexes: [],
applyOverrides: true,
buildReasonMessage: wrappedParams.buildReasonMessage,
buildReasonMessage,
indicesToQuery: ['test*'],
alertTimestampOverride: undefined,
ruleExecutionLogger,
@ -204,7 +213,8 @@ describe('wrapSuppressedAlerts', () => {
_index: 'test*',
},
],
...wrappedParams,
sharedParams,
buildReasonMessage,
});
expect(wrappedAlerts[0]._source[ALERT_INSTANCE_ID]).toBe(
@ -225,7 +235,8 @@ describe('wrapSuppressedAlerts', () => {
_index: 'test*',
},
],
...wrappedParams,
sharedParams,
buildReasonMessage,
});
expect(wrappedAlerts[0]._source).toEqual(

30
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts
View file

@ -10,7 +10,7 @@ import objectHash from 'object-hash';
import { TIMESTAMP } from '@kbn/rule-data-utils';
import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas';
import type { SignalSourceHit } from '../types';
import type { SecuritySharedParams, SignalSourceHit } from '../types';
import type {
BaseFieldsLatest,
WrappedFieldsLatest,
@ -18,9 +18,9 @@ import type {
import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert';
import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils';
import type { SharedParams } from './utils';
import { generateId } from './utils';
import type { BuildReasonMessage } from './reason_formatters';
import type { EqlRuleParams, MachineLearningRuleParams, ThreatRuleParams } from '../../rule_schema';
/**
* wraps suppressed alerts
@ -29,21 +29,25 @@ import type { BuildReasonMessage } from './reason_formatters';
*/
export const wrapSuppressedAlerts = ({
events,
spaceId,
completeRule,
mergeStrategy,
indicesToQuery,
buildReasonMessage,
alertTimestampOverride,
ruleExecutionLogger,
publicBaseUrl,
primaryTimestamp,
secondaryTimestamp,
intendedTimestamp,
sharedParams,
}: {
events: SignalSourceHit[];
buildReasonMessage: BuildReasonMessage;
} & SharedParams): Array<WrappedFieldsLatest<BaseFieldsLatest & SuppressionFieldsLatest>> => {
sharedParams: SecuritySharedParams<MachineLearningRuleParams | EqlRuleParams | ThreatRuleParams>;
}): Array<WrappedFieldsLatest<BaseFieldsLatest & SuppressionFieldsLatest>> => {
const {
completeRule,
spaceId,
mergeStrategy,
inputIndex: indicesToQuery,
alertTimestampOverride,
ruleExecutionLogger,
publicBaseUrl,
intendedTimestamp,
primaryTimestamp,
secondaryTimestamp,
} = sharedParams;
return events.map((event) => {
const suppressionTerms = getSuppressionTerms({
alertSuppression: completeRule?.ruleParams?.alertSuppression,

1
x-pack/solutions/security/plugins/security_solution/server/plugin.ts
View file

@ -326,7 +326,6 @@ export class Plugin implements ISecuritySolutionPlugin {
logger: this.logger,
ml: plugins.ml,
eventsTelemetry: this.telemetryEventsSender,
version: pluginContext.env.packageInfo.version,
licensing: plugins.licensing,
scheduleNotificationResponseActionsService: getScheduleNotificationResponseActionsService({
endpointAppContextService: this.endpointAppContextService,

2
x-pack/solutions/security/plugins/security_solution/tsconfig.json
View file

@ -240,6 +240,6 @@
"@kbn/charts-theme",
"@kbn/product-doc-base-plugin",
"@kbn/shared-ux-error-boundary",
"@kbn/security-ai-prompts",
"@kbn/security-ai-prompts"
]
}