github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][CTI] Rule Preview backend update (introduces /preview endpoint) (#112441)

Browse source 
Co-authored-by: Davis Plumlee <davis.plumlee@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Ece Özalp 2021-10-20 12:06:50 -04:00 • committed by GitHub
parent c9bca2cd59
commit b12e21d9aa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
67 changed files with 1335 additions and 824 deletions
Download patch file Download diff file Expand all files Collapse all files

3
x-pack/plugins/security_solution/common/constants.ts
View file

@ -31,6 +31,7 @@ export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults';
export const DEFAULT_APP_REFRESH_INTERVAL = 'securitySolution:refreshIntervalDefaults';
export const DEFAULT_ALERTS_INDEX = '.alerts-security.alerts';
export const DEFAULT_SIGNALS_INDEX = '.siem-signals';
export const DEFAULT_PREVIEW_INDEX = '.siem-preview-signals';
export const DEFAULT_LISTS_INDEX = '.lists';
export const DEFAULT_ITEMS_INDEX = '.items';
// The DEFAULT_MAX_SIGNALS value exists also in `x-pack/plugins/cases/common/constants.ts`
@ -248,6 +249,8 @@ export const DETECTION_ENGINE_TAGS_URL = `${DETECTION_ENGINE_URL}/tags`;
export const DETECTION_ENGINE_RULES_STATUS_URL = `${DETECTION_ENGINE_RULES_URL}/_find_statuses`;
export const DETECTION_ENGINE_PREPACKAGED_RULES_STATUS_URL = `${DETECTION_ENGINE_RULES_URL}/prepackaged/_status`;
export const DETECTION_ENGINE_RULES_BULK_ACTION = `${DETECTION_ENGINE_RULES_URL}/_bulk_action`;
export const DETECTION_ENGINE_RULES_PREVIEW = `${DETECTION_ENGINE_RULES_URL}/preview`;
export const DETECTION_ENGINE_RULES_PREVIEW_INDEX_URL = `${DETECTION_ENGINE_RULES_PREVIEW}/index`;
export const TIMELINE_RESOLVE_URL = '/api/timeline/resolve';
export const TIMELINE_URL = '/api/timeline';

5
x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts
View file

@ -362,6 +362,11 @@ export type MachineLearningCreateSchema = CreateSchema<
export const createRulesSchema = t.intersection([sharedCreateSchema, createTypeSpecific]);
export type CreateRulesSchema = t.TypeOf<typeof createRulesSchema>;
export const previewRulesSchema = t.intersection([
sharedCreateSchema,
createTypeSpecific,
t.type({ invocationCount: t.number }),
]);
type UpdateSchema<T> = SharedUpdateSchema & T;
export type EqlUpdateSchema = UpdateSchema<t.TypeOf<typeof eqlCreateParams>>;

8
x-pack/plugins/security_solution/public/detections/components/rules/query_preview/helpers.ts
View file

@ -29,6 +29,8 @@ export const isNoisy = (hits: number, timeframe: Unit): boolean => {
return hits > 1;
} else if (timeframe === 'd') {
return hits / 24 > 1;
} else if (timeframe === 'w') {
return hits / 168 > 1;
} else if (timeframe === 'M') {
return hits / 730 > 1;
}
@ -48,6 +50,12 @@ export const getTimeframeOptions = (ruleType: Type): EuiSelectOption[] => {
{ value: 'h', text: 'Last hour' },
{ value: 'd', text: 'Last day' },
];
} else if (ruleType === 'threat_match') {
return [
{ value: 'h', text: i18n.LAST_HOUR },
{ value: 'd', text: i18n.LAST_DAY },
{ value: 'w', text: i18n.LAST_WEEK },
];
} else {
return [
{ value: 'h', text: 'Last hour' },

16
x-pack/plugins/security_solution/public/detections/components/rules/query_preview/translations.ts
View file

@ -7,6 +7,22 @@
import { i18n } from '@kbn/i18n';
export const LAST_HOUR = i18n.translate('xpack.securitySolution.stepDefineRule.lastHour', {
defaultMessage: 'Last hour',
});
export const LAST_DAY = i18n.translate('xpack.securitySolution.stepDefineRule.lastDay', {
defaultMessage: 'Last day',
});
export const LAST_WEEK = i18n.translate('xpack.securitySolution.stepDefineRule.lastWeek', {
defaultMessage: 'Last week',
});
export const LAST_MONTH = i18n.translate('xpack.securitySolution.stepDefineRule.lastMonth', {
defaultMessage: 'Last month',
});
export const QUERY_PREVIEW_BUTTON = i18n.translate(
'xpack.securitySolution.stepDefineRule.previewQueryButton',
{

1
x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.test.tsx
View file

@ -11,6 +11,7 @@ import { shallow } from 'enzyme';
import { StepDefineRule } from './index';
jest.mock('../../../../common/lib/kibana');
jest.mock('../../../containers/detection_engine/alerts/use_preview_index');
describe('StepDefineRule', () => {
it('renders correctly', () => {

2
x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx
View file

@ -57,6 +57,7 @@ import { EqlQueryBar } from '../eql_query_bar';
import { ThreatMatchInput } from '../threatmatch_input';
import { BrowserField, BrowserFields, useFetchIndex } from '../../../../common/containers/source';
import { PreviewQuery } from '../query_preview';
import { usePreviewIndex } from '../../../containers/detection_engine/alerts/use_preview_index';
const CommonUseField = getUseField({ component: Field });
@ -136,6 +137,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
onSubmit,
setForm,
}) => {
usePreviewIndex();
const mlCapabilities = useMlCapabilities();
const [openTimelineSearch, setOpenTimelineSearch] = useState(false);
const [indexModified, setIndexModified] = useState(false);

20
x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.test.ts
View file

@ -21,6 +21,7 @@ import {
getUserPrivilege,
createSignalIndex,
createHostIsolation,
createPreviewIndex,
} from './api';
import { coreMock } from '../../../../../../../../src/core/public/mocks';
@ -165,6 +166,25 @@ describe('Detections Alerts API', () => {
});
});
describe('createPreviewIndex', () => {
beforeEach(() => {
fetchMock.mockClear();
fetchMock.mockResolvedValue({ acknowledged: true });
});
test('check parameter url', async () => {
await createPreviewIndex();
expect(fetchMock).toHaveBeenCalledWith('/api/detection_engine/rules/preview/index', {
method: 'POST',
});
});
test('happy path', async () => {
const previewResp = await createPreviewIndex();
expect(previewResp).toEqual({ acknowledged: true });
});
});
describe('createHostIsolation', () => {
const postMock = coreStartMock.http.post;

13
x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.ts
View file

@ -14,6 +14,7 @@ import {
DETECTION_ENGINE_INDEX_URL,
DETECTION_ENGINE_PRIVILEGES_URL,
ALERTS_AS_DATA_FIND_URL,
DETECTION_ENGINE_RULES_PREVIEW_INDEX_URL,
} from '../../../../../common/constants';
import { HOST_METADATA_GET_ROUTE } from '../../../../../common/endpoint/constants';
import { KibanaServices } from '../../../../common/lib/kibana';
@ -132,6 +133,18 @@ export const createSignalIndex = async ({ signal }: BasicSignals): Promise<Alert
signal,
});
/**
* Create Signal Index if needed it
*
* @param signal AbortSignal for cancelling request
*
* @throws An error if response is not OK
*/
export const createPreviewIndex = async (): Promise<AlertsIndex> =>
KibanaServices.get().http.fetch<AlertsIndex>(DETECTION_ENGINE_RULES_PREVIEW_INDEX_URL, {
method: 'POST',
});
/**
* Get Host Isolation index
*

15
x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_preview_index.tsx Normal file
View file

@ -0,0 +1,15 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { useEffect } from 'react';
import { createPreviewIndex } from './api';
export const usePreviewIndex = () => {
useEffect(() => {
createPreviewIndex();
}, []);
};

1
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/index.test.tsx
View file

@ -27,6 +27,7 @@ jest.mock('react-router-dom', () => {
});
jest.mock('../../../../../common/lib/kibana');
jest.mock('../../../../containers/detection_engine/lists/use_lists_config');
jest.mock('../../../../containers/detection_engine/alerts/use_preview_index');
jest.mock('../../../../../common/components/link_to');
jest.mock('../../../../components/user_info');
jest.mock('../../../../../common/hooks/use_app_toasts');

1
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.test.tsx
View file

@ -85,6 +85,7 @@ jest.mock('react-router-dom', () => {
});
jest.mock('../../../../../common/lib/kibana');
jest.mock('../../../../containers/detection_engine/alerts/use_preview_index');
const mockRedirectLegacyUrl = jest.fn();
const mockGetLegacyUrlConflict = jest.fn();

4
x-pack/plugins/security_solution/server/client/client.ts
View file

@ -6,18 +6,22 @@
*/
import { ConfigType } from '../config';
import { DEFAULT_PREVIEW_INDEX } from '../../common/constants';
export class AppClient {
private readonly signalsIndex: string;
private readonly spaceId: string;
private readonly previewIndex: string;
constructor(_spaceId: string, private config: ConfigType) {
const configuredSignalsIndex = this.config.signalsIndex;
this.signalsIndex = `${configuredSignalsIndex}-${_spaceId}`;
this.previewIndex = `${DEFAULT_PREVIEW_INDEX}-${_spaceId}`;
this.spaceId = _spaceId;
}
public getSignalsIndex = (): string => this.signalsIndex;
public getPreviewIndex = (): string => this.previewIndex;
public getSpaceId = (): string => this.spaceId;
}

88
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_preview_index_route.ts Normal file
View file

@ -0,0 +1,88 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
transformError,
getIndexExists,
getPolicyExists,
setPolicy,
createBootstrapIndex,
} from '@kbn/securitysolution-es-utils';
import type {
AppClient,
SecuritySolutionPluginRouter,
SecuritySolutionRequestHandlerContext,
} from '../../../../types';
import { DETECTION_ENGINE_RULES_PREVIEW_INDEX_URL } from '../../../../../common/constants';
import { buildSiemResponse } from '../utils';
import { getSignalsTemplate, SIGNALS_TEMPLATE_VERSION } from './get_signals_template';
import previewPolicy from './preview_policy.json';
import { getIndexVersion } from './get_index_version';
import { isOutdated } from '../../migrations/helpers';
import { templateNeedsUpdate } from './check_template_version';
export const createPreviewIndexRoute = (router: SecuritySolutionPluginRouter) => {
router.post(
{
path: DETECTION_ENGINE_RULES_PREVIEW_INDEX_URL,
validate: false,
options: {
tags: ['access:securitySolution'],
},
},
async (context, request, response) => {
const siemResponse = buildSiemResponse(response);
try {
const siemClient = context.securitySolution?.getAppClient();
if (!siemClient) {
return siemResponse.error({ statusCode: 404 });
}
await createPreviewIndex(context, siemClient);
return response.ok({ body: { acknowledged: true } });
} catch (err) {
const error = transformError(err);
return siemResponse.error({
body: error.message,
statusCode: error.statusCode,
});
}
}
);
};
export const createPreviewIndex = async (
context: SecuritySolutionRequestHandlerContext,
siemClient: AppClient
) => {
const esClient = context.core.elasticsearch.client.asCurrentUser;
const index = siemClient.getPreviewIndex();
const indexExists = await getIndexExists(esClient, index);
const policyExists = await getPolicyExists(esClient, index);
if (!policyExists) {
await setPolicy(esClient, index, previewPolicy);
}
if (await templateNeedsUpdate({ alias: index, esClient })) {
await esClient.indices.putIndexTemplate({
name: index,
body: getSignalsTemplate(index) as Record<string, unknown>,
});
}
if (indexExists) {
const indexVersion = await getIndexVersion(esClient, index);
if (isOutdated({ current: indexVersion, target: SIGNALS_TEMPLATE_VERSION })) {
await esClient.indices.rollover({ alias: index });
}
} else {
await createBootstrapIndex(esClient, index);
}
};

12
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts
View file

@ -24,8 +24,8 @@ import signalExtraFields from './signal_extra_fields.json';
@description This value represents the template version assumed by app code.
If this number is greater than the user's signals index version, the
detections UI will attempt to update the signals template and roll over to
a new signals index.
a new signals index.
Since we create a new index for new versions, this version on an existing index should never change.
If making mappings changes in a patch release, this number should be incremented by 1.
@ -43,8 +43,8 @@ export const SIGNALS_TEMPLATE_VERSION = 57;
This version number can change over time on existing indices as we add backwards compatibility fields.
If any .siem-signals-<space id> indices have an aliases_version less than this value, the detections
UI will call create_index_route and and go through the index update process. Increment this number if
If any .siem-signals-<space id> indices have an aliases_version less than this value, the detections
UI will call create_index_route and and go through the index update process. Increment this number if
making changes to the field aliases we use to make signals forwards-compatible.
*/
export const SIGNALS_FIELD_ALIASES_VERSION = 1;
@ -52,14 +52,14 @@ export const SIGNALS_FIELD_ALIASES_VERSION = 1;
/**
@constant
@type {number}
@description This value represents the minimum required index version (SIGNALS_TEMPLATE_VERSION) for EQL
@description This value represents the minimum required index version (SIGNALS_TEMPLATE_VERSION) for EQL
rules to write signals correctly. If the write index has a `version` less than this value, the EQL rule
will throw an error on execution.
*/
export const MIN_EQL_RULE_INDEX_VERSION = 2;
export const ALIAS_VERSION_FIELD = 'aliases_version';
export const getSignalsTemplate = (index: string, spaceId: string, aadIndexAliasName: string) => {
export const getSignalsTemplate = (index: string, spaceId?: string, aadIndexAliasName?: string) => {
const fieldAliases = createSignalsFieldAliases();
const template = {
index_patterns: [`${index}-*`],

21
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/preview_policy.json Normal file
View file

@ -0,0 +1,21 @@
{
"policy": {
"phases": {
"hot": {
"actions": {
"rollover": {
"max_age": "1d",
"max_primary_shard_size": "50gb"
}
},
"min_age": "0ms"
},
"delete": {
"min_age": "1d",
"actions": {
"delete": {}
}
}
}
}
}

226
x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/preview_rules_route.ts Normal file
View file

@ -0,0 +1,226 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import moment from 'moment';
import uuid from 'uuid';
import { transformError } from '@kbn/securitysolution-es-utils';
import { buildSiemResponse } from '../utils';
import { convertCreateAPIToInternalSchema } from '../../schemas/rule_converters';
import { RuleParams } from '../../schemas/rule_schemas';
import { signalRulesAlertType } from '../../signals/signal_rule_alert_type';
import { createWarningsAndErrors } from '../../signals/preview/preview_rule_execution_log_client';
import { parseInterval } from '../../signals/utils';
import { buildMlAuthz } from '../../../machine_learning/authz';
import { throwHttpError } from '../../../machine_learning/validation';
import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
import { SetupPlugins } from '../../../../plugin';
import type { SecuritySolutionPluginRouter } from '../../../../types';
import { createRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/create_rules_type_dependents';
import { DETECTION_ENGINE_RULES_PREVIEW } from '../../../../../common/constants';
import { previewRulesSchema } from '../../../../../common/detection_engine/schemas/request';
import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas';
import {
AlertInstanceContext,
AlertInstanceState,
AlertTypeState,
parseDuration,
} from '../../../../../../alerting/common';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { ExecutorType } from '../../../../../../alerting/server/types';
import { AlertInstance } from '../../../../../../alerting/server';
import { ConfigType } from '../../../../config';
import { IEventLogService } from '../../../../../../event_log/server';
import { alertInstanceFactoryStub } from '../../signals/preview/alert_instance_factory_stub';
import { CreateRuleOptions } from '../../rule_types/types';
enum InvocationCount {
HOUR = 1,
DAY = 24,
WEEK = 168,
}
export const previewRulesRoute = async (
router: SecuritySolutionPluginRouter,
config: ConfigType,
ml: SetupPlugins['ml'],
security: SetupPlugins['security'],
ruleOptions: CreateRuleOptions
) => {
router.post(
{
path: DETECTION_ENGINE_RULES_PREVIEW,
validate: {
body: buildRouteValidation(previewRulesSchema),
},
options: {
tags: ['access:securitySolution'],
},
},
async (context, request, response) => {
const siemResponse = buildSiemResponse(response);
const validationErrors = createRuleValidateTypeDependents(request.body);
if (validationErrors.length) {
return siemResponse.error({ statusCode: 400, body: validationErrors });
}
try {
const savedObjectsClient = context.core.savedObjects.client;
const siemClient = context.securitySolution?.getAppClient();
if (!siemClient) {
return siemResponse.error({ statusCode: 404 });
}
if (request.body.type !== 'threat_match') {
return response.ok({ body: { errors: ['Not an indicator match rule'] } });
}
let invocationCount = request.body.invocationCount;
if (
![InvocationCount.HOUR, InvocationCount.DAY, InvocationCount.WEEK].includes(
invocationCount
)
) {
return response.ok({ body: { errors: ['Invalid invocation count'] } });
}
const internalRule = convertCreateAPIToInternalSchema(request.body, siemClient, false);
const previewRuleParams = internalRule.params;
const mlAuthz = buildMlAuthz({
license: context.licensing.license,
ml,
request,
savedObjectsClient,
});
throwHttpError(await mlAuthz.validateRuleType(internalRule.params.type));
await context.lists?.getExceptionListClient().createEndpointList();
const spaceId = siemClient.getSpaceId();
const previewIndex = siemClient.getPreviewIndex();
const previewId = uuid.v4();
const username = security?.authc.getCurrentUser(request)?.username;
const { previewRuleExecutionLogClient, warningsAndErrorsStore } = createWarningsAndErrors();
const runState: Record<string, unknown> = {};
const runExecutors = async <
TParams extends RuleParams,
TState extends AlertTypeState,
TInstanceState extends AlertInstanceState,
TInstanceContext extends AlertInstanceContext,
TActionGroupIds extends string = ''
>(
executor: ExecutorType<
TParams,
TState,
TInstanceState,
TInstanceContext,
TActionGroupIds
>,
ruleTypeId: string,
ruleTypeName: string,
params: TParams,
alertInstanceFactory: (
id: string
) => Pick<
AlertInstance<TInstanceState, TInstanceContext, TActionGroupIds>,
'getState' | 'replaceState' | 'scheduleActions' | 'scheduleActionsWithSubGroup'
>
) => {
let statePreview = runState as TState;
const startedAt = moment();
const parsedDuration = parseDuration(internalRule.schedule.interval) ?? 0;
startedAt.subtract(moment.duration(parsedDuration * invocationCount));
let previousStartedAt = null;
const rule = {
...internalRule,
createdAt: new Date(),
createdBy: username ?? 'preview-created-by',
producer: 'preview-producer',
ruleTypeId,
ruleTypeName,
updatedAt: new Date(),
updatedBy: username ?? 'preview-updated-by',
};
while (invocationCount > 0) {
statePreview = (await executor({
alertId: previewId,
createdBy: rule.createdBy,
name: rule.name,
params,
previousStartedAt,
rule,
services: {
alertInstanceFactory,
savedObjectsClient: context.core.savedObjects.client,
scopedClusterClient: context.core.elasticsearch.client,
},
spaceId,
startedAt: startedAt.toDate(),
state: statePreview,
tags: [],
updatedBy: rule.updatedBy,
})) as TState;
previousStartedAt = startedAt.toDate();
startedAt.add(parseInterval(internalRule.schedule.interval));
invocationCount--;
}
};
const signalRuleAlertType = signalRulesAlertType({
...ruleOptions,
lists: context.lists,
config,
indexNameOverride: previewIndex,
ruleExecutionLogClientOverride: previewRuleExecutionLogClient,
// unused as we override the ruleExecutionLogClient
eventLogService: {} as unknown as IEventLogService,
eventsTelemetry: undefined,
ml: undefined,
});
await runExecutors(
signalRuleAlertType.executor,
signalRuleAlertType.id,
signalRuleAlertType.name,
previewRuleParams,
alertInstanceFactoryStub
);
const errors = warningsAndErrorsStore
.filter((item) => item.newStatus === RuleExecutionStatus.failed)
.map((item) => item.message);
const warnings = warningsAndErrorsStore
.filter(
(item) =>
item.newStatus === RuleExecutionStatus['partial failure'] ||
item.newStatus === RuleExecutionStatus.warning
)
.map((item) => item.message);
return response.ok({
body: {
previewId,
errors,
warnings,
},
});
} catch (err) {
const error = transformError(err as Error);
return siemResponse.error({
body: {
errors: [error.message],
},
statusCode: error.statusCode,
});
}
}
);
};

53
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts
View file

@ -36,7 +36,6 @@ import { bulkCreateFactory, wrapHitsFactory, wrapSequencesFactory } from './fact
import { RuleExecutionLogClient, truncateMessageList } from '../rule_execution_log';
import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas';
import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions';
import { AlertAttributes } from '../signals/types';
/* eslint-disable complexity */
export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
@ -56,6 +55,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
spaceId,
state,
updatedBy: updatedByUser,
rule,
} = options;
let runState = state;
const { from, maxSignals, meta, ruleId, timestampOverride, to } = params;
@ -69,17 +69,20 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
eventLogService,
underlyingClient: config.ruleExecutionLog.underlyingClient,
});
const ruleSO = await savedObjectsClient.get<AlertAttributes<typeof params>>(
'alert',
alertId
);
const completeRule = {
ruleConfig: rule,
ruleParams: params,
alertId,
};
const {
actions,
name,
alertTypeId,
schedule: { interval },
} = ruleSO.attributes;
ruleTypeId,
} = completeRule.ruleConfig;
const refresh = actions.length ? 'wait_for' : false;
const buildRuleMessage = buildRuleMessageFactory({
@ -97,7 +100,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
spaceId,
ruleId: alertId,
ruleName: name,
ruleType: alertTypeId,
ruleType: ruleTypeId,
};
await ruleStatusClient.logStatusChange({
...basicLogArguments,
@ -108,8 +111,8 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
const notificationRuleParams: NotificationRuleTypeParams = {
...params,
name: name as string,
id: ruleSO.id as string,
name,
id: alertId,
} as unknown as NotificationRuleTypeParams;
// check if rule has permissions to access given index pattern
@ -181,7 +184,9 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
interval,
maxSignals: DEFAULT_MAX_SIGNALS,
buildRuleMessage,
startedAt,
});
if (remainingGap.asMilliseconds() > 0) {
const gapString = remainingGap.humanize();
const gapMessage = buildRuleMessage(
@ -220,18 +225,18 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
);
const wrapHits = wrapHitsFactory({
logger,
ignoreFields,
mergeStrategy,
ruleSO,
completeRule,
spaceId,
signalsIndex: '',
});
const wrapSequences = wrapSequencesFactory({
logger,
ignoreFields,
mergeStrategy,
ruleSO,
completeRule,
spaceId,
});
@ -245,7 +250,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
bulkCreate,
exceptionItems,
listClient,
rule: ruleSO,
completeRule,
searchAfterSize,
tuple,
wrapHits,
@ -290,7 +295,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
const resultsLink = getNotificationResultsLink({
from: fromInMs,
to: toInMs,
id: ruleSO.id,
id: alertId,
kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined)
?.kibana_siem_app_url,
});
@ -299,12 +304,12 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
buildRuleMessage(`Found ${createdSignalsCount} signals for notification.`)
);
if (ruleSO.attributes.throttle != null) {
if (completeRule.ruleConfig.throttle != null) {
await scheduleThrottledNotificationActions({
alertInstance: services.alertInstanceFactory(alertId),
throttle: ruleSO.attributes.throttle,
throttle: completeRule.ruleConfig.throttle ?? '',
startedAt,
id: ruleSO.id,
id: alertId,
kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined)
?.kibana_siem_app_url,
outputIndex: ruleDataClient.indexName,
@ -358,12 +363,12 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
);
} else {
// NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early
if (ruleSO.attributes.throttle != null) {
if (completeRule.ruleConfig.throttle != null) {
await scheduleThrottledNotificationActions({
alertInstance: services.alertInstanceFactory(alertId),
throttle: ruleSO.attributes.throttle,
throttle: completeRule.ruleConfig.throttle ?? '',
startedAt,
id: ruleSO.id,
id: completeRule.alertId,
kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined)
?.kibana_siem_app_url,
outputIndex: ruleDataClient.indexName,
@ -392,12 +397,12 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
}
} catch (error) {
// NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early
if (ruleSO.attributes.throttle != null) {
if (completeRule.ruleConfig.throttle != null) {
await scheduleThrottledNotificationActions({
alertInstance: services.alertInstanceFactory(alertId),
throttle: ruleSO.attributes.throttle,
throttle: completeRule.ruleConfig.throttle ?? '',
startedAt,
id: ruleSO.id,
id: completeRule.alertId,
kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined)
?.kibana_siem_app_url,
outputIndex: ruleDataClient.indexName,

6
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts
View file

@ -7,7 +7,7 @@
import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
import { EQL_RULE_TYPE_ID } from '../../../../../common/constants';
import { EqlRuleParams, eqlRuleParams } from '../../schemas/rule_schemas';
import { CompleteRule, eqlRuleParams, EqlRuleParams } from '../../schemas/rule_schemas';
import { eqlExecutor } from '../../signals/executors/eql';
import { CreateRuleOptions, SecurityAlertType } from '../types';
@ -50,7 +50,7 @@ export const createEqlAlertType = (
runOpts: {
bulkCreate,
exceptionItems,
rule,
completeRule,
searchAfterSize,
tuple,
wrapHits,
@ -65,7 +65,7 @@ export const createEqlAlertType = (
exceptionItems,
experimentalFeatures,
logger,
rule,
completeRule: completeRule as CompleteRule<EqlRuleParams>,
searchAfterSize,
services,
tuple,

31
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts
View file

@ -9,9 +9,8 @@ import { Logger } from 'kibana/server';
import { ALERT_RULE_CONSUMER } from '@kbn/rule-data-utils';
import { sampleDocNoSortId } from '../../../signals/__mocks__/es_results';
import { sampleDocNoSortId, sampleRuleGuid } from '../../../signals/__mocks__/es_results';
import { buildAlertGroupFromSequence } from './build_alert_group_from_sequence';
import { getRulesSchemaMock } from '../../../../../../common/detection_engine/schemas/response/rules_schema.mocks';
import {
ALERT_ANCESTORS,
ALERT_BUILDING_BLOCK_TYPE,
@ -19,7 +18,8 @@ import {
ALERT_GROUP_ID,
} from '../../field_maps/field_names';
import { SERVER_APP_ID } from '../../../../../../common/constants';
import { getQueryRuleParams } from '../../../schemas/rule_schemas.mock';
import { getCompleteRuleMock, getQueryRuleParams } from '../../../schemas/rule_schemas.mock';
import { QueryRuleParams } from '../../../schemas/rule_schemas';
const SPACE_ID = 'space';
@ -40,24 +40,7 @@ describe('buildAlert', () => {
});
test('it builds an alert as expected without original_event if event does not exist', () => {
const rule = getRulesSchemaMock();
const ruleSO = {
attributes: {
actions: [],
alertTypeId: 'siem.signals',
createdAt: new Date().toISOString(),
createdBy: 'gandalf',
params: getQueryRuleParams(),
schedule: { interval: '1m' },
throttle: 'derp',
updatedAt: new Date().toISOString(),
updatedBy: 'galadriel',
...rule,
},
id: 'abcd',
references: [],
type: 'rule',
};
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const eqlSequence = {
join_keys: [],
events: [
@ -68,7 +51,7 @@ describe('buildAlert', () => {
const alertGroup = buildAlertGroupFromSequence(
loggerMock,
eqlSequence,
ruleSO,
completeRule,
'allFields',
SPACE_ID,
jest.fn()
@ -128,14 +111,14 @@ describe('buildAlert', () => {
depth: 1,
id: alertGroup[0]._id,
index: '',
rule: 'abcd',
rule: sampleRuleGuid,
type: 'signal',
},
{
depth: 1,
id: alertGroup[1]._id,
index: '',
rule: 'abcd',
rule: sampleRuleGuid,
type: 'signal',
},
]),

18
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.ts
View file

@ -9,10 +9,9 @@ import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils';
import { Logger } from 'kibana/server';
import { SavedObject } from 'src/core/types';
import type { ConfigType } from '../../../../../config';
import { buildRuleWithoutOverrides } from '../../../signals/build_rule';
import { AlertAttributes, Ancestor, SignalSource } from '../../../signals/types';
import { Ancestor, SignalSource } from '../../../signals/types';
import { RACAlert, WrappedRACAlert } from '../../types';
import { buildAlert, buildAncestors, generateAlertId } from './build_alert';
import { buildBulkBody } from './build_bulk_body';
@ -25,31 +24,32 @@ import {
ALERT_GROUP_ID,
ALERT_GROUP_INDEX,
} from '../../field_maps/field_names';
import { CompleteRule, RuleParams } from '../../../schemas/rule_schemas';
/**
* Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed -
* one signal for each event in the sequence, and a "shell" signal that ties them all together. All N+1 signals
* share the same signal.group.id to make it easy to query them.
* @param sequence The raw ES documents that make up the sequence
* @param ruleSO SavedObject representing the rule that found the sequence
* @param completeRule object representing the rule that found the sequence
*/
export const buildAlertGroupFromSequence = (
logger: Logger,
sequence: EqlSequence<SignalSource>,
ruleSO: SavedObject<AlertAttributes>,
completeRule: CompleteRule<RuleParams>,
mergeStrategy: ConfigType['alertMergeStrategy'],
spaceId: string | null | undefined,
buildReasonMessage: BuildReasonMessage
): WrappedRACAlert[] => {
const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event));
if (ancestors.some((ancestor) => ancestor?.rule === ruleSO.id)) {
if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) {
return [];
}
let buildingBlocks: RACAlert[] = [];
try {
buildingBlocks = sequence.events.map((event) => ({
...buildBulkBody(spaceId, ruleSO, event, mergeStrategy, [], false, buildReasonMessage),
...buildBulkBody(spaceId, completeRule, event, mergeStrategy, [], false, buildReasonMessage),
[ALERT_BUILDING_BLOCK_TYPE]: 'default',
}));
} catch (error) {
@ -70,7 +70,7 @@ export const buildAlertGroupFromSequence = (
// Now that we have an array of building blocks for the events in the sequence,
// we can build the signal that links the building blocks together
// and also insert the group id (which is also the "shell" signal _id) in each building block
const doc = buildAlertRoot(wrappedBuildingBlocks, ruleSO, spaceId, buildReasonMessage);
const doc = buildAlertRoot(wrappedBuildingBlocks, completeRule, spaceId, buildReasonMessage);
const sequenceAlert = {
_id: generateAlertId(doc),
_index: '',
@ -87,11 +87,11 @@ export const buildAlertGroupFromSequence = (
export const buildAlertRoot = (
wrappedBuildingBlocks: WrappedRACAlert[],
ruleSO: SavedObject<AlertAttributes>,
completeRule: CompleteRule<RuleParams>,
spaceId: string | null | undefined,
buildReasonMessage: BuildReasonMessage
): RACAlert => {
const rule = buildRuleWithoutOverrides(ruleSO);
const rule = buildRuleWithoutOverrides(completeRule);
const reason = buildReasonMessage({ rule });
const doc = buildAlert(wrappedBuildingBlocks, rule, spaceId, reason);
const mergedAlerts = objectArrayIntersection(wrappedBuildingBlocks.map((alert) => alert._source));

12
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts
View file

@ -6,16 +6,16 @@
*/
import { TIMESTAMP } from '@kbn/rule-data-utils';
import { SavedObject } from 'src/core/types';
import { BaseHit } from '../../../../../../common/detection_engine/types';
import type { ConfigType } from '../../../../../config';
import { buildRuleWithOverrides, buildRuleWithoutOverrides } from '../../../signals/build_rule';
import { BuildReasonMessage } from '../../../signals/reason_formatters';
import { getMergeStrategy } from '../../../signals/source_fields_merging/strategies';
import { AlertAttributes, SignalSource, SignalSourceHit, SimpleHit } from '../../../signals/types';
import { SignalSource, SignalSourceHit, SimpleHit } from '../../../signals/types';
import { RACAlert } from '../../types';
import { additionalAlertFields, buildAlert } from './build_alert';
import { filterSource } from './filter_source';
import { CompleteRule, RuleParams } from '../../../schemas/rule_schemas';
const isSourceDoc = (
hit: SignalSourceHit
@ -28,13 +28,13 @@ const isSourceDoc = (
* "best effort" merged "fields" with the "_source" object, then build the signal object,
* then the event object, and finally we strip away any additional temporary data that was added
* such as the "threshold_result".
* @param ruleSO The rule saved object to build overrides
* @param completeRule The rule saved object to build overrides
* @param doc The SignalSourceHit with "_source", "fields", and additional data such as "threshold_result"
* @returns The body that can be added to a bulk call for inserting the signal.
*/
export const buildBulkBody = (
spaceId: string | null | undefined,
ruleSO: SavedObject<AlertAttributes>,
completeRule: CompleteRule<RuleParams>,
doc: SimpleHit,
mergeStrategy: ConfigType['alertMergeStrategy'],
ignoreFields: ConfigType['alertIgnoreFields'],
@ -43,8 +43,8 @@ export const buildBulkBody = (
): RACAlert => {
const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields });
const rule = applyOverrides
? buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {})
: buildRuleWithoutOverrides(ruleSO);
? buildRuleWithOverrides(completeRule, mergedDoc._source ?? {})
: buildRuleWithoutOverrides(completeRule);
const filteredSource = filterSource(mergedDoc);
const timestamp = new Date().toISOString();

67
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts
View file

@ -5,54 +5,49 @@
* 2.0.
*/
import { Logger } from 'kibana/server';
import type { ConfigType } from '../../../../config';
import { filterDuplicateSignals } from '../../signals/filter_duplicate_signals';
import { SearchAfterAndBulkCreateParams, SimpleHit, WrapHits } from '../../signals/types';
import { CompleteRule, RuleParams } from '../../schemas/rule_schemas';
import { ConfigType } from '../../../../config';
import { SimpleHit, WrapHits } from '../../signals/types';
import { generateId } from '../../signals/utils';
import { buildBulkBody } from './utils/build_bulk_body';
import { filterDuplicateSignals } from '../../signals/filter_duplicate_signals';
import { WrappedRACAlert } from '../types';
export const wrapHitsFactory =
({
logger,
completeRule,
ignoreFields,
mergeStrategy,
ruleSO,
signalsIndex,
spaceId,
}: {
logger: Logger;
ruleSO: SearchAfterAndBulkCreateParams['ruleSO'];
mergeStrategy: ConfigType['alertMergeStrategy'];
completeRule: CompleteRule<RuleParams>;
ignoreFields: ConfigType['alertIgnoreFields'];
mergeStrategy: ConfigType['alertMergeStrategy'];
signalsIndex: string;
spaceId: string | null | undefined;
}): WrapHits =>
(events, buildReasonMessage) => {
try {
const wrappedDocs = events.map((event) => {
return {
_index: '',
_id: generateId(
event._index,
event._id,
String(event._version),
ruleSO.attributes.params.ruleId ?? ''
),
_source: buildBulkBody(
spaceId,
ruleSO,
event as SimpleHit,
mergeStrategy,
ignoreFields,
true,
buildReasonMessage
),
};
});
const wrappedDocs: WrappedRACAlert[] = events.flatMap((event) => [
{
_index: signalsIndex,
_id: generateId(
event._index,
event._id,
String(event._version),
completeRule.ruleParams.ruleId ?? ''
),
_source: buildBulkBody(
spaceId,
completeRule,
event as SimpleHit,
mergeStrategy,
ignoreFields,
true,
buildReasonMessage
),
},
]);
return filterDuplicateSignals(ruleSO.id, wrappedDocs, true);
} catch (error) {
logger.error(error);
return [];
}
return filterDuplicateSignals(completeRule.alertId, wrappedDocs, false);
};

9
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_sequences_factory.ts
View file

@ -7,21 +7,22 @@
import { Logger } from 'kibana/server';
import { SearchAfterAndBulkCreateParams, WrapSequences } from '../../signals/types';
import { WrapSequences } from '../../signals/types';
import { buildAlertGroupFromSequence } from './utils/build_alert_group_from_sequence';
import { ConfigType } from '../../../../config';
import { WrappedRACAlert } from '../types';
import { CompleteRule, RuleParams } from '../../schemas/rule_schemas';
export const wrapSequencesFactory =
({
logger,
ruleSO,
completeRule,
ignoreFields,
mergeStrategy,
spaceId,
}: {
logger: Logger;
ruleSO: SearchAfterAndBulkCreateParams['ruleSO'];
completeRule: CompleteRule<RuleParams>;
ignoreFields: ConfigType['alertIgnoreFields'];
mergeStrategy: ConfigType['alertMergeStrategy'];
spaceId: string | null | undefined;
@ -33,7 +34,7 @@ export const wrapSequencesFactory =
...buildAlertGroupFromSequence(
logger,
sequence,
ruleSO,
completeRule,
mergeStrategy,
spaceId,
buildReasonMessage

2
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts
View file

@ -50,6 +50,8 @@ describe('Indicator Match Alerts', () => {
threatQuery: '*:*',
to: 'now',
type: 'threat_match',
query: '*:*',
language: 'kuery',
};
const { services, dependencies, executor } = createRuleTypeMocks('threat_match', params);
const securityRuleTypeWrapper = createSecurityRuleTypeWrapper({

6
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts
View file

@ -7,7 +7,7 @@
import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
import { INDICATOR_RULE_TYPE_ID } from '../../../../../common/constants';
import { ThreatRuleParams, threatRuleParams } from '../../schemas/rule_schemas';
import { CompleteRule, threatRuleParams, ThreatRuleParams } from '../../schemas/rule_schemas';
import { threatMatchExecutor } from '../../signals/executors/threat_match';
import { CreateRuleOptions, SecurityAlertType } from '../types';
@ -52,7 +52,7 @@ export const createIndicatorMatchAlertType = (
bulkCreate,
exceptionItems,
listClient,
rule,
completeRule,
searchAfterSize,
tuple,
wrapHits,
@ -69,7 +69,7 @@ export const createIndicatorMatchAlertType = (
eventsTelemetry: undefined,
listClient,
logger,
rule,
completeRule: completeRule as CompleteRule<ThreatRuleParams>,
searchAfterSize,
services,
tuple,

10
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts
View file

@ -7,7 +7,11 @@
import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
import { ML_RULE_TYPE_ID } from '../../../../../common/constants';
import { MachineLearningRuleParams, machineLearningRuleParams } from '../../schemas/rule_schemas';
import {
CompleteRule,
machineLearningRuleParams,
MachineLearningRuleParams,
} from '../../schemas/rule_schemas';
import { mlExecutor } from '../../signals/executors/ml';
import { CreateRuleOptions, SecurityAlertType } from '../types';
@ -52,7 +56,7 @@ export const createMlAlertType = (
bulkCreate,
exceptionItems,
listClient,
rule,
completeRule,
tuple,
wrapHits,
},
@ -67,7 +71,7 @@ export const createMlAlertType = (
listClient,
logger,
ml,
rule,
completeRule: completeRule as CompleteRule<MachineLearningRuleParams>,
services,
tuple,
wrapHits,

3
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts
View file

@ -51,6 +51,7 @@ describe('Custom Query Alerts', () => {
index: ['*'],
from: 'now-1m',
to: 'now',
language: 'kuery',
};
services.scopedClusterClient.asCurrentUser.search.mockReturnValue(
@ -95,6 +96,8 @@ describe('Custom Query Alerts', () => {
index: ['*'],
from: 'now-1m',
to: 'now',
language: 'kuery',
type: 'query',
};
services.scopedClusterClient.asCurrentUser.search.mockReturnValue(

6
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts
View file

@ -7,7 +7,7 @@
import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
import { QUERY_RULE_TYPE_ID } from '../../../../../common/constants';
import { QueryRuleParams, queryRuleParams } from '../../schemas/rule_schemas';
import { CompleteRule, queryRuleParams, QueryRuleParams } from '../../schemas/rule_schemas';
import { queryExecutor } from '../../signals/executors/query';
import { CreateRuleOptions, SecurityAlertType } from '../types';
@ -52,7 +52,7 @@ export const createQueryAlertType = (
bulkCreate,
exceptionItems,
listClient,
rule,
completeRule,
searchAfterSize,
tuple,
wrapHits,
@ -69,7 +69,7 @@ export const createQueryAlertType = (
eventsTelemetry: undefined,
listClient,
logger,
rule,
completeRule: completeRule as CompleteRule<QueryRuleParams>,
searchAfterSize,
services,
tuple,

6
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts
View file

@ -7,7 +7,7 @@
import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
import { THRESHOLD_RULE_TYPE_ID } from '../../../../../common/constants';
import { thresholdRuleParams, ThresholdRuleParams } from '../../schemas/rule_schemas';
import { CompleteRule, thresholdRuleParams, ThresholdRuleParams } from '../../schemas/rule_schemas';
import { thresholdExecutor } from '../../signals/executors/threshold';
import { ThresholdAlertState } from '../../signals/types';
import { CreateRuleOptions, SecurityAlertType } from '../types';
@ -48,7 +48,7 @@ export const createThresholdAlertType = (
producer: 'security-solution',
async executor(execOptions) {
const {
runOpts: { buildRuleMessage, bulkCreate, exceptionItems, rule, tuple, wrapHits },
runOpts: { buildRuleMessage, bulkCreate, exceptionItems, completeRule, tuple, wrapHits },
services,
startedAt,
state,
@ -60,7 +60,7 @@ export const createThresholdAlertType = (
exceptionItems,
experimentalFeatures,
logger,
rule,
completeRule: completeRule as CompleteRule<ThresholdRuleParams>,
services,
startedAt,
state,

6
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts
View file

@ -12,7 +12,6 @@ import { Logger } from '@kbn/logging';
import { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import { AlertExecutorOptions, AlertType } from '../../../../../alerting/server';
import { SavedObject } from '../../../../../../../src/core/server';
import {
AlertInstanceContext,
AlertInstanceState,
@ -26,10 +25,9 @@ import { PersistenceServices, IRuleDataClient } from '../../../../../rule_regist
import { BaseHit } from '../../../../common/detection_engine/types';
import { ConfigType } from '../../../config';
import { SetupPlugins } from '../../../plugin';
import { RuleParams } from '../schemas/rule_schemas';
import { CompleteRule, RuleParams } from '../schemas/rule_schemas';
import { BuildRuleMessage } from '../signals/rule_messages';
import {
AlertAttributes,
BulkCreate,
SearchAfterAndBulkCreateReturnType,
WrapHits,
@ -57,7 +55,7 @@ export interface RunOpts<TParams extends RuleParams> {
bulkCreate: BulkCreate;
exceptionItems: ExceptionListItemSchema[];
listClient: ListClient;
rule: SavedObject<AlertAttributes<TParams>>;
completeRule: CompleteRule<RuleParams>;
searchAfterSize: number;
tuple: {
to: Moment;

30
x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.mock.ts
View file

@ -10,12 +10,16 @@ import { getListArrayMock } from '../../../../common/detection_engine/schemas/ty
import { getThreatMappingMock } from '../signals/threat_mapping/build_threat_mapping_filter.mock';
import {
BaseRuleParams,
CompleteRule,
EqlRuleParams,
MachineLearningRuleParams,
QueryRuleParams,
RuleParams,
ThreatRuleParams,
ThresholdRuleParams,
} from './rule_schemas';
import { SanitizedRuleConfig } from '../../../../../alerting/common';
import { sampleRuleGuid } from '../signals/__mocks__/es_results';
const getBaseRuleParams = (): BaseRuleParams => {
return {
@ -132,3 +136,29 @@ export const getThreatRuleParams = (): ThreatRuleParams => {
itemsPerSearch: undefined,
};
};
export const getRuleConfigMock = (type: string = 'rule-type'): SanitizedRuleConfig => ({
actions: [],
enabled: true,
name: 'rule-name',
tags: ['some fake tag 1', 'some fake tag 2'],
createdBy: 'sample user',
createdAt: new Date('2020-03-27T22:55:59.577Z'),
updatedAt: new Date('2020-03-27T22:55:59.577Z'),
updatedBy: 'sample user',
schedule: {
interval: '5m',
},
throttle: 'no_actions',
consumer: 'sample consumer',
notifyWhen: null,
producer: 'sample producer',
ruleTypeId: `${type}-id`,
ruleTypeName: type,
});
export const getCompleteRuleMock = <T extends RuleParams>(params: T): CompleteRule<T> => ({
alertId: sampleRuleGuid,
ruleParams: params,
ruleConfig: getRuleConfigMock(),
});

7
x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts
View file

@ -72,6 +72,7 @@ import {
EQL_RULE_TYPE_ID,
THRESHOLD_RULE_TYPE_ID,
} from '../../../../common/constants';
import { SanitizedRuleConfig } from '../../../../../alerting/common';
const nonEqlLanguages = t.keyof({ kuery: null, lucene: null });
export const baseRuleParams = t.exact(
@ -199,6 +200,12 @@ export type TypeSpecificRuleParams = t.TypeOf<typeof typeSpecificRuleParams>;
export const ruleParams = t.intersection([baseRuleParams, typeSpecificRuleParams]);
export type RuleParams = t.TypeOf<typeof ruleParams>;
export interface CompleteRule<T extends RuleParams> {
alertId: string;
ruleParams: T;
ruleConfig: SanitizedRuleConfig;
}
export const notifyWhen = t.union([
t.literal('onActionGroupChange'),
t.literal('onActiveAlert'),

32
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule_preview.sh Normal file
View file

@ -0,0 +1,32 @@
#!/bin/sh
#
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License
# 2.0; you may not use this file except in compliance with the Elastic License
# 2.0.
#
set -e
./check_env_variables.sh
# Uses a default if no argument is specified
RULES=(${@:-./rules/queries/query_preview_threat_match.json})
# Example: ./post_rule.sh
# Example: ./post_rule.sh ./rules/queries/query_with_rule_id.json
# Example glob: ./post_rule.sh ./rules/queries/*
for RULE in "${RULES[@]}"
do {
[ -e "$RULE" ] || continue
curl -s -k \
-H 'Content-Type: application/json' \
-H 'kbn-xsrf: 123' \
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
-X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/rules/preview \
-d @${RULE} \
| jq -S .;
} &
done
wait

49
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_preview_threat_match.json Normal file
View file

@ -0,0 +1,49 @@
{
"name": "preview query",
"description": "preview query for custom query rule",
"false_positives": [
"https://www.example.com/some-article-about-a-false-positive",
"some text string about why another condition could be a false positive"
],
"rule_id": "preview-placeholder-rule-id",
"filters": [
{
"exists": {
"field": "file.hash.md5"
}
}
],
"enabled": false,
"invocationCount": 500,
"index": ["custom-events"],
"interval": "5m",
"query": "file.hash.md5 : *",
"language": "kuery",
"risk_score": 1,
"tags": ["tag 1", "tag 2", "any tag you want"],
"to": "now",
"from": "now-6m",
"severity": "high",
"type": "threat_match",
"references": [
"http://www.example.com/some-article-about-attack",
"Some plain text string here explaining why this is a valid thing to look out for"
],
"timeline_id": "timeline_id",
"timeline_title": "timeline_title",
"threat_index": ["custom-threats"],
"threat_query": "*:*",
"threat_mapping": [
{
"entries": [
{
"field": "file.hash.md5",
"type": "mapping",
"value": "threat.indicator.file.hash.md5"
}
]
}
],
"note": "# note markdown",
"version": 1
}

48
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts
View file

@ -9,7 +9,6 @@ import {
sampleDocNoSortId,
sampleIdGuid,
sampleDocWithAncestors,
sampleRuleSO,
sampleWrappedSignalHit,
expectedRule,
} from './__mocks__/es_results';
@ -22,7 +21,12 @@ import {
} from './build_bulk_body';
import { SignalHit, SignalSourceHit } from './types';
import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template';
import { getQueryRuleParams, getThresholdRuleParams } from '../schemas/rule_schemas.mock';
import {
getCompleteRuleMock,
getQueryRuleParams,
getThresholdRuleParams,
} from '../schemas/rule_schemas.mock';
import { QueryRuleParams, ThresholdRuleParams } from '../schemas/rule_schemas';
// This allows us to not have to use ts-expect-error with delete in the code.
type SignalHitOptionalTimestamp = Omit<SignalHit, '@timestamp'> & {
@ -35,12 +39,12 @@ describe('buildBulkBody', () => {
});
test('bulk body builds well-defined body', () => {
const ruleSO = sampleRuleSO(getQueryRuleParams());
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const doc = sampleDocNoSortId();
const buildReasonMessage = jest.fn().mockReturnValue('reasonable reason');
delete doc._source.source;
const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(
ruleSO,
completeRule,
doc,
'missingFields',
[],
@ -93,7 +97,7 @@ describe('buildBulkBody', () => {
});
test('bulk body builds well-defined body with threshold results', () => {
const ruleSO = sampleRuleSO(getThresholdRuleParams());
const completeRule = getCompleteRuleMock<ThresholdRuleParams>(getThresholdRuleParams());
const baseDoc = sampleDocNoSortId();
const buildReasonMessage = jest.fn().mockReturnValue('reasonable reason');
const doc: SignalSourceHit & { _source: Required<SignalSourceHit>['_source'] } = {
@ -112,7 +116,7 @@ describe('buildBulkBody', () => {
};
delete doc._source.source;
const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(
ruleSO,
completeRule,
doc,
'missingFields',
[],
@ -187,7 +191,7 @@ describe('buildBulkBody', () => {
});
test('bulk body builds original_event if it exists on the event to begin with', () => {
const ruleSO = sampleRuleSO(getQueryRuleParams());
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const doc = sampleDocNoSortId();
const buildReasonMessage = jest.fn().mockReturnValue('reasonable reason');
delete doc._source.source;
@ -198,7 +202,7 @@ describe('buildBulkBody', () => {
kind: 'event',
};
const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(
ruleSO,
completeRule,
doc,
'missingFields',
[],
@ -260,7 +264,7 @@ describe('buildBulkBody', () => {
});
test('bulk body builds original_event if it exists on the event to begin with but no kind information', () => {
const ruleSO = sampleRuleSO(getQueryRuleParams());
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const doc = sampleDocNoSortId();
const buildReasonMessage = jest.fn().mockReturnValue('reasonable reason');
delete doc._source.source;
@ -270,7 +274,7 @@ describe('buildBulkBody', () => {
dataset: 'socket',
};
const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(
ruleSO,
completeRule,
doc,
'missingFields',
[],
@ -331,7 +335,7 @@ describe('buildBulkBody', () => {
});
test('bulk body builds original_event if it exists on the event to begin with with only kind information', () => {
const ruleSO = sampleRuleSO(getQueryRuleParams());
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const doc = sampleDocNoSortId();
const buildReasonMessage = jest.fn().mockReturnValue('reasonable reason');
delete doc._source.source;
@ -339,7 +343,7 @@ describe('buildBulkBody', () => {
kind: 'event',
};
const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(
ruleSO,
completeRule,
doc,
'missingFields',
[],
@ -395,7 +399,7 @@ describe('buildBulkBody', () => {
});
test('bulk body builds "original_signal" if it exists already as a numeric', () => {
const ruleSO = sampleRuleSO(getQueryRuleParams());
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const sampleDoc = sampleDocNoSortId();
const buildReasonMessage = jest.fn().mockReturnValue('reasonable reason');
delete sampleDoc._source.source;
@ -407,7 +411,7 @@ describe('buildBulkBody', () => {
},
} as unknown as SignalSourceHit;
const { '@timestamp': timestamp, ...fakeSignalSourceHit } = buildBulkBody(
ruleSO,
completeRule,
doc,
'missingFields',
[],
@ -459,7 +463,7 @@ describe('buildBulkBody', () => {
});
test('bulk body builds "original_signal" if it exists already as an object', () => {
const ruleSO = sampleRuleSO(getQueryRuleParams());
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const sampleDoc = sampleDocNoSortId();
const buildReasonMessage = jest.fn().mockReturnValue('reasonable reason');
delete sampleDoc._source.source;
@ -471,7 +475,7 @@ describe('buildBulkBody', () => {
},
} as unknown as SignalSourceHit;
const { '@timestamp': timestamp, ...fakeSignalSourceHit } = buildBulkBody(
ruleSO,
completeRule,
doc,
'missingFields',
[],
@ -531,11 +535,11 @@ describe('buildSignalFromSequence', () => {
const block2 = sampleWrappedSignalHit();
block2._source.new_key = 'new_key_value';
const blocks = [block1, block2];
const ruleSO = sampleRuleSO(getQueryRuleParams());
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const buildReasonMessage = jest.fn().mockReturnValue('reasonable reason');
const signal: SignalHitOptionalTimestamp = buildSignalFromSequence(
blocks,
ruleSO,
completeRule,
buildReasonMessage
);
// Timestamp will potentially always be different so remove it for the test
@ -622,11 +626,11 @@ describe('buildSignalFromSequence', () => {
const block2 = sampleWrappedSignalHit();
block2._source['@timestamp'] = '2021-05-20T22:28:46+0000';
block2._source.someKey = 'someOtherValue';
const ruleSO = sampleRuleSO(getQueryRuleParams());
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const buildReasonMessage = jest.fn().mockReturnValue('reasonable reason');
const signal: SignalHitOptionalTimestamp = buildSignalFromSequence(
[block1, block2],
ruleSO,
completeRule,
buildReasonMessage
);
// Timestamp will potentially always be different so remove it for the test
@ -712,11 +716,11 @@ describe('buildSignalFromEvent', () => {
test('builds a basic signal from a single event', () => {
const ancestor = sampleDocWithAncestors().hits.hits[0];
delete ancestor._source.source;
const ruleSO = sampleRuleSO(getQueryRuleParams());
const completeRule = getCompleteRuleMock<QueryRuleParams>(getQueryRuleParams());
const buildReasonMessage = jest.fn().mockReturnValue('reasonable reason');
const signal: SignalHitOptionalTimestamp = buildSignalFromEvent(
ancestor,
ruleSO,
completeRule,
true,
'missingFields',
[],

29
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts
View file

@ -6,10 +6,8 @@
*/
import { TIMESTAMP } from '@kbn/rule-data-utils';
import { SavedObject } from 'src/core/types';
import { getMergeStrategy } from './source_fields_merging/strategies';
import {
AlertAttributes,
SignalSourceHit,
SignalHit,
Signal,
@ -24,25 +22,26 @@ import { EqlSequence } from '../../../../common/detection_engine/types';
import { generateSignalId, wrapBuildingBlocks, wrapSignal } from './utils';
import type { ConfigType } from '../../../config';
import { BuildReasonMessage } from './reason_formatters';
import { CompleteRule, RuleParams } from '../schemas/rule_schemas';
/**
* Formats the search_after result for insertion into the signals index. We first create a
* "best effort" merged "fields" with the "_source" object, then build the signal object,
* then the event object, and finally we strip away any additional temporary data that was added
* such as the "threshold_result".
* @param ruleSO The rule saved object to build overrides
* @param completeRule The rule object to build overrides
* @param doc The SignalSourceHit with "_source", "fields", and additional data such as "threshold_result"
* @returns The body that can be added to a bulk call for inserting the signal.
*/
export const buildBulkBody = (
ruleSO: SavedObject<AlertAttributes>,
completeRule: CompleteRule<RuleParams>,
doc: SignalSourceHit,
mergeStrategy: ConfigType['alertMergeStrategy'],
ignoreFields: ConfigType['alertIgnoreFields'],
buildReasonMessage: BuildReasonMessage
): SignalHit => {
const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields });
const rule = buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {});
const rule = buildRuleWithOverrides(completeRule, mergedDoc._source ?? {});
const timestamp = new Date().toISOString();
const reason = buildReasonMessage({ mergedDoc, rule });
const signal: Signal = {
@ -74,12 +73,12 @@ export const buildBulkBody = (
* one signal for each event in the sequence, and a "shell" signal that ties them all together. All N+1 signals
* share the same signal.group.id to make it easy to query them.
* @param sequence The raw ES documents that make up the sequence
* @param ruleSO SavedObject representing the rule that found the sequence
* @param completeRule rule object representing the rule that found the sequence
* @param outputIndex Index to write the resulting signals to
*/
export const buildSignalGroupFromSequence = (
sequence: EqlSequence<SignalSource>,
ruleSO: SavedObject<AlertAttributes>,
completeRule: CompleteRule<RuleParams>,
outputIndex: string,
mergeStrategy: ConfigType['alertMergeStrategy'],
ignoreFields: ConfigType['alertIgnoreFields'],
@ -89,7 +88,7 @@ export const buildSignalGroupFromSequence = (
sequence.events.map((event) => {
const signal = buildSignalFromEvent(
event,
ruleSO,
completeRule,
false,
mergeStrategy,
ignoreFields,
@ -103,7 +102,7 @@ export const buildSignalGroupFromSequence = (
if (
wrappedBuildingBlocks.some((block) =>
block._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleSO.id)
block._source.signal?.ancestors.some((ancestor) => ancestor.rule === completeRule.alertId)
)
) {
return [];
@ -113,7 +112,7 @@ export const buildSignalGroupFromSequence = (
// we can build the signal that links the building blocks together
// and also insert the group id (which is also the "shell" signal _id) in each building block
const sequenceSignal = wrapSignal(
buildSignalFromSequence(wrappedBuildingBlocks, ruleSO, buildReasonMessage),
buildSignalFromSequence(wrappedBuildingBlocks, completeRule, buildReasonMessage),
outputIndex
);
wrappedBuildingBlocks.forEach((block, idx) => {
@ -130,10 +129,10 @@ export const buildSignalGroupFromSequence = (
export const buildSignalFromSequence = (
events: WrappedSignalHit[],
ruleSO: SavedObject<AlertAttributes>,
completeRule: CompleteRule<RuleParams>,
buildReasonMessage: BuildReasonMessage
): SignalHit => {
const rule = buildRuleWithoutOverrides(ruleSO);
const rule = buildRuleWithoutOverrides(completeRule);
const timestamp = new Date().toISOString();
const mergedEvents = objectArrayIntersection(events.map((event) => event._source));
const reason = buildReasonMessage({ rule, mergedDoc: mergedEvents as SignalSourceHit });
@ -157,7 +156,7 @@ export const buildSignalFromSequence = (
export const buildSignalFromEvent = (
event: BaseSignalHit,
ruleSO: SavedObject<AlertAttributes>,
completeRule: CompleteRule<RuleParams>,
applyOverrides: boolean,
mergeStrategy: ConfigType['alertMergeStrategy'],
ignoreFields: ConfigType['alertIgnoreFields'],
@ -165,8 +164,8 @@ export const buildSignalFromEvent = (
): SignalHit => {
const mergedEvent = getMergeStrategy(mergeStrategy)({ doc: event, ignoreFields });
const rule = applyOverrides
? buildRuleWithOverrides(ruleSO, mergedEvent._source ?? {})
: buildRuleWithoutOverrides(ruleSO);
? buildRuleWithOverrides(completeRule, mergedEvent._source ?? {})
: buildRuleWithoutOverrides(completeRule);
const timestamp = new Date().toISOString();
const reason = buildReasonMessage({ mergedDoc: mergedEvent, rule });
const signal: Signal = {

72
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts
View file

@ -6,38 +6,47 @@
*/
import { buildRuleWithOverrides, buildRuleWithoutOverrides } from './build_rule';
import {
sampleDocNoSortId,
expectedRule,
sampleDocSeverity,
sampleRuleSO,
} from './__mocks__/es_results';
import { sampleDocNoSortId, expectedRule, sampleDocSeverity } from './__mocks__/es_results';
import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants';
import { getQueryRuleParams, getThreatRuleParams } from '../schemas/rule_schemas.mock';
import { ThreatRuleParams } from '../schemas/rule_schemas';
import {
getCompleteRuleMock,
getQueryRuleParams,
getThreatRuleParams,
} from '../schemas/rule_schemas.mock';
import {
CompleteRule,
QueryRuleParams,
RuleParams,
ThreatRuleParams,
} from '../schemas/rule_schemas';
describe('buildRuleWithoutOverrides', () => {
let params: RuleParams;
let completeRule: CompleteRule<QueryRuleParams>;
beforeEach(() => {
params = getQueryRuleParams();
completeRule = getCompleteRuleMock<QueryRuleParams>(params);
});
test('builds a rule using rule alert', () => {
const ruleSO = sampleRuleSO(getQueryRuleParams());
const rule = buildRuleWithoutOverrides(ruleSO);
const rule = buildRuleWithoutOverrides(completeRule);
expect(rule).toEqual(expectedRule());
});
test('builds a rule and removes internal tags', () => {
const ruleSO = sampleRuleSO(getQueryRuleParams());
ruleSO.attributes.tags = [
completeRule.ruleConfig.tags = [
'some fake tag 1',
'some fake tag 2',
`${INTERNAL_RULE_ID_KEY}:rule-1`,
`${INTERNAL_IMMUTABLE_KEY}:true`,
];
const rule = buildRuleWithoutOverrides(ruleSO);
const rule = buildRuleWithoutOverrides(completeRule);
expect(rule.tags).toEqual(['some fake tag 1', 'some fake tag 2']);
});
test('it builds a rule as expected with filters present', () => {
const ruleSO = sampleRuleSO(getQueryRuleParams());
const ruleFilters = [
{
query: 'host.name: Rebecca',
@ -49,8 +58,8 @@ describe('buildRuleWithoutOverrides', () => {
query: 'host.name: Braden',
},
];
ruleSO.attributes.params.filters = ruleFilters;
const rule = buildRuleWithoutOverrides(ruleSO);
completeRule.ruleParams.filters = ruleFilters;
const rule = buildRuleWithoutOverrides(completeRule);
expect(rule.filters).toEqual(ruleFilters);
});
@ -90,8 +99,8 @@ describe('buildRuleWithoutOverrides', () => {
threatIndex: ['threat_index'],
threatLanguage: 'kuery',
};
const ruleSO = sampleRuleSO(ruleParams);
const threatMatchRule = buildRuleWithoutOverrides(ruleSO);
const threatMatchCompleteRule = getCompleteRuleMock<ThreatRuleParams>(ruleParams);
const threatMatchRule = buildRuleWithoutOverrides(threatMatchCompleteRule);
const expected: Partial<RulesSchema> = {
threat_mapping: ruleParams.threatMapping,
threat_filters: ruleParams.threatFilters,
@ -105,10 +114,17 @@ describe('buildRuleWithoutOverrides', () => {
});
describe('buildRuleWithOverrides', () => {
let params: RuleParams;
let completeRule: CompleteRule<QueryRuleParams>;
beforeEach(() => {
params = getQueryRuleParams();
completeRule = getCompleteRuleMock<QueryRuleParams>(params);
});
test('it applies rule name override in buildRule', () => {
const ruleSO = sampleRuleSO(getQueryRuleParams());
ruleSO.attributes.params.ruleNameOverride = 'someKey';
const rule = buildRuleWithOverrides(ruleSO, sampleDocNoSortId()._source);
completeRule.ruleParams.ruleNameOverride = 'someKey';
const rule = buildRuleWithOverrides(completeRule, sampleDocNoSortId()._source!);
const expected = {
...expectedRule(),
name: 'someValue',
@ -123,8 +139,7 @@ describe('buildRuleWithOverrides', () => {
test('it applies risk score override in buildRule', () => {
const newRiskScore = 79;
const ruleSO = sampleRuleSO(getQueryRuleParams());
ruleSO.attributes.params.riskScoreMapping = [
completeRule.ruleParams.riskScoreMapping = [
{
field: 'new_risk_score',
// value and risk_score aren't used for anything but are required in the schema
@ -135,11 +150,11 @@ describe('buildRuleWithOverrides', () => {
];
const doc = sampleDocNoSortId();
doc._source.new_risk_score = newRiskScore;
const rule = buildRuleWithOverrides(ruleSO, doc._source);
const rule = buildRuleWithOverrides(completeRule, doc._source!);
const expected = {
...expectedRule(),
risk_score: newRiskScore,
risk_score_mapping: ruleSO.attributes.params.riskScoreMapping,
risk_score_mapping: completeRule.ruleParams.riskScoreMapping,
meta: {
riskScoreOverridden: true,
someMeta: 'someField',
@ -150,8 +165,7 @@ describe('buildRuleWithOverrides', () => {
test('it applies severity override in buildRule', () => {
const eventSeverity = '42';
const ruleSO = sampleRuleSO(getQueryRuleParams());
ruleSO.attributes.params.severityMapping = [
completeRule.ruleParams.severityMapping = [
{
field: 'event.severity',
value: eventSeverity,
@ -160,11 +174,11 @@ describe('buildRuleWithOverrides', () => {
},
];
const doc = sampleDocSeverity(Number(eventSeverity));
const rule = buildRuleWithOverrides(ruleSO, doc._source!);
const rule = buildRuleWithOverrides(completeRule, doc._source!);
const expected = {
...expectedRule(),
severity: 'critical',
severity_mapping: ruleSO.attributes.params.severityMapping,
severity_mapping: completeRule.ruleParams.severityMapping,
meta: {
severityOverrideField: 'event.severity',
someMeta: 'someField',

50
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts
View file

@ -5,41 +5,53 @@
* 2.0.
*/
import { SavedObject } from 'src/core/types';
import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
import { buildRiskScoreFromMapping } from './mappings/build_risk_score_from_mapping';
import { AlertAttributes, SignalSource } from './types';
import { SignalSource } from './types';
import { buildSeverityFromMapping } from './mappings/build_severity_from_mapping';
import { buildRuleNameFromMapping } from './mappings/build_rule_name_from_mapping';
import { RuleParams } from '../schemas/rule_schemas';
import { CompleteRule, RuleParams } from '../schemas/rule_schemas';
import { commonParamsCamelToSnake, typeSpecificCamelToSnake } from '../schemas/rule_converters';
import { transformTags } from '../routes/rules/utils';
import { transformAlertToRuleAction } from '../../../../common/detection_engine/transform_actions';
export const buildRuleWithoutOverrides = (ruleSO: SavedObject<AlertAttributes>): RulesSchema => {
const ruleParams = ruleSO.attributes.params;
export const buildRuleWithoutOverrides = (completeRule: CompleteRule<RuleParams>): RulesSchema => {
const ruleParams = completeRule.ruleParams;
const {
actions,
schedule,
name,
tags,
enabled,
createdBy,
updatedBy,
throttle,
createdAt,
updatedAt,
} = completeRule.ruleConfig;
return {
id: ruleSO.id,
actions: ruleSO.attributes.actions,
interval: ruleSO.attributes.schedule.interval,
name: ruleSO.attributes.name,
tags: transformTags(ruleSO.attributes.tags),
enabled: ruleSO.attributes.enabled,
created_by: ruleSO.attributes.createdBy,
updated_by: ruleSO.attributes.updatedBy,
throttle: ruleSO.attributes.throttle,
created_at: ruleSO.attributes.createdAt,
updated_at: ruleSO.updated_at ?? '',
actions: actions.map(transformAlertToRuleAction),
created_at: createdAt.toISOString(),
created_by: createdBy ?? '',
enabled,
id: completeRule.alertId,
interval: schedule.interval,
name,
tags: transformTags(tags),
throttle: throttle ?? undefined,
updated_at: updatedAt.toISOString(),
updated_by: updatedBy ?? '',
...commonParamsCamelToSnake(ruleParams),
...typeSpecificCamelToSnake(ruleParams),
};
};
export const buildRuleWithOverrides = (
ruleSO: SavedObject<AlertAttributes>,
completeRule: CompleteRule<RuleParams>,
eventSource: SignalSource
): RulesSchema => {
const ruleWithoutOverrides = buildRuleWithoutOverrides(ruleSO);
return applyRuleOverrides(ruleWithoutOverrides, eventSource, ruleSO.attributes.params);
const ruleWithoutOverrides = buildRuleWithoutOverrides(completeRule);
return applyRuleOverrides(ruleWithoutOverrides, eventSource, completeRule.ruleParams);
};
export const applyRuleOverrides = (

5
x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts
View file

@ -27,7 +27,8 @@ export const bulkCreateFactory =
logger: Logger,
esClient: ElasticsearchClient,
buildRuleMessage: BuildRuleMessage,
refreshForBulkCreate: RefreshTypes
refreshForBulkCreate: RefreshTypes,
indexNameOverride?: string
) =>
async <T>(wrappedDocs: Array<BaseHit<T>>): Promise<GenericBulkCreateResponse<T>> => {
if (wrappedDocs.length === 0) {
@ -43,7 +44,7 @@ export const bulkCreateFactory =
const bulkBody = wrappedDocs.flatMap((wrappedDoc) => [
{
create: {
_index: wrappedDoc._index,
_index: indexNameOverride ?? wrappedDoc._index,
_id: wrappedDoc._id,
},
},

8
x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
View file

@ -8,7 +8,7 @@ import type { estypes } from '@elastic/elasticsearch';
import { flow, omit } from 'lodash/fp';
import set from 'set-value';
import { Logger, SavedObject } from '../../../../../../../src/core/server';
import { Logger } from '../../../../../../../src/core/server';
import {
AlertInstanceContext,
AlertInstanceState,
@ -17,13 +17,13 @@ import {
import { GenericBulkCreateResponse } from './bulk_create_factory';
import { AnomalyResults, Anomaly } from '../../machine_learning';
import { BuildRuleMessage } from './rule_messages';
import { AlertAttributes, BulkCreate, WrapHits } from './types';
import { MachineLearningRuleParams } from '../schemas/rule_schemas';
import { BulkCreate, WrapHits } from './types';
import { CompleteRule, MachineLearningRuleParams } from '../schemas/rule_schemas';
import { buildReasonMessageForMlAlert } from './reason_formatters';
interface BulkCreateMlSignalsParams {
someResult: AnomalyResults;
ruleSO: SavedObject<AlertAttributes<MachineLearningRuleParams>>;
completeRule: CompleteRule<MachineLearningRuleParams>;
services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
logger: Logger;
id: string;

28
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts
View file

@ -11,12 +11,13 @@ import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server
import { eqlExecutor } from './eql';
import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock';
import { getEqlRuleParams } from '../../schemas/rule_schemas.mock';
import { getCompleteRuleMock, getEqlRuleParams } from '../../schemas/rule_schemas.mock';
import { getIndexVersion } from '../../routes/index/get_index_version';
import { SIGNALS_TEMPLATE_VERSION } from '../../routes/index/get_signals_template';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks';
import { allowedExperimentalValues } from '../../../../../common/experimental_features';
import { EqlRuleParams } from '../../schemas/rule_schemas';
jest.mock('../../routes/index/get_index_version');
@ -26,28 +27,7 @@ describe('eql_executor', () => {
let alertServices: AlertServicesMock;
(getIndexVersion as jest.Mock).mockReturnValue(SIGNALS_TEMPLATE_VERSION);
const params = getEqlRuleParams();
const eqlSO = {
id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
type: 'alert',
version: '1',
updated_at: '2020-03-27T22:55:59.577Z',
attributes: {
actions: [],
alertTypeId: 'siem.signals',
enabled: true,
name: 'rule-name',
tags: ['some fake tag 1', 'some fake tag 2'],
createdBy: 'sample user',
createdAt: '2020-03-27T22:55:59.577Z',
updatedBy: 'sample user',
schedule: {
interval: '5m',
},
throttle: 'no_actions',
params,
},
references: [],
};
const eqlCompleteRule = getCompleteRuleMock<EqlRuleParams>(params);
const tuple = {
from: dateMath.parse(params.from)!,
to: dateMath.parse(params.to)!,
@ -72,7 +52,7 @@ describe('eql_executor', () => {
it('should set a warning when exception list for eql rule contains value list exceptions', async () => {
const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })];
const response = await eqlExecutor({
rule: eqlSO,
completeRule: eqlCompleteRule,
tuple,
exceptionItems,
experimentalFeatures: allowedExperimentalValues,

12
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts
View file

@ -7,7 +7,6 @@
import { ApiResponse } from '@elastic/elasticsearch';
import { performance } from 'perf_hooks';
import { SavedObject } from 'src/core/types';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import { Logger } from 'src/core/server';
import {
@ -20,11 +19,9 @@ import { hasLargeValueItem } from '../../../../../common/detection_engine/utils'
import { isOutdated } from '../../migrations/helpers';
import { getIndexVersion } from '../../routes/index/get_index_version';
import { MIN_EQL_RULE_INDEX_VERSION } from '../../routes/index/get_signals_template';
import { EqlRuleParams } from '../../schemas/rule_schemas';
import { getInputIndex } from '../get_input_output_index';
import {
AlertAttributes,
BulkCreate,
WrapHits,
WrapSequences,
@ -36,9 +33,10 @@ import {
import { createSearchAfterReturnType, makeFloatString } from '../utils';
import { ExperimentalFeatures } from '../../../../../common/experimental_features';
import { buildReasonMessageForEqlAlert } from '../reason_formatters';
import { CompleteRule, EqlRuleParams } from '../../schemas/rule_schemas';
export const eqlExecutor = async ({
rule,
completeRule,
tuple,
exceptionItems,
experimentalFeatures,
@ -50,7 +48,7 @@ export const eqlExecutor = async ({
wrapHits,
wrapSequences,
}: {
rule: SavedObject<AlertAttributes<EqlRuleParams>>;
completeRule: CompleteRule<EqlRuleParams>;
tuple: RuleRangeTuple;
exceptionItems: ExceptionListItemSchema[];
experimentalFeatures: ExperimentalFeatures;
@ -63,7 +61,9 @@ export const eqlExecutor = async ({
wrapSequences: WrapSequences;
}): Promise<SearchAfterAndBulkCreateReturnType> => {
const result = createSearchAfterReturnType();
const ruleParams = rule.attributes.params;
const ruleParams = completeRule.ruleParams;
if (hasLargeValueItem(exceptionItems)) {
result.warningMessages.push(
'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules'

21
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts
View file

@ -10,13 +10,13 @@ import { loggingSystemMock } from 'src/core/server/mocks';
import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks';
import { mlExecutor } from './ml';
import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
import { getMlRuleParams } from '../../schemas/rule_schemas.mock';
import { getCompleteRuleMock, getMlRuleParams } from '../../schemas/rule_schemas.mock';
import { buildRuleMessageFactory } from '../rule_messages';
import { getListClientMock } from '../../../../../../lists/server/services/lists/list_client.mock';
import { findMlSignals } from '../find_ml_signals';
import { bulkCreateMlSignals } from '../bulk_create_ml_signals';
import { mlPluginServerMock } from '../../../../../../ml/server/mocks';
import { sampleRuleSO } from '../__mocks__/es_results';
import { MachineLearningRuleParams } from '../../schemas/rule_schemas';
jest.mock('../find_ml_signals');
jest.mock('../bulk_create_ml_signals');
@ -28,17 +28,18 @@ describe('ml_executor', () => {
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
let alertServices: AlertServicesMock;
const params = getMlRuleParams();
const mlSO = sampleRuleSO(params);
const mlCompleteRule = getCompleteRuleMock<MachineLearningRuleParams>(params);
const tuple = {
from: dateMath.parse(params.from)!,
to: dateMath.parse(params.to)!,
maxSignals: params.maxSignals,
};
const buildRuleMessage = buildRuleMessageFactory({
id: mlSO.id,
ruleId: mlSO.attributes.params.ruleId,
name: mlSO.attributes.name,
index: mlSO.attributes.params.outputIndex,
id: mlCompleteRule.alertId,
ruleId: mlCompleteRule.ruleParams.ruleId,
name: mlCompleteRule.ruleConfig.name,
index: mlCompleteRule.ruleParams.outputIndex,
});
beforeEach(() => {
@ -66,7 +67,7 @@ describe('ml_executor', () => {
it('should throw an error if ML plugin was not available', async () => {
await expect(
mlExecutor({
rule: mlSO,
completeRule: mlCompleteRule,
tuple,
ml: undefined,
exceptionItems,
@ -83,7 +84,7 @@ describe('ml_executor', () => {
it('should record a partial failure if Machine learning job summary was null', async () => {
jobsSummaryMock.mockResolvedValue([]);
const response = await mlExecutor({
rule: mlSO,
completeRule: mlCompleteRule,
tuple,
ml: mlMock,
exceptionItems,
@ -109,7 +110,7 @@ describe('ml_executor', () => {
]);
const response = await mlExecutor({
rule: mlSO,
completeRule: mlCompleteRule,
tuple,
ml: mlMock,
exceptionItems,

17
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts
View file

@ -6,7 +6,6 @@
*/
import { KibanaRequest, Logger } from 'src/core/server';
import { SavedObject } from 'src/core/types';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import {
AlertInstanceContext,
@ -15,17 +14,17 @@ import {
} from '../../../../../../alerting/server';
import { ListClient } from '../../../../../../lists/server';
import { isJobStarted } from '../../../../../common/machine_learning/helpers';
import { SetupPlugins } from '../../../../plugin';
import { MachineLearningRuleParams } from '../../schemas/rule_schemas';
import { CompleteRule, MachineLearningRuleParams } from '../../schemas/rule_schemas';
import { bulkCreateMlSignals } from '../bulk_create_ml_signals';
import { filterEventsAgainstList } from '../filters/filter_events_against_list';
import { findMlSignals } from '../find_ml_signals';
import { BuildRuleMessage } from '../rule_messages';
import { AlertAttributes, BulkCreate, RuleRangeTuple, WrapHits } from '../types';
import { BulkCreate, RuleRangeTuple, WrapHits } from '../types';
import { createErrorsFromShard, createSearchAfterReturnType, mergeReturns } from '../utils';
import { SetupPlugins } from '../../../../plugin';
export const mlExecutor = async ({
rule,
completeRule,
tuple,
ml,
listClient,
@ -36,7 +35,7 @@ export const mlExecutor = async ({
bulkCreate,
wrapHits,
}: {
rule: SavedObject<AlertAttributes<MachineLearningRuleParams>>;
completeRule: CompleteRule<MachineLearningRuleParams>;
tuple: RuleRangeTuple;
ml: SetupPlugins['ml'];
listClient: ListClient;
@ -48,7 +47,7 @@ export const mlExecutor = async ({
wrapHits: WrapHits;
}) => {
const result = createSearchAfterReturnType();
const ruleParams = rule.attributes.params;
const ruleParams = completeRule.ruleParams;
if (ml == null) {
throw new Error('ML plugin unavailable during rule execution');
}
@ -110,10 +109,10 @@ export const mlExecutor = async ({
const { success, errors, bulkCreateDuration, createdItemsCount, createdItems } =
await bulkCreateMlSignals({
someResult: filteredAnomalyResults,
ruleSO: rule,
completeRule,
services,
logger,
id: rule.id,
id: completeRule.alertId,
signalsIndex: ruleParams.outputIndex,
buildRuleMessage,
bulkCreate,

16
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts
View file

@ -5,7 +5,6 @@
* 2.0.
*/
import { SavedObject } from 'src/core/types';
import { Logger } from 'src/core/server';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import {
@ -17,15 +16,15 @@ import { ListClient } from '../../../../../../lists/server';
import { getFilter } from '../get_filter';
import { getInputIndex } from '../get_input_output_index';
import { searchAfterAndBulkCreate } from '../search_after_bulk_create';
import { AlertAttributes, RuleRangeTuple, BulkCreate, WrapHits } from '../types';
import { RuleRangeTuple, BulkCreate, WrapHits } from '../types';
import { TelemetryEventsSender } from '../../../telemetry/sender';
import { BuildRuleMessage } from '../rule_messages';
import { QueryRuleParams, SavedQueryRuleParams } from '../../schemas/rule_schemas';
import { CompleteRule, SavedQueryRuleParams, QueryRuleParams } from '../../schemas/rule_schemas';
import { ExperimentalFeatures } from '../../../../../common/experimental_features';
import { buildReasonMessageForQueryAlert } from '../reason_formatters';
export const queryExecutor = async ({
rule,
completeRule,
tuple,
listClient,
exceptionItems,
@ -39,7 +38,7 @@ export const queryExecutor = async ({
bulkCreate,
wrapHits,
}: {
rule: SavedObject<AlertAttributes<QueryRuleParams | SavedQueryRuleParams>>;
completeRule: CompleteRule<QueryRuleParams | SavedQueryRuleParams>;
tuple: RuleRangeTuple;
listClient: ListClient;
exceptionItems: ExceptionListItemSchema[];
@ -53,7 +52,8 @@ export const queryExecutor = async ({
bulkCreate: BulkCreate;
wrapHits: WrapHits;
}) => {
const ruleParams = rule.attributes.params;
const ruleParams = completeRule.ruleParams;
const inputIndex = await getInputIndex({
experimentalFeatures,
services,
@ -76,11 +76,11 @@ export const queryExecutor = async ({
tuple,
listClient,
exceptionsList: exceptionItems,
ruleSO: rule,
completeRule,
services,
logger,
eventsTelemetry,
id: rule.id,
id: completeRule.alertId,
inputIndexPattern: inputIndex,
signalsIndex: ruleParams.outputIndex,
filter: esFilter,

47
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts
View file

@ -5,7 +5,6 @@
* 2.0.
*/
import { SavedObject } from 'src/core/types';
import { Logger } from 'src/core/server';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import {
@ -15,15 +14,15 @@ import {
} from '../../../../../../alerting/server';
import { ListClient } from '../../../../../../lists/server';
import { getInputIndex } from '../get_input_output_index';
import { RuleRangeTuple, AlertAttributes, BulkCreate, WrapHits } from '../types';
import { RuleRangeTuple, BulkCreate, WrapHits } from '../types';
import { TelemetryEventsSender } from '../../../telemetry/sender';
import { BuildRuleMessage } from '../rule_messages';
import { createThreatSignals } from '../threat_mapping/create_threat_signals';
import { ThreatRuleParams } from '../../schemas/rule_schemas';
import { CompleteRule, ThreatRuleParams } from '../../schemas/rule_schemas';
import { ExperimentalFeatures } from '../../../../../common/experimental_features';
export const threatMatchExecutor = async ({
rule,
completeRule,
tuple,
listClient,
exceptionItems,
@ -37,7 +36,7 @@ export const threatMatchExecutor = async ({
bulkCreate,
wrapHits,
}: {
rule: SavedObject<AlertAttributes<ThreatRuleParams>>;
completeRule: CompleteRule<ThreatRuleParams>;
tuple: RuleRangeTuple;
listClient: ListClient;
exceptionItems: ExceptionListItemSchema[];
@ -51,7 +50,7 @@ export const threatMatchExecutor = async ({
bulkCreate: BulkCreate;
wrapHits: WrapHits;
}) => {
const ruleParams = rule.attributes.params;
const ruleParams = completeRule.ruleParams;
const inputIndex = await getInputIndex({
experimentalFeatures,
services,
@ -59,32 +58,32 @@ export const threatMatchExecutor = async ({
index: ruleParams.index,
});
return createThreatSignals({
tuple,
threatMapping: ruleParams.threatMapping,
query: ruleParams.query,
inputIndex,
type: ruleParams.type,
filters: ruleParams.filters ?? [],
language: ruleParams.language,
savedId: ruleParams.savedId,
services,
alertId: completeRule.alertId,
buildRuleMessage,
bulkCreate,
completeRule,
concurrentSearches: ruleParams.concurrentSearches ?? 1,
eventsTelemetry,
exceptionItems,
filters: ruleParams.filters ?? [],
inputIndex,
itemsPerSearch: ruleParams.itemsPerSearch ?? 9000,
language: ruleParams.language,
listClient,
logger,
eventsTelemetry,
alertId: rule.id,
outputIndex: ruleParams.outputIndex,
ruleSO: rule,
query: ruleParams.query,
savedId: ruleParams.savedId,
searchAfterSize,
services,
threatFilters: ruleParams.threatFilters ?? [],
threatQuery: ruleParams.threatQuery,
threatLanguage: ruleParams.threatLanguage,
buildRuleMessage,
threatIndex: ruleParams.threatIndex,
threatIndicatorPath: ruleParams.threatIndicatorPath,
concurrentSearches: ruleParams.concurrentSearches ?? 1,
itemsPerSearch: ruleParams.itemsPerSearch ?? 9000,
bulkCreate,
threatLanguage: ruleParams.threatLanguage,
threatMapping: ruleParams.threatMapping,
threatQuery: ruleParams.threatQuery,
tuple,
type: ruleParams.type,
wrapHits,
});
};

38
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts
View file

@ -13,48 +13,30 @@ import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server
import { thresholdExecutor } from './threshold';
import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock';
import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock';
import { getThresholdRuleParams, getCompleteRuleMock } from '../../schemas/rule_schemas.mock';
import { buildRuleMessageFactory } from '../rule_messages';
import { sampleEmptyDocSearchResults } from '../__mocks__/es_results';
import { allowedExperimentalValues } from '../../../../../common/experimental_features';
import { ThresholdRuleParams } from '../../schemas/rule_schemas';
describe('threshold_executor', () => {
const version = '8.0.0';
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
let alertServices: AlertServicesMock;
const params = getThresholdRuleParams();
const thresholdSO = {
id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
type: 'alert',
version: '1',
updated_at: '2020-03-27T22:55:59.577Z',
attributes: {
actions: [],
alertTypeId: 'siem.signals',
enabled: true,
name: 'rule-name',
tags: ['some fake tag 1', 'some fake tag 2'],
createdBy: 'sample user',
createdAt: '2020-03-27T22:55:59.577Z',
updatedBy: 'sample user',
schedule: {
interval: '5m',
},
throttle: 'no_actions',
params,
},
references: [],
};
const thresholdCompleteRule = getCompleteRuleMock<ThresholdRuleParams>(params);
const tuple = {
from: dateMath.parse(params.from)!,
to: dateMath.parse(params.to)!,
maxSignals: params.maxSignals,
};
const buildRuleMessage = buildRuleMessageFactory({
id: thresholdSO.id,
ruleId: thresholdSO.attributes.params.ruleId,
name: thresholdSO.attributes.name,
index: thresholdSO.attributes.params.outputIndex,
id: thresholdCompleteRule.alertId,
ruleId: thresholdCompleteRule.ruleParams.ruleId,
name: thresholdCompleteRule.ruleConfig.name,
index: thresholdCompleteRule.ruleParams.outputIndex,
});
beforeEach(() => {
@ -69,7 +51,7 @@ describe('threshold_executor', () => {
it('should set a warning when exception list for threshold rule contains value list exceptions', async () => {
const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })];
const response = await thresholdExecutor({
rule: thresholdSO,
completeRule: thresholdCompleteRule,
tuple,
exceptionItems,
experimentalFeatures: allowedExperimentalValues,

12
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts
View file

@ -9,7 +9,6 @@ import { SearchHit } from '@elastic/elasticsearch/api/types';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import { Logger } from 'src/core/server';
import { SavedObject } from 'src/core/types';
import {
AlertInstanceContext,
@ -17,7 +16,7 @@ import {
AlertServices,
} from '../../../../../../alerting/server';
import { hasLargeValueItem } from '../../../../../common/detection_engine/utils';
import { ThresholdRuleParams } from '../../schemas/rule_schemas';
import { CompleteRule, ThresholdRuleParams } from '../../schemas/rule_schemas';
import { getFilter } from '../get_filter';
import { getInputIndex } from '../get_input_output_index';
import {
@ -27,7 +26,6 @@ import {
getThresholdSignalHistory,
} from '../threshold';
import {
AlertAttributes,
BulkCreate,
RuleRangeTuple,
SearchAfterAndBulkCreateReturnType,
@ -44,7 +42,7 @@ import { ExperimentalFeatures } from '../../../../../common/experimental_feature
import { buildThresholdSignalHistory } from '../threshold/build_signal_history';
export const thresholdExecutor = async ({
rule,
completeRule,
tuple,
exceptionItems,
experimentalFeatures,
@ -57,7 +55,7 @@ export const thresholdExecutor = async ({
bulkCreate,
wrapHits,
}: {
rule: SavedObject<AlertAttributes<ThresholdRuleParams>>;
completeRule: CompleteRule<ThresholdRuleParams>;
tuple: RuleRangeTuple;
exceptionItems: ExceptionListItemSchema[];
experimentalFeatures: ExperimentalFeatures;
@ -71,7 +69,7 @@ export const thresholdExecutor = async ({
wrapHits: WrapHits;
}): Promise<SearchAfterAndBulkCreateReturnType & { state: ThresholdAlertState }> => {
let result = createSearchAfterReturnType();
const ruleParams = rule.attributes.params;
const ruleParams = completeRule.ruleParams;
// Get state or build initial state (on upgrade)
const { signalHistory, searchErrors: previousSearchErrors } = state.initialized
@ -150,7 +148,7 @@ export const thresholdExecutor = async ({
const { success, bulkCreateDuration, createdItemsCount, createdItems, errors } =
await bulkCreateThresholdSignals({
someResult: thresholdResults,
ruleSO: rule,
completeRule,
filter: esFilter,
services,
logger,

51
x-pack/plugins/security_solution/server/lib/detection_engine/signals/preview/alert_instance_factory_stub.ts Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { RuleParams } from '../../schemas/rule_schemas';
import {
AlertInstanceContext,
AlertInstanceState,
AlertTypeState,
} from '../../../../../../alerting/common';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { AlertInstance } from '../../../../../../alerting/server/alert_instance';
export const alertInstanceFactoryStub = <
TParams extends RuleParams,
TState extends AlertTypeState,
TInstanceState extends AlertInstanceState,
TInstanceContext extends AlertInstanceContext,
TActionGroupIds extends string = ''
>(
id: string
) => ({
getState() {
return {} as unknown as TInstanceState;
},
replaceState(state: TInstanceState) {
return new AlertInstance<TInstanceState, TInstanceContext, TActionGroupIds>({
state: {} as TInstanceState,
meta: { lastScheduledActions: { group: 'default', date: new Date() } },
});
},
scheduleActions(actionGroup: TActionGroupIds, alertcontext: TInstanceContext) {
return new AlertInstance<TInstanceState, TInstanceContext, TActionGroupIds>({
state: {} as TInstanceState,
meta: { lastScheduledActions: { group: 'default', date: new Date() } },
});
},
scheduleActionsWithSubGroup(
actionGroup: TActionGroupIds,
subgroup: string,
alertcontext: TInstanceContext
) {
return new AlertInstance<TInstanceState, TInstanceContext, TActionGroupIds>({
state: {} as TInstanceState,
meta: { lastScheduledActions: { group: 'default', date: new Date() } },
});
},
});

48
x-pack/plugins/security_solution/server/lib/detection_engine/signals/preview/preview_rule_execution_log_client.ts Normal file
View file

@ -0,0 +1,48 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectsFindResult } from 'kibana/server';
import {
LogExecutionMetricsArgs,
IRuleExecutionLogClient,
FindBulkExecutionLogArgs,
FindBulkExecutionLogResponse,
FindExecutionLogArgs,
LogStatusChangeArgs,
UpdateExecutionLogArgs,
} from '../../rule_execution_log';
import { IRuleStatusSOAttributes } from '../../rules/types';
export const createWarningsAndErrors = () => {
const warningsAndErrorsStore: LogStatusChangeArgs[] = [];
const previewRuleExecutionLogClient: IRuleExecutionLogClient = {
async delete(id: string): Promise<void> {
return Promise.resolve(undefined);
},
async find(
args: FindExecutionLogArgs
): Promise<Array<SavedObjectsFindResult<IRuleStatusSOAttributes>>> {
return Promise.resolve([]);
},
async findBulk(args: FindBulkExecutionLogArgs): Promise<FindBulkExecutionLogResponse> {
return Promise.resolve({});
},
async logStatusChange(args: LogStatusChangeArgs): Promise<void> {
warningsAndErrorsStore.push(args);
return Promise.resolve(undefined);
},
async update(args: UpdateExecutionLogArgs): Promise<void> {
return Promise.resolve(undefined);
},
async logExecutionMetrics(args: LogExecutionMetricsArgs): Promise<void> {
return Promise.resolve(undefined);
},
};
return { previewRuleExecutionLogClient, warningsAndErrorsStore };
};

35
x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts
View file

@ -8,7 +8,6 @@
import {
sampleEmptyDocSearchResults,
sampleRuleGuid,
sampleRuleSO,
mockLogger,
repeatedSearchResultsWithSortId,
repeatedSearchResultsWithNoSortId,
@ -27,12 +26,13 @@ import { getSearchListItemResponseMock } from '../../../../../lists/common/schem
import { getRuleRangeTuples } from './utils';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks';
import { getQueryRuleParams } from '../schemas/rule_schemas.mock';
import { getCompleteRuleMock, getQueryRuleParams } from '../schemas/rule_schemas.mock';
import { bulkCreateFactory } from './bulk_create_factory';
import { wrapHitsFactory } from './wrap_hits_factory';
import { mockBuildRuleMessage } from './__mocks__/build_rule_message.mock';
import { ResponseError } from '@elastic/elasticsearch/lib/errors';
import { BuildReasonMessage } from './reason_formatters';
import { QueryRuleParams } from '../schemas/rule_schemas';
const buildRuleMessage = mockBuildRuleMessage;
@ -45,7 +45,7 @@ describe('searchAfterAndBulkCreate', () => {
let listClient = listMock.getListClient();
const someGuids = Array.from({ length: 13 }).map(() => uuid.v4());
const sampleParams = getQueryRuleParams();
const ruleSO = sampleRuleSO(getQueryRuleParams());
const queryCompleteRule = getCompleteRuleMock<QueryRuleParams>(sampleParams);
sampleParams.maxSignals = 30;
let tuple: RuleRangeTuple;
beforeEach(() => {
@ -58,6 +58,7 @@ describe('searchAfterAndBulkCreate', () => {
tuple = getRuleRangeTuples({
logger: mockLogger,
previousStartedAt: new Date(),
startedAt: new Date(),
from: sampleParams.from,
to: sampleParams.to,
interval: '5m',
@ -71,7 +72,7 @@ describe('searchAfterAndBulkCreate', () => {
false
);
wrapHits = wrapHitsFactory({
ruleSO,
completeRule: queryCompleteRule,
signalsIndex: DEFAULT_SIGNALS_INDEX,
mergeStrategy: 'missingFields',
ignoreFields: [],
@ -184,7 +185,7 @@ describe('searchAfterAndBulkCreate', () => {
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
tuple,
ruleSO,
completeRule: queryCompleteRule,
listClient,
exceptionsList: [exceptionItem],
services: mockService,
@ -288,7 +289,7 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleSO,
completeRule: queryCompleteRule,
tuple,
listClient,
exceptionsList: [exceptionItem],
@ -367,7 +368,7 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleSO,
completeRule: queryCompleteRule,
tuple,
listClient,
exceptionsList: [exceptionItem],
@ -427,7 +428,7 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleSO,
completeRule: queryCompleteRule,
tuple,
listClient,
exceptionsList: [exceptionItem],
@ -507,7 +508,7 @@ describe('searchAfterAndBulkCreate', () => {
);
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleSO,
completeRule: queryCompleteRule,
tuple,
listClient,
exceptionsList: [],
@ -563,7 +564,7 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleSO,
completeRule: queryCompleteRule,
tuple,
listClient,
exceptionsList: [exceptionItem],
@ -636,7 +637,7 @@ describe('searchAfterAndBulkCreate', () => {
},
];
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleSO,
completeRule: queryCompleteRule,
tuple,
listClient,
exceptionsList: [exceptionItem],
@ -711,7 +712,7 @@ describe('searchAfterAndBulkCreate', () => {
)
);
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleSO,
completeRule: queryCompleteRule,
tuple,
listClient,
exceptionsList: [],
@ -766,7 +767,7 @@ describe('searchAfterAndBulkCreate', () => {
listClient,
exceptionsList: [exceptionItem],
tuple,
ruleSO,
completeRule: queryCompleteRule,
services: mockService,
logger: mockLogger,
eventsTelemetry: undefined,
@ -814,7 +815,7 @@ describe('searchAfterAndBulkCreate', () => {
listClient,
exceptionsList: [exceptionItem],
tuple,
ruleSO,
completeRule: queryCompleteRule,
services: mockService,
logger: mockLogger,
eventsTelemetry: undefined,
@ -876,7 +877,7 @@ describe('searchAfterAndBulkCreate', () => {
listClient,
exceptionsList: [exceptionItem],
tuple,
ruleSO,
completeRule: queryCompleteRule,
services: mockService,
logger: mockLogger,
eventsTelemetry: undefined,
@ -996,7 +997,7 @@ describe('searchAfterAndBulkCreate', () => {
);
const { success, createdSignalsCount, lastLookBackDate, errors } =
await searchAfterAndBulkCreate({
ruleSO,
completeRule: queryCompleteRule,
tuple,
listClient,
exceptionsList: [],
@ -1093,7 +1094,7 @@ describe('searchAfterAndBulkCreate', () => {
const mockEnrichment = jest.fn((a) => a);
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
enrichment: mockEnrichment,
ruleSO,
completeRule: queryCompleteRule,
tuple,
listClient,
exceptionsList: [],

24
x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts
View file

@ -23,25 +23,25 @@ import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } fr
// search_after through documents and re-index using bulk endpoint.
export const searchAfterAndBulkCreate = async ({
tuple,
ruleSO,
buildReasonMessage,
buildRuleMessage,
bulkCreate,
completeRule,
enrichment = identity,
eventsTelemetry,
exceptionsList,
services,
filter,
inputIndexPattern,
listClient,
logger,
eventsTelemetry,
inputIndexPattern,
filter,
pageSize,
buildRuleMessage,
buildReasonMessage,
enrichment = identity,
bulkCreate,
wrapHits,
services,
sortOrder,
trackTotalHits,
tuple,
wrapHits,
}: SearchAfterAndBulkCreateParams): Promise<SearchAfterAndBulkCreateReturnType> => {
const ruleParams = ruleSO.attributes.params;
const ruleParams = completeRule.ruleParams;
let toReturn = createSearchAfterReturnType();
// sortId tells us where to start our next consecutive search_after query

54
x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts
View file

@ -92,9 +92,9 @@ const getPayload = (
ruleTypeName: 'Name of rule',
enabled: true,
schedule: {
interval: '1h',
interval: '5m',
},
actions: [],
actions: ruleAlert.actions,
createdBy: 'elastic',
updatedBy: 'elastic',
createdAt: new Date('2019-12-13T16:50:33.400Z'),
@ -216,7 +216,7 @@ describe('signal_rule_alert_type', () => {
});
it('should warn about the gap between runs if gap is very large', async () => {
payload.previousStartedAt = moment().subtract(100, 'm').toDate();
payload.previousStartedAt = moment(payload.startedAt).subtract(100, 'm').toDate();
await alert.executor(payload);
expect(logger.warn).toHaveBeenCalled();
expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith(
@ -357,20 +357,16 @@ describe('signal_rule_alert_type', () => {
},
];
alertServices.savedObjectsClient.get.mockResolvedValue({
id: 'rule-id',
type: 'type',
references: [],
attributes: ruleAlert,
});
payload.params.meta = {};
const modifiedPayload = getPayload(
ruleAlert,
alertServices
) as jest.Mocked<RuleExecutorOptions>;
await alert.executor(payload);
await alert.executor(modifiedPayload);
expect(scheduleNotificationActions).toHaveBeenCalledWith(
expect.objectContaining({
resultsLink:
'/app/security/detections/rules/id/rule-id?timerange=(global:(linkTo:!(timeline),timerange:(from:100,kind:absolute,to:100)),timeline:(linkTo:!(global),timerange:(from:100,kind:absolute,to:100)))',
resultsLink: `/app/security/detections/rules/id/${ruleAlert.id}?timerange=(global:(linkTo:!(timeline),timerange:(from:100,kind:absolute,to:100)),timeline:(linkTo:!(global),timerange:(from:100,kind:absolute,to:100)))`,
})
);
});
@ -390,20 +386,16 @@ describe('signal_rule_alert_type', () => {
},
];
alertServices.savedObjectsClient.get.mockResolvedValue({
id: 'rule-id',
type: 'type',
references: [],
attributes: ruleAlert,
});
delete payload.params.meta;
const modifiedPayload = getPayload(
ruleAlert,
alertServices
) as jest.Mocked<RuleExecutorOptions>;
await alert.executor(payload);
await alert.executor(modifiedPayload);
expect(scheduleNotificationActions).toHaveBeenCalledWith(
expect.objectContaining({
resultsLink:
'/app/security/detections/rules/id/rule-id?timerange=(global:(linkTo:!(timeline),timerange:(from:100,kind:absolute,to:100)),timeline:(linkTo:!(global),timerange:(from:100,kind:absolute,to:100)))',
resultsLink: `/app/security/detections/rules/id/${ruleAlert.id}?timerange=(global:(linkTo:!(timeline),timerange:(from:100,kind:absolute,to:100)),timeline:(linkTo:!(global),timerange:(from:100,kind:absolute,to:100)))`,
})
);
});
@ -423,20 +415,16 @@ describe('signal_rule_alert_type', () => {
},
];
alertServices.savedObjectsClient.get.mockResolvedValue({
id: 'rule-id',
type: 'type',
references: [],
attributes: ruleAlert,
});
payload.params.meta = { kibana_siem_app_url: 'http://localhost' };
const modifiedPayload = getPayload(
ruleAlert,
alertServices
) as jest.Mocked<RuleExecutorOptions>;
await alert.executor(payload);
await alert.executor(modifiedPayload);
expect(scheduleNotificationActions).toHaveBeenCalledWith(
expect.objectContaining({
resultsLink:
'http://localhost/detections/rules/id/rule-id?timerange=(global:(linkTo:!(timeline),timerange:(from:100,kind:absolute,to:100)),timeline:(linkTo:!(global),timerange:(from:100,kind:absolute,to:100)))',
resultsLink: `http://localhost/detections/rules/id/${ruleAlert.id}?timerange=(global:(linkTo:!(timeline),timerange:(from:100,kind:absolute,to:100)),timeline:(linkTo:!(global),timerange:(from:100,kind:absolute,to:100)))`,
})
);
});

134
x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
View file

@ -6,7 +6,7 @@
*/
/* eslint-disable complexity */
import { Logger, SavedObject } from 'src/core/server';
import { Logger } from 'src/core/server';
import isEmpty from 'lodash/isEmpty';
import * as t from 'io-ts';
@ -26,7 +26,7 @@ import {
} from '../../../../common/detection_engine/utils';
import { SetupPlugins } from '../../../plugin';
import { getInputIndex } from './get_input_output_index';
import { AlertAttributes, SignalRuleAlertTypeDefinition, ThresholdAlertState } from './types';
import { SignalRuleAlertTypeDefinition, ThresholdAlertState } from './types';
import {
getListsClient,
getExceptions,
@ -59,6 +59,7 @@ import {
ruleParams,
RuleParams,
savedQueryRuleParams,
CompleteRule,
} from '../schemas/rule_schemas';
import { bulkCreateFactory } from './bulk_create_factory';
import { wrapHitsFactory } from './wrap_hits_factory';
@ -66,7 +67,11 @@ import { wrapSequencesFactory } from './wrap_sequences_factory';
import { ConfigType } from '../../../config';
import { ExperimentalFeatures } from '../../../../common/experimental_features';
import { injectReferences, extractReferences } from './saved_object_references';
import { RuleExecutionLogClient, truncateMessageList } from '../rule_execution_log';
import {
IRuleExecutionLogClient,
RuleExecutionLogClient,
truncateMessageList,
} from '../rule_execution_log';
import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas';
import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions';
import { IEventLogService } from '../../../../../event_log/server';
@ -80,15 +85,19 @@ export const signalRulesAlertType = ({
lists,
config,
eventLogService,
indexNameOverride,
ruleExecutionLogClientOverride,
}: {
logger: Logger;
eventsTelemetry: TelemetryEventsSender | undefined;
experimentalFeatures: ExperimentalFeatures;
version: string;
ml: SetupPlugins['ml'];
ml: SetupPlugins['ml'] | undefined;
lists: SetupPlugins['lists'] | undefined;
config: ConfigType;
eventLogService: IEventLogService;
indexNameOverride?: string;
ruleExecutionLogClientOverride?: IRuleExecutionLogClient;
}): SignalRuleAlertTypeDefinition => {
const { alertMergeStrategy: mergeStrategy, alertIgnoreFields: ignoreFields } = config;
return {
@ -127,31 +136,40 @@ export const signalRulesAlertType = ({
params,
spaceId,
updatedBy: updatedByUser,
rule,
}) {
const { ruleId, maxSignals, meta, outputIndex, timestampOverride, type } = params;
const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE);
let hasError: boolean = false;
let result = createSearchAfterReturnType();
const ruleStatusClient = new RuleExecutionLogClient({
eventLogService,
savedObjectsClient: services.savedObjectsClient,
underlyingClient: config.ruleExecutionLog.underlyingClient,
});
const ruleStatusClient = ruleExecutionLogClientOverride
? ruleExecutionLogClientOverride
: new RuleExecutionLogClient({
eventLogService,
savedObjectsClient: services.savedObjectsClient,
underlyingClient: config.ruleExecutionLog.underlyingClient,
});
const completeRule: CompleteRule<RuleParams> = {
alertId,
ruleConfig: rule,
ruleParams: params,
};
const savedObject = await services.savedObjectsClient.get<AlertAttributes>('alert', alertId);
const {
actions,
name,
alertTypeId,
schedule: { interval },
} = savedObject.attributes;
ruleTypeId,
} = completeRule.ruleConfig;
const refresh = actions.length ? 'wait_for' : false;
const buildRuleMessage = buildRuleMessageFactory({
id: alertId,
ruleId,
name,
index: outputIndex,
index: indexNameOverride ?? outputIndex,
});
logger.debug(buildRuleMessage('[+] Starting Signal Rule execution'));
@ -161,7 +179,7 @@ export const signalRulesAlertType = ({
spaceId,
ruleId: alertId,
ruleName: name,
ruleType: alertTypeId,
ruleType: ruleTypeId,
};
await ruleStatusClient.logStatusChange({
@ -172,7 +190,7 @@ export const signalRulesAlertType = ({
const notificationRuleParams: NotificationRuleTypeParams = {
...params,
name,
id: savedObject.id,
id: alertId,
};
// check if rule has permissions to access given index pattern
@ -235,7 +253,9 @@ export const signalRulesAlertType = ({
interval,
maxSignals,
buildRuleMessage,
startedAt,
});
if (remainingGap.asMilliseconds() > 0) {
const gapString = remainingGap.humanize();
const gapMessage = buildRuleMessage(
@ -268,28 +288,32 @@ export const signalRulesAlertType = ({
logger,
services.scopedClusterClient.asCurrentUser,
buildRuleMessage,
refresh
refresh,
indexNameOverride
);
const wrapHits = wrapHitsFactory({
ruleSO: savedObject,
signalsIndex: params.outputIndex,
completeRule,
signalsIndex: indexNameOverride ?? params.outputIndex,
mergeStrategy,
ignoreFields,
});
const wrapSequences = wrapSequencesFactory({
ruleSO: savedObject,
completeRule,
signalsIndex: params.outputIndex,
mergeStrategy,
ignoreFields,
});
if (isMlRule(type)) {
const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams);
const mlRuleCompleteRule = asTypeSpecificCompleteRule(
completeRule,
machineLearningRuleParams
);
for (const tuple of tuples) {
result = await mlExecutor({
rule: mlRuleSO,
completeRule: mlRuleCompleteRule,
tuple,
ml,
listClient,
@ -302,10 +326,13 @@ export const signalRulesAlertType = ({
});
}
} else if (isThresholdRule(type)) {
const thresholdRuleSO = asTypeSpecificSO(savedObject, thresholdRuleParams);
const thresholdCompleteRule = asTypeSpecificCompleteRule(
completeRule,
thresholdRuleParams
);
for (const tuple of tuples) {
result = await thresholdExecutor({
rule: thresholdRuleSO,
completeRule: thresholdCompleteRule,
tuple,
exceptionItems,
experimentalFeatures,
@ -320,10 +347,10 @@ export const signalRulesAlertType = ({
});
}
} else if (isThreatMatchRule(type)) {
const threatRuleSO = asTypeSpecificSO(savedObject, threatRuleParams);
const threatCompleteRule = asTypeSpecificCompleteRule(completeRule, threatRuleParams);
for (const tuple of tuples) {
result = await threatMatchExecutor({
rule: threatRuleSO,
completeRule: threatCompleteRule,
tuple,
listClient,
exceptionItems,
@ -339,10 +366,10 @@ export const signalRulesAlertType = ({
});
}
} else if (isQueryRule(type)) {
const queryRuleSO = validateQueryRuleTypes(savedObject);
const queryCompleteRule = validateQueryRuleTypes(completeRule);
for (const tuple of tuples) {
result = await queryExecutor({
rule: queryRuleSO,
completeRule: queryCompleteRule,
tuple,
listClient,
exceptionItems,
@ -358,10 +385,10 @@ export const signalRulesAlertType = ({
});
}
} else if (isEqlRule(type)) {
const eqlRuleSO = asTypeSpecificSO(savedObject, eqlRuleParams);
const eqlCompleteRule = asTypeSpecificCompleteRule(completeRule, eqlRuleParams);
for (const tuple of tuples) {
result = await eqlExecutor({
rule: eqlRuleSO,
completeRule: eqlCompleteRule,
tuple,
exceptionItems,
experimentalFeatures,
@ -395,7 +422,7 @@ export const signalRulesAlertType = ({
const resultsLink = getNotificationResultsLink({
from: fromInMs,
to: toInMs,
id: savedObject.id,
id: alertId,
kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined)
?.kibana_siem_app_url,
});
@ -404,15 +431,15 @@ export const signalRulesAlertType = ({
buildRuleMessage(`Found ${result.createdSignalsCount} signals for notification.`)
);
if (savedObject.attributes.throttle != null) {
if (completeRule.ruleConfig.throttle != null) {
await scheduleThrottledNotificationActions({
alertInstance: services.alertInstanceFactory(alertId),
throttle: savedObject.attributes.throttle,
throttle: completeRule.ruleConfig.throttle,
startedAt,
id: savedObject.id,
id: alertId,
kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined)
?.kibana_siem_app_url,
outputIndex,
outputIndex: indexNameOverride ?? outputIndex,
ruleId,
signals: result.createdSignals,
esClient: services.scopedClusterClient.asCurrentUser,
@ -434,7 +461,9 @@ export const signalRulesAlertType = ({
logger.debug(buildRuleMessage('[+] Signal Rule execution completed.'));
logger.debug(
buildRuleMessage(
`[+] Finished indexing ${result.createdSignalsCount} signals into ${outputIndex}`
`[+] Finished indexing ${result.createdSignalsCount} signals into ${
indexNameOverride ?? outputIndex
}`
)
);
if (!hasError && !wroteWarningStatus && !result.warning) {
@ -462,12 +491,12 @@ export const signalRulesAlertType = ({
);
} else {
// NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early
if (savedObject.attributes.throttle != null) {
if (completeRule.ruleConfig.throttle != null) {
await scheduleThrottledNotificationActions({
alertInstance: services.alertInstanceFactory(alertId),
throttle: savedObject.attributes.throttle,
throttle: completeRule.ruleConfig.throttle ?? '',
startedAt,
id: savedObject.id,
id: completeRule.alertId,
kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined)
?.kibana_siem_app_url,
outputIndex,
@ -496,12 +525,12 @@ export const signalRulesAlertType = ({
}
} catch (error) {
// NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early
if (savedObject.attributes.throttle != null) {
if (completeRule.ruleConfig.throttle != null) {
await scheduleThrottledNotificationActions({
alertInstance: services.alertInstanceFactory(alertId),
throttle: savedObject.attributes.throttle,
throttle: completeRule.ruleConfig.throttle ?? '',
startedAt,
id: savedObject.id,
id: completeRule.alertId,
kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined)
?.kibana_siem_app_url,
outputIndex,
@ -534,11 +563,11 @@ export const signalRulesAlertType = ({
};
};
const validateQueryRuleTypes = (ruleSO: SavedObject<AlertAttributes>) => {
if (ruleSO.attributes.params.type === 'query') {
return asTypeSpecificSO(ruleSO, queryRuleParams);
const validateQueryRuleTypes = (completeRule: CompleteRule<RuleParams>) => {
if (completeRule.ruleParams.type === 'query') {
return asTypeSpecificCompleteRule(completeRule, queryRuleParams);
} else {
return asTypeSpecificSO(ruleSO, savedQueryRuleParams);
return asTypeSpecificCompleteRule(completeRule, savedQueryRuleParams);
}
};
@ -549,22 +578,19 @@ const validateQueryRuleTypes = (ruleSO: SavedObject<AlertAttributes>) => {
* checks if the required type specific fields actually exist on the SO and prevents rule executors from
* accessing fields that only exist on other rule types.
*
* @param ruleSO SavedObject typed as an object with all fields from all different rule types
* @param completeRule rule typed as an object with all fields from all different rule types
* @param schema io-ts schema for the specific rule type the SavedObject claims to be
*/
export const asTypeSpecificSO = <T extends t.Mixed>(
ruleSO: SavedObject<AlertAttributes>,
export const asTypeSpecificCompleteRule = <T extends t.Mixed>(
completeRule: CompleteRule<RuleParams>,
schema: T
) => {
const [validated, errors] = validateNonExact(ruleSO.attributes.params, schema);
const [validated, errors] = validateNonExact(completeRule.ruleParams, schema);
if (validated == null || errors != null) {
throw new Error(`Rule attempted to execute with invalid params: ${errors}`);
}
return {
...ruleSO,
attributes: {
...ruleSO.attributes,
params: validated,
},
...completeRule,
ruleParams: validated,
};
};

60
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts
View file

@ -14,28 +14,28 @@ import { CreateThreatSignalOptions } from './types';
import { SearchAfterAndBulkCreateReturnType } from '../types';
export const createThreatSignal = async ({
tuple,
threatMapping,
threatEnrichment,
query,
inputIndex,
type,
filters,
language,
savedId,
services,
alertId,
buildRuleMessage,
bulkCreate,
completeRule,
currentResult,
currentThreatList,
eventsTelemetry,
exceptionItems,
filters,
inputIndex,
language,
listClient,
logger,
eventsTelemetry,
alertId,
outputIndex,
ruleSO,
query,
savedId,
searchAfterSize,
buildRuleMessage,
currentThreatList,
currentResult,
bulkCreate,
services,
threatEnrichment,
threatMapping,
tuple,
type,
wrapHits,
}: CreateThreatSignalOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
const threatFilter = buildThreatMappingFilter({
@ -71,25 +71,25 @@ export const createThreatSignal = async ({
);
const result = await searchAfterAndBulkCreate({
tuple,
listClient,
exceptionsList: exceptionItems,
ruleSO,
services,
logger,
buildReasonMessage: buildReasonMessageForThreatMatchAlert,
buildRuleMessage,
bulkCreate,
completeRule,
enrichment: threatEnrichment,
eventsTelemetry,
exceptionsList: exceptionItems,
filter: esFilter,
id: alertId,
inputIndexPattern: inputIndex,
signalsIndex: outputIndex,
filter: esFilter,
listClient,
logger,
pageSize: searchAfterSize,
buildRuleMessage,
buildReasonMessage: buildReasonMessageForThreatMatchAlert,
enrichment: threatEnrichment,
bulkCreate,
wrapHits,
services,
signalsIndex: outputIndex,
sortOrder: 'desc',
trackTotalHits: false,
tuple,
wrapHits,
});
logger.debug(

74
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts
View file

@ -15,39 +15,39 @@ import { buildExecutionIntervalValidator, combineConcurrentResults } from './uti
import { buildThreatEnrichment } from './build_threat_enrichment';
export const createThreatSignals = async ({
tuple,
threatMapping,
query,
inputIndex,
type,
filters,
language,
savedId,
services,
alertId,
buildRuleMessage,
bulkCreate,
completeRule,
concurrentSearches,
eventsTelemetry,
exceptionItems,
filters,
inputIndex,
itemsPerSearch,
language,
listClient,
logger,
eventsTelemetry,
alertId,
outputIndex,
ruleSO,
query,
savedId,
searchAfterSize,
services,
threatFilters,
threatQuery,
threatLanguage,
buildRuleMessage,
threatIndex,
threatIndicatorPath,
concurrentSearches,
itemsPerSearch,
bulkCreate,
threatLanguage,
threatMapping,
threatQuery,
tuple,
type,
wrapHits,
}: CreateThreatSignalsOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
const params = ruleSO.attributes.params;
const params = completeRule.ruleParams;
logger.debug(buildRuleMessage('Indicator matching rule starting'));
const perPage = concurrentSearches * itemsPerSearch;
const verifyExecutionCanProceed = buildExecutionIntervalValidator(
ruleSO.attributes.schedule.interval
completeRule.ruleConfig.schedule.interval
);
let results: SearchAfterAndBulkCreateReturnType = {
@ -108,28 +108,28 @@ export const createThreatSignals = async ({
const concurrentSearchesPerformed = chunks.map<Promise<SearchAfterAndBulkCreateReturnType>>(
(slicedChunk) =>
createThreatSignal({
tuple,
threatEnrichment,
threatMapping,
query,
inputIndex,
type,
filters,
language,
savedId,
services,
alertId,
buildRuleMessage,
bulkCreate,
completeRule,
currentResult: results,
currentThreatList: slicedChunk,
eventsTelemetry,
exceptionItems,
filters,
inputIndex,
language,
listClient,
logger,
eventsTelemetry,
alertId,
outputIndex,
ruleSO,
query,
savedId,
searchAfterSize,
buildRuleMessage,
currentThreatList: slicedChunk,
currentResult: results,
bulkCreate,
services,
threatEnrichment,
threatMapping,
tuple,
type,
wrapHits,
})
);

113
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts
View file

@ -24,107 +24,106 @@ import {
AlertInstanceState,
AlertServices,
} from '../../../../../../alerting/server';
import { ElasticsearchClient, Logger, SavedObject } from '../../../../../../../../src/core/server';
import { ElasticsearchClient, Logger } from '../../../../../../../../src/core/server';
import { TelemetryEventsSender } from '../../../telemetry/sender';
import { BuildRuleMessage } from '../rule_messages';
import {
AlertAttributes,
BulkCreate,
RuleRangeTuple,
SearchAfterAndBulkCreateReturnType,
SignalsEnrichment,
WrapHits,
} from '../types';
import { ThreatRuleParams } from '../../schemas/rule_schemas';
import { CompleteRule, ThreatRuleParams } from '../../schemas/rule_schemas';
export type SortOrderOrUndefined = 'asc' | 'desc' | undefined;
export interface CreateThreatSignalsOptions {
tuple: RuleRangeTuple;
threatMapping: ThreatMapping;
query: string;
inputIndex: string[];
type: Type;
filters: unknown[];
language: LanguageOrUndefined;
savedId: string | undefined;
services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
alertId: string;
buildRuleMessage: BuildRuleMessage;
bulkCreate: BulkCreate;
completeRule: CompleteRule<ThreatRuleParams>;
concurrentSearches: ConcurrentSearches;
eventsTelemetry: TelemetryEventsSender | undefined;
exceptionItems: ExceptionListItemSchema[];
filters: unknown[];
inputIndex: string[];
itemsPerSearch: ItemsPerSearch;
language: LanguageOrUndefined;
listClient: ListClient;
logger: Logger;
eventsTelemetry: TelemetryEventsSender | undefined;
alertId: string;
outputIndex: string;
ruleSO: SavedObject<AlertAttributes<ThreatRuleParams>>;
query: string;
savedId: string | undefined;
searchAfterSize: number;
services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
threatFilters: unknown[];
threatQuery: ThreatQuery;
buildRuleMessage: BuildRuleMessage;
threatIndex: ThreatIndex;
threatIndicatorPath: ThreatIndicatorPathOrUndefined;
threatLanguage: ThreatLanguageOrUndefined;
concurrentSearches: ConcurrentSearches;
itemsPerSearch: ItemsPerSearch;
bulkCreate: BulkCreate;
threatMapping: ThreatMapping;
threatQuery: ThreatQuery;
tuple: RuleRangeTuple;
type: Type;
wrapHits: WrapHits;
}
export interface CreateThreatSignalOptions {
tuple: RuleRangeTuple;
threatMapping: ThreatMapping;
threatEnrichment: SignalsEnrichment;
query: string;
inputIndex: string[];
type: Type;
filters: unknown[];
language: LanguageOrUndefined;
savedId: string | undefined;
services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
alertId: string;
buildRuleMessage: BuildRuleMessage;
bulkCreate: BulkCreate;
completeRule: CompleteRule<ThreatRuleParams>;
currentResult: SearchAfterAndBulkCreateReturnType;
currentThreatList: ThreatListItem[];
eventsTelemetry: TelemetryEventsSender | undefined;
exceptionItems: ExceptionListItemSchema[];
filters: unknown[];
inputIndex: string[];
language: LanguageOrUndefined;
listClient: ListClient;
logger: Logger;
eventsTelemetry: TelemetryEventsSender | undefined;
alertId: string;
outputIndex: string;
ruleSO: SavedObject<AlertAttributes<ThreatRuleParams>>;
query: string;
savedId: string | undefined;
searchAfterSize: number;
buildRuleMessage: BuildRuleMessage;
currentThreatList: ThreatListItem[];
currentResult: SearchAfterAndBulkCreateReturnType;
bulkCreate: BulkCreate;
services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
threatEnrichment: SignalsEnrichment;
threatMapping: ThreatMapping;
tuple: RuleRangeTuple;
type: Type;
wrapHits: WrapHits;
}
export interface BuildThreatMappingFilterOptions {
threatMapping: ThreatMapping;
threatList: ThreatListItem[];
chunkSize?: number;
threatList: ThreatListItem[];
threatMapping: ThreatMapping;
}
export interface FilterThreatMappingOptions {
threatMapping: ThreatMapping;
threatListItem: ThreatListItem;
threatMapping: ThreatMapping;
}
export interface CreateInnerAndClausesOptions {
threatMappingEntries: ThreatMappingEntries;
threatListItem: ThreatListItem;
threatMappingEntries: ThreatMappingEntries;
}
export interface CreateAndOrClausesOptions {
threatMapping: ThreatMapping;
threatListItem: ThreatListItem;
threatMapping: ThreatMapping;
}
export interface BuildEntriesMappingFilterOptions {
threatMapping: ThreatMapping;
threatList: ThreatListItem[];
chunkSize: number;
threatList: ThreatListItem[];
threatMapping: ThreatMapping;
}
export interface SplitShouldClausesOptions {
should: BooleanFilter[];
chunkSize: number;
should: BooleanFilter[];
}
export interface BooleanFilter {
@ -132,35 +131,35 @@ export interface BooleanFilter {
}
export interface GetThreatListOptions {
buildRuleMessage: BuildRuleMessage;
esClient: ElasticsearchClient;
query: string;
language: ThreatLanguageOrUndefined;
exceptionItems: ExceptionListItemSchema[];
index: string[];
language: ThreatLanguageOrUndefined;
listClient: ListClient;
logger: Logger;
perPage?: number;
query: string;
searchAfter: string[] | undefined;
sortField: string | undefined;
sortOrder: SortOrderOrUndefined;
threatFilters: unknown[];
exceptionItems: ExceptionListItemSchema[];
listClient: ListClient;
buildRuleMessage: BuildRuleMessage;
logger: Logger;
}
export interface ThreatListCountOptions {
esClient: ElasticsearchClient;
query: string;
language: ThreatLanguageOrUndefined;
threatFilters: unknown[];
index: string[];
exceptionItems: ExceptionListItemSchema[];
index: string[];
language: ThreatLanguageOrUndefined;
query: string;
threatFilters: unknown[];
}
export interface GetSortWithTieBreakerOptions {
sortField: string | undefined;
sortOrder: SortOrderOrUndefined;
index: string[];
listItemIndex: string;
sortField: string | undefined;
sortOrder: SortOrderOrUndefined;
}
export interface ThreatListDoc {

9
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts
View file

@ -13,7 +13,7 @@ import {
ThresholdNormalized,
TimestampOverrideOrUndefined,
} from '../../../../../common/detection_engine/schemas/common/schemas';
import { Logger, SavedObject } from '../../../../../../../../src/core/server';
import { Logger } from '../../../../../../../../src/core/server';
import {
AlertInstanceContext,
AlertInstanceState,
@ -33,15 +33,14 @@ import type {
SignalSource,
SignalSearchResponse,
ThresholdSignalHistory,
AlertAttributes,
BulkCreate,
WrapHits,
} from '../types';
import { ThresholdRuleParams } from '../../schemas/rule_schemas';
import { CompleteRule, ThresholdRuleParams } from '../../schemas/rule_schemas';
interface BulkCreateThresholdSignalsParams {
someResult: SignalSearchResponse;
ruleSO: SavedObject<AlertAttributes<ThresholdRuleParams>>;
completeRule: CompleteRule<ThresholdRuleParams>;
services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
inputIndexPattern: string[];
logger: Logger;
@ -228,7 +227,7 @@ export const transformThresholdResultsToEcs = (
export const bulkCreateThresholdSignals = async (
params: BulkCreateThresholdSignalsParams
): Promise<GenericBulkCreateResponse<{}>> => {
const ruleParams = params.ruleSO.attributes.params;
const ruleParams = params.completeRule.ruleParams;
const thresholdResults = params.someResult;
const ecsResults = transformThresholdResultsToEcs(
thresholdResults,

6
x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts
View file

@ -28,10 +28,10 @@ import {
EqlSequence,
} from '../../../../common/detection_engine/types';
import { ListClient } from '../../../../../lists/server';
import { Logger, SavedObject } from '../../../../../../../src/core/server';
import { Logger } from '../../../../../../../src/core/server';
import { BuildRuleMessage } from './rule_messages';
import { TelemetryEventsSender } from '../../telemetry/sender';
import { RuleParams } from '../schemas/rule_schemas';
import { CompleteRule, RuleParams } from '../schemas/rule_schemas';
import { GenericBulkCreateResponse } from './bulk_create_factory';
import { EcsFieldMap } from '../../../../../rule_registry/common/assets/field_maps/ecs_field_map';
import { TypeOfFieldMap } from '../../../../../rule_registry/common/field_map';
@ -300,7 +300,7 @@ export interface SearchAfterAndBulkCreateParams {
from: moment.Moment;
maxSignals: number;
};
ruleSO: SavedObject<AlertAttributes>;
completeRule: CompleteRule<RuleParams>;
services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
listClient: ListClient;
exceptionsList: ExceptionListItemSchema[];

184
x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts
View file

@ -22,7 +22,6 @@ moment.suppressDeprecationWarnings = true;
import {
generateId,
parseInterval,
getDriftTolerance,
getGapBetweenRuns,
getNumCatchupIntervals,
errorAggregator,
@ -112,105 +111,13 @@ describe('utils', () => {
});
});
describe('getDriftTolerance', () => {
test('it returns a drift tolerance in milliseconds of 1 minute when "from" overlaps "to" by 1 minute and the interval is 5 minutes', () => {
const drift = getDriftTolerance({
from: 'now-6m',
to: 'now',
intervalDuration: moment.duration(5, 'minutes'),
});
expect(drift).not.toBeNull();
expect(drift?.asMilliseconds()).toEqual(moment.duration(1, 'minute').asMilliseconds());
});
test('it returns a drift tolerance of 0 when "from" equals the interval', () => {
const drift = getDriftTolerance({
from: 'now-5m',
to: 'now',
intervalDuration: moment.duration(5, 'minutes'),
});
expect(drift?.asMilliseconds()).toEqual(0);
});
test('it returns a drift tolerance of 5 minutes when "from" is 10 minutes but the interval is 5 minutes', () => {
const drift = getDriftTolerance({
from: 'now-10m',
to: 'now',
intervalDuration: moment.duration(5, 'minutes'),
});
expect(drift).not.toBeNull();
expect(drift?.asMilliseconds()).toEqual(moment.duration(5, 'minutes').asMilliseconds());
});
test('it returns a drift tolerance of 10 minutes when "from" is 10 minutes ago and the interval is 0', () => {
const drift = getDriftTolerance({
from: 'now-10m',
to: 'now',
intervalDuration: moment.duration(0, 'milliseconds'),
});
expect(drift).not.toBeNull();
expect(drift?.asMilliseconds()).toEqual(moment.duration(10, 'minutes').asMilliseconds());
});
test('returns a drift tolerance of 1 minute when "from" is invalid and defaults to "now-6m" and interval is 5 minutes', () => {
const drift = getDriftTolerance({
from: 'invalid',
to: 'now',
intervalDuration: moment.duration(5, 'minutes'),
});
expect(drift).not.toBeNull();
expect(drift?.asMilliseconds()).toEqual(moment.duration(1, 'minute').asMilliseconds());
});
test('returns a drift tolerance of 1 minute when "from" does not include `now` and defaults to "now-6m" and interval is 5 minutes', () => {
const drift = getDriftTolerance({
from: '10m',
to: 'now',
intervalDuration: moment.duration(5, 'minutes'),
});
expect(drift).not.toBeNull();
expect(drift?.asMilliseconds()).toEqual(moment.duration(1, 'minute').asMilliseconds());
});
test('returns a drift tolerance of 4 minutes when "to" is "now-x", from is a valid input and interval is 5 minute', () => {
const drift = getDriftTolerance({
from: 'now-10m',
to: 'now-1m',
intervalDuration: moment.duration(5, 'minutes'),
});
expect(drift).not.toBeNull();
expect(drift?.asMilliseconds()).toEqual(moment.duration(4, 'minutes').asMilliseconds());
});
test('it returns expected drift tolerance when "from" is an ISO string', () => {
const drift = getDriftTolerance({
from: moment().subtract(10, 'minutes').toISOString(),
to: 'now',
intervalDuration: moment.duration(5, 'minutes'),
});
expect(drift).not.toBeNull();
expect(drift?.asMilliseconds()).toEqual(moment.duration(5, 'minutes').asMilliseconds());
});
test('it returns expected drift tolerance when "to" is an ISO string', () => {
const drift = getDriftTolerance({
from: 'now-6m',
to: moment().toISOString(),
intervalDuration: moment.duration(5, 'minutes'),
});
expect(drift).not.toBeNull();
expect(drift?.asMilliseconds()).toEqual(moment.duration(1, 'minute').asMilliseconds());
});
});
describe('getGapBetweenRuns', () => {
test('it returns a gap of 0 when "from" and interval match each other and the previous started was from the previous interval time', () => {
const gap = getGapBetweenRuns({
previousStartedAt: nowDate.clone().subtract(5, 'minutes').toDate(),
intervalDuration: moment.duration(5, 'minutes'),
from: 'now-5m',
to: 'now',
now: nowDate.clone(),
startedAt: nowDate.clone().toDate(),
originalFrom: nowDate.clone().subtract(5, 'minutes'),
originalTo: nowDate.clone(),
});
expect(gap).not.toBeNull();
expect(gap?.asMilliseconds()).toEqual(0);
@ -219,10 +126,9 @@ describe('utils', () => {
test('it returns a negative gap of 1 minute when "from" overlaps to by 1 minute and the previousStartedAt was 5 minutes ago', () => {
const gap = getGapBetweenRuns({
previousStartedAt: nowDate.clone().subtract(5, 'minutes').toDate(),
intervalDuration: moment.duration(5, 'minutes'),
from: 'now-6m',
to: 'now',
now: nowDate.clone(),
startedAt: nowDate.clone().toDate(),
originalFrom: nowDate.clone().subtract(6, 'minutes'),
originalTo: nowDate.clone(),
});
expect(gap).not.toBeNull();
expect(gap?.asMilliseconds()).toEqual(moment.duration(-1, 'minute').asMilliseconds());
@ -231,10 +137,9 @@ describe('utils', () => {
test('it returns a negative gap of 5 minutes when "from" overlaps to by 1 minute and the previousStartedAt was 5 minutes ago', () => {
const gap = getGapBetweenRuns({
previousStartedAt: nowDate.clone().subtract(5, 'minutes').toDate(),
intervalDuration: moment.duration(5, 'minutes'),
from: 'now-10m',
to: 'now',
now: nowDate.clone(),
startedAt: nowDate.clone().toDate(),
originalFrom: nowDate.clone().subtract(10, 'minutes'),
originalTo: nowDate.clone(),
});
expect(gap).not.toBeNull();
expect(gap?.asMilliseconds()).toEqual(moment.duration(-5, 'minute').asMilliseconds());
@ -243,10 +148,9 @@ describe('utils', () => {
test('it returns a negative gap of 1 minute when "from" overlaps to by 1 minute and the previousStartedAt was 10 minutes ago and so was the interval', () => {
const gap = getGapBetweenRuns({
previousStartedAt: nowDate.clone().subtract(10, 'minutes').toDate(),
intervalDuration: moment.duration(10, 'minutes'),
from: 'now-11m',
to: 'now',
now: nowDate.clone(),
startedAt: nowDate.clone().toDate(),
originalFrom: nowDate.clone().subtract(11, 'minutes'),
originalTo: nowDate.clone(),
});
expect(gap).not.toBeNull();
expect(gap?.asMilliseconds()).toEqual(moment.duration(-1, 'minute').asMilliseconds());
@ -255,10 +159,9 @@ describe('utils', () => {
test('it returns a gap of only -30 seconds when the from overlaps with now by 1 minute, the interval is 5 minutes but the previous started is 30 seconds more', () => {
const gap = getGapBetweenRuns({
previousStartedAt: nowDate.clone().subtract(5, 'minutes').subtract(30, 'seconds').toDate(),
intervalDuration: moment.duration(5, 'minutes'),
from: 'now-6m',
to: 'now',
now: nowDate.clone(),
startedAt: nowDate.clone().toDate(),
originalFrom: nowDate.clone().subtract(6, 'minutes'),
originalTo: nowDate.clone(),
});
expect(gap).not.toBeNull();
expect(gap?.asMilliseconds()).toEqual(moment.duration(-30, 'seconds').asMilliseconds());
@ -267,10 +170,9 @@ describe('utils', () => {
test('it returns an exact 0 gap when the from overlaps with now by 1 minute, the interval is 5 minutes but the previous started is one minute late', () => {
const gap = getGapBetweenRuns({
previousStartedAt: nowDate.clone().subtract(6, 'minutes').toDate(),
intervalDuration: moment.duration(5, 'minutes'),
from: 'now-6m',
to: 'now',
now: nowDate.clone(),
startedAt: nowDate.clone().toDate(),
originalFrom: nowDate.clone().subtract(6, 'minutes'),
originalTo: nowDate.clone(),
});
expect(gap).not.toBeNull();
expect(gap?.asMilliseconds()).toEqual(moment.duration(0, 'minute').asMilliseconds());
@ -279,10 +181,9 @@ describe('utils', () => {
test('it returns a gap of 30 seconds when the from overlaps with now by 1 minute, the interval is 5 minutes but the previous started is one minute and 30 seconds late', () => {
const gap = getGapBetweenRuns({
previousStartedAt: nowDate.clone().subtract(6, 'minutes').subtract(30, 'seconds').toDate(),
intervalDuration: moment.duration(5, 'minutes'),
from: 'now-6m',
to: 'now',
now: nowDate.clone(),
startedAt: nowDate.clone().toDate(),
originalFrom: nowDate.clone().subtract(6, 'minutes'),
originalTo: nowDate.clone(),
});
expect(gap).not.toBeNull();
expect(gap?.asMilliseconds()).toEqual(moment.duration(30, 'seconds').asMilliseconds());
@ -291,10 +192,9 @@ describe('utils', () => {
test('it returns a gap of 1 minute when the from overlaps with now by 1 minute, the interval is 5 minutes but the previous started is two minutes late', () => {
const gap = getGapBetweenRuns({
previousStartedAt: nowDate.clone().subtract(7, 'minutes').toDate(),
intervalDuration: moment.duration(5, 'minutes'),
from: 'now-6m',
to: 'now',
now: nowDate.clone(),
startedAt: nowDate.clone().toDate(),
originalFrom: nowDate.clone().subtract(6, 'minutes'),
originalTo: nowDate.clone(),
});
expect(gap?.asMilliseconds()).not.toBeNull();
expect(gap?.asMilliseconds()).toEqual(moment.duration(1, 'minute').asMilliseconds());
@ -303,37 +203,12 @@ describe('utils', () => {
test('it returns 0 if given a previousStartedAt of null', () => {
const gap = getGapBetweenRuns({
previousStartedAt: null,
intervalDuration: moment.duration(5, 'minutes'),
from: 'now-5m',
to: 'now',
now: nowDate.clone(),
startedAt: nowDate.clone().toDate(),
originalFrom: nowDate.clone().subtract(5, 'minutes'),
originalTo: nowDate.clone(),
});
expect(gap.asMilliseconds()).toEqual(0);
});
test('it returns the expected result when "from" is an invalid string such as "invalid"', () => {
const gap = getGapBetweenRuns({
previousStartedAt: nowDate.clone().subtract(7, 'minutes').toDate(),
intervalDuration: moment.duration(5, 'minutes'),
from: 'invalid',
to: 'now',
now: nowDate.clone(),
});
expect(gap?.asMilliseconds()).not.toBeNull();
expect(gap?.asMilliseconds()).toEqual(moment.duration(1, 'minute').asMilliseconds());
});
test('it returns the expected result when "to" is an invalid string such as "invalid"', () => {
const gap = getGapBetweenRuns({
previousStartedAt: nowDate.clone().subtract(7, 'minutes').toDate(),
intervalDuration: moment.duration(5, 'minutes'),
from: 'now-6m',
to: 'invalid',
now: nowDate.clone(),
});
expect(gap?.asMilliseconds()).not.toBeNull();
expect(gap?.asMilliseconds()).toEqual(moment.duration(1, 'minute').asMilliseconds());
});
});
describe('errorAggregator', () => {
@ -572,6 +447,7 @@ describe('utils', () => {
const { tuples, remainingGap } = getRuleRangeTuples({
logger: mockLogger,
previousStartedAt: moment().subtract(30, 's').toDate(),
startedAt: moment().subtract(30, 's').toDate(),
interval: '30s',
from: 'now-30s',
to: 'now',
@ -588,6 +464,7 @@ describe('utils', () => {
const { tuples, remainingGap } = getRuleRangeTuples({
logger: mockLogger,
previousStartedAt: moment().subtract(30, 's').toDate(),
startedAt: moment().subtract(30, 's').toDate(),
interval: 'invalid',
from: 'now-30s',
to: 'now',
@ -604,6 +481,7 @@ describe('utils', () => {
const { tuples, remainingGap } = getRuleRangeTuples({
logger: mockLogger,
previousStartedAt: moment().subtract(65, 's').toDate(),
startedAt: moment().toDate(),
interval: '50s',
from: 'now-55s',
to: 'now',
@ -619,6 +497,7 @@ describe('utils', () => {
const { tuples, remainingGap } = getRuleRangeTuples({
logger: mockLogger,
previousStartedAt: moment().subtract(65, 's').toDate(), // 64 is 5 times the interval + lookback, which will trigger max lookback
startedAt: moment().toDate(),
interval: '10s',
from: 'now-13s',
to: 'now',
@ -641,6 +520,7 @@ describe('utils', () => {
const { tuples, remainingGap } = getRuleRangeTuples({
logger: mockLogger,
previousStartedAt: moment().subtract(-15, 's').toDate(),
startedAt: moment().subtract(-15, 's').toDate(),
interval: '10s',
from: 'now-13s',
to: 'now',

85
x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts
View file

@ -16,7 +16,6 @@ import { ALERT_INSTANCE_ID, ALERT_RULE_UUID } from '@kbn/rule-data-utils';
import type { ListArray, ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import { MAX_EXCEPTION_LIST_SIZE } from '@kbn/securitysolution-list-constants';
import { hasLargeValueList } from '@kbn/securitysolution-list-utils';
import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils';
import {
TimestampOverrideOrUndefined,
@ -414,46 +413,23 @@ export const parseInterval = (intervalString: string): moment.Duration | null =>
}
};
export const getDriftTolerance = ({
from,
to,
intervalDuration,
now = moment(),
}: {
from: string;
to: string;
intervalDuration: moment.Duration;
now?: moment.Moment;
}): moment.Duration => {
const toDate = parseScheduleDates(to) ?? now;
const fromDate = parseScheduleDates(from) ?? dateMath.parse('now-6m');
const timeSegment = toDate.diff(fromDate);
const duration = moment.duration(timeSegment);
return duration.subtract(intervalDuration);
};
export const getGapBetweenRuns = ({
previousStartedAt,
intervalDuration,
from,
to,
now = moment(),
originalFrom,
originalTo,
startedAt,
}: {
previousStartedAt: Date | undefined | null;
intervalDuration: moment.Duration;
from: string;
to: string;
now?: moment.Moment;
originalFrom: moment.Moment;
originalTo: moment.Moment;
startedAt: Date;
}): moment.Duration => {
if (previousStartedAt == null) {
return moment.duration(0);
}
const driftTolerance = getDriftTolerance({ from, to, intervalDuration });
const diff = moment.duration(now.diff(previousStartedAt));
const drift = diff.subtract(intervalDuration);
return drift.subtract(driftTolerance);
const driftTolerance = moment.duration(originalTo.diff(originalFrom));
const currentDuration = moment.duration(moment(startedAt).diff(previousStartedAt));
return currentDuration.subtract(driftTolerance);
};
export const makeFloatString = (num: number): string => Number(num).toFixed(2);
@ -508,6 +484,7 @@ export const getRuleRangeTuples = ({
interval,
maxSignals,
buildRuleMessage,
startedAt,
}: {
logger: Logger;
previousStartedAt: Date | null | undefined;
@ -516,9 +493,10 @@ export const getRuleRangeTuples = ({
interval: string;
maxSignals: number;
buildRuleMessage: BuildRuleMessage;
startedAt: Date;
}) => {
const originalTo = dateMath.parse(to);
const originalFrom = dateMath.parse(from);
const originalTo = dateMath.parse(to, { forceNow: startedAt });
const originalFrom = dateMath.parse(from, { forceNow: startedAt });
if (originalTo == null || originalFrom == null) {
throw new Error(buildRuleMessage('dateMath parse failed'));
}
@ -534,14 +512,19 @@ export const getRuleRangeTuples = ({
logger.error(`Failed to compute gap between rule runs: could not parse rule interval`);
return { tuples, remainingGap: moment.duration(0) };
}
const gap = getGapBetweenRuns({ previousStartedAt, intervalDuration, from, to });
const gap = getGapBetweenRuns({
previousStartedAt,
originalTo,
originalFrom,
startedAt,
});
const catchup = getNumCatchupIntervals({
gap,
intervalDuration,
});
const catchupTuples = getCatchupTuples({
to: originalTo,
from: originalFrom,
originalTo,
originalFrom,
ruleParamsMaxSignals: maxSignals,
catchup,
intervalDuration,
@ -564,22 +547,22 @@ export const getRuleRangeTuples = ({
* @param intervalDuration moment.Duration the interval which the rule runs
*/
export const getCatchupTuples = ({
to,
from,
originalTo,
originalFrom,
ruleParamsMaxSignals,
catchup,
intervalDuration,
}: {
to: moment.Moment;
from: moment.Moment;
originalTo: moment.Moment;
originalFrom: moment.Moment;
ruleParamsMaxSignals: number;
catchup: number;
intervalDuration: moment.Duration;
}): RuleRangeTuple[] => {
const catchupTuples: RuleRangeTuple[] = [];
const intervalInMilliseconds = intervalDuration.asMilliseconds();
let currentTo = to;
let currentFrom = from;
let currentTo = originalTo;
let currentFrom = originalFrom;
// This loop will create tuples with overlapping time ranges, the same way rule runs have overlapping time
// ranges due to the additional lookback. We could choose to create tuples that don't overlap here by using the
// "from" value from one tuple as "to" in the next one, however, the overlap matters for rule types like EQL and
@ -719,6 +702,20 @@ export const createSearchAfterReturnTypeFromResponse = ({
});
};
export interface PreviewReturnType {
totalCount: number;
matrixHistogramData: unknown[];
errors?: string[] | undefined;
warningMessages?: string[] | undefined;
}
export const createPreviewReturnType = (): PreviewReturnType => ({
matrixHistogramData: [],
totalCount: 0,
errors: [],
warningMessages: [],
});
export const createSearchAfterReturnType = ({
success,
warning,

18
x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts
View file

@ -5,20 +5,21 @@
* 2.0.
*/
import { SearchAfterAndBulkCreateParams, WrapHits, WrappedSignalHit } from './types';
import { WrapHits, WrappedSignalHit } from './types';
import { generateId } from './utils';
import { buildBulkBody } from './build_bulk_body';
import { filterDuplicateSignals } from './filter_duplicate_signals';
import type { ConfigType } from '../../../config';
import { CompleteRule, RuleParams } from '../schemas/rule_schemas';
export const wrapHitsFactory =
({
ruleSO,
completeRule,
signalsIndex,
mergeStrategy,
ignoreFields,
}: {
ruleSO: SearchAfterAndBulkCreateParams['ruleSO'];
completeRule: CompleteRule<RuleParams>;
signalsIndex: string;
mergeStrategy: ConfigType['alertMergeStrategy'];
ignoreFields: ConfigType['alertIgnoreFields'];
@ -27,15 +28,10 @@ export const wrapHitsFactory =
const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [
{
_index: signalsIndex,
_id: generateId(
doc._index,
doc._id,
String(doc._version),
ruleSO.attributes.params.ruleId ?? ''
),
_source: buildBulkBody(ruleSO, doc, mergeStrategy, ignoreFields, buildReasonMessage),
_id: generateId(doc._index, doc._id, String(doc._version), completeRule.alertId ?? ''),
_source: buildBulkBody(completeRule, doc, mergeStrategy, ignoreFields, buildReasonMessage),
},
]);
return filterDuplicateSignals(ruleSO.id, wrappedDocs, false);
return filterDuplicateSignals(completeRule.alertId, wrappedDocs, false);
};

9
x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts
View file

@ -5,18 +5,19 @@
* 2.0.
*/
import { SearchAfterAndBulkCreateParams, WrappedSignalHit, WrapSequences } from './types';
import { WrappedSignalHit, WrapSequences } from './types';
import { buildSignalGroupFromSequence } from './build_bulk_body';
import { ConfigType } from '../../../config';
import { CompleteRule, RuleParams } from '../schemas/rule_schemas';
export const wrapSequencesFactory =
({
ruleSO,
completeRule,
signalsIndex,
mergeStrategy,
ignoreFields,
}: {
ruleSO: SearchAfterAndBulkCreateParams['ruleSO'];
completeRule: CompleteRule<RuleParams>;
signalsIndex: string;
mergeStrategy: ConfigType['alertMergeStrategy'];
ignoreFields: ConfigType['alertIgnoreFields'];
@ -27,7 +28,7 @@ export const wrapSequencesFactory =
...acc,
...buildSignalGroupFromSequence(
sequence,
ruleSO,
completeRule,
signalsIndex,
mergeStrategy,
ignoreFields,

33
x-pack/plugins/security_solution/server/plugin.ts
View file

@ -176,6 +176,14 @@ export class Plugin implements ISecuritySolutionPlugin {
const { ruleDataService } = plugins.ruleRegistry;
let ruleDataClient: IRuleDataClient | null = null;
// rule options are used both to create and preview rules.
const ruleOptions: CreateRuleOptions = {
experimentalFeatures,
logger: this.logger,
ml: plugins.ml,
version: pluginContext.env.packageInfo.version,
};
if (isRuleRegistryEnabled) {
// NOTE: this is not used yet
// TODO: convert the aliases to FieldMaps. Requires enhancing FieldMap to support alias path.
@ -203,14 +211,6 @@ export class Plugin implements ISecuritySolutionPlugin {
secondaryAlias: config.signalsIndex,
});
// Register rule types via rule-registry
const createRuleOptions: CreateRuleOptions = {
experimentalFeatures,
logger,
ml: plugins.ml,
version: pluginContext.env.packageInfo.version,
};
const securityRuleTypeWrapper = createSecurityRuleTypeWrapper({
lists: plugins.lists,
logger: this.logger,
@ -219,17 +219,13 @@ export class Plugin implements ISecuritySolutionPlugin {
eventLogService,
});
plugins.alerting.registerType(securityRuleTypeWrapper(createEqlAlertType(createRuleOptions)));
plugins.alerting.registerType(securityRuleTypeWrapper(createEqlAlertType(ruleOptions)));
plugins.alerting.registerType(
securityRuleTypeWrapper(createIndicatorMatchAlertType(createRuleOptions))
);
plugins.alerting.registerType(securityRuleTypeWrapper(createMlAlertType(createRuleOptions)));
plugins.alerting.registerType(
securityRuleTypeWrapper(createQueryAlertType(createRuleOptions))
);
plugins.alerting.registerType(
securityRuleTypeWrapper(createThresholdAlertType(createRuleOptions))
securityRuleTypeWrapper(createIndicatorMatchAlertType(ruleOptions))
);
plugins.alerting.registerType(securityRuleTypeWrapper(createMlAlertType(ruleOptions)));
plugins.alerting.registerType(securityRuleTypeWrapper(createQueryAlertType(ruleOptions)));
plugins.alerting.registerType(securityRuleTypeWrapper(createThresholdAlertType(ruleOptions)));
}
// TODO We need to get the endpoint routes inside of initRoutes
@ -240,7 +236,8 @@ export class Plugin implements ISecuritySolutionPlugin {
plugins.security,
plugins.ml,
logger,
isRuleRegistryEnabled
isRuleRegistryEnabled,
ruleOptions
);
registerEndpointRoutes(router, endpointContext);
registerLimitedConcurrencyRoutes(core);

11
x-pack/plugins/security_solution/server/routes/index.ts
View file

@ -6,7 +6,6 @@
*/
import { Logger } from 'src/core/server';
import { SecuritySolutionPluginRouter } from '../types';
import { createRulesRoute } from '../lib/detection_engine/routes/rules/create_rules_route';
@ -57,8 +56,11 @@ import { persistPinnedEventRoute } from '../lib/timeline/routes/pinned_events';
import { SetupPlugins } from '../plugin';
import { ConfigType } from '../config';
import { installPrepackedTimelinesRoute } from '../lib/timeline/routes/prepackaged_timelines/install_prepackaged_timelines';
import { previewRulesRoute } from '../lib/detection_engine/routes/rules/preview_rules_route';
import { CreateRuleOptions } from '../lib/detection_engine/rule_types/types';
// eslint-disable-next-line no-restricted-imports
import { legacyCreateLegacyNotificationRoute } from '../lib/detection_engine/routes/rules/legacy_create_legacy_notification';
import { createPreviewIndexRoute } from '../lib/detection_engine/routes/index/create_preview_index_route';
export const initRoutes = (
router: SecuritySolutionPluginRouter,
@ -67,7 +69,8 @@ export const initRoutes = (
security: SetupPlugins['security'],
ml: SetupPlugins['ml'],
logger: Logger,
isRuleRegistryEnabled: boolean
isRuleRegistryEnabled: boolean,
ruleOptions: CreateRuleOptions
) => {
// Detection Engine Rule routes that have the REST endpoints of /api/detection_engine/rules
// All REST rule creation, deletion, updating, etc......
@ -77,6 +80,7 @@ export const initRoutes = (
patchRulesRoute(router, ml, isRuleRegistryEnabled);
deleteRulesRoute(router, isRuleRegistryEnabled);
findRulesRoute(router, logger, isRuleRegistryEnabled);
previewRulesRoute(router, config, ml, security, ruleOptions);
// Once we no longer have the legacy notifications system/"side car actions" this should be removed.
legacyCreateLegacyNotificationRoute(router, logger);
@ -129,6 +133,9 @@ export const initRoutes = (
readIndexRoute(router, config);
deleteIndexRoute(router);
// Detection Engine Preview Index /api/detection_engine/preview/index
createPreviewIndexRoute(router);
// Detection Engine tags routes that have the REST endpoints of /api/detection_engine/tags
readTagsRoute(router, isRuleRegistryEnabled);

1
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts
View file

@ -162,7 +162,6 @@ export default ({ getService }: FtrProviderContext) => {
enabled: true,
created_by: 'elastic',
updated_by: 'elastic',
throttle: null,
description: 'Test ML rule description',
risk_score: 50,
severity: 'critical',

18
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts
View file

@ -235,7 +235,7 @@ export default ({ getService }: FtrProviderContext) => {
parents: [
{
rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
id: signalNoRule.parents[0].id, // id is always changing so skip testing it
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -250,7 +250,7 @@ export default ({ getService }: FtrProviderContext) => {
},
{
rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
id: signalNoRule.ancestors[1].id, // id is always changing so skip testing it
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -260,7 +260,7 @@ export default ({ getService }: FtrProviderContext) => {
depth: 2,
parent: {
rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
id: signalNoRule.parent?.id, // parent.id is always changing so skip testing it
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1248,7 +1248,7 @@ export default ({ getService }: FtrProviderContext) => {
parents: [
{
rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
id: signalNoRule.parents[0].id, // id is always changing so skip testing it
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1263,7 +1263,7 @@ export default ({ getService }: FtrProviderContext) => {
},
{
rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
id: signalNoRule.ancestors[1].id, // id is always changing so skip testing it
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1273,7 +1273,7 @@ export default ({ getService }: FtrProviderContext) => {
depth: 2,
parent: {
rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
id: signalNoRule.parent?.id, // parent.id is always changing so skip testing it
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1408,7 +1408,7 @@ export default ({ getService }: FtrProviderContext) => {
parents: [
{
rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
id: signalNoRule.parents[0].id, // id is always changing so skip testing it
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1423,7 +1423,7 @@ export default ({ getService }: FtrProviderContext) => {
},
{
rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
id: signalNoRule.ancestors[1].id, // id is always changing so skip testing it
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1433,7 +1433,7 @@ export default ({ getService }: FtrProviderContext) => {
depth: 2,
parent: {
rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
id: signalNoRule.parent?.id, // parent.id is always changing so skip testing it
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,