github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Alerts] adds suppression for missing fields options for security rules (#155055)

Browse source 
## Summary

- adresses https://github.com/elastic/kibana/issues/150101
- fixes https://github.com/elastic/kibana/issues/155242
- introduces UI options for selections 2 modes for suppressing or not
suppressing alerts with missing **Group By** fields
- adds accordion that contains all suppression related logic

### UX changes
#### Rule edit page
<details>
<summary>Accordion closed</summary>
<img width="1042" alt="Screenshot 2023-04-21 at 16 09 44"
src="https://user-images.githubusercontent.com/92328789/233700543-8a5091e0-6455-4d76-b6b6-7a280d747d0c.png">
</details>

<details>
<summary>Accordion opened</summary>
<img width="1017" alt="Screenshot 2023-04-24 at 19 44 33"
src="https://user-images.githubusercontent.com/92328789/234087516-58b88dab-0285-47ca-a016-bfff31dbebae.png">
</details>

#### Rule Details page

<img width="2293" alt="Screenshot 2023-04-19 at 18 50 13"
src="https://user-images.githubusercontent.com/92328789/234004667-d879bfff-0d11-4bc9-ab5b-7ad904e29d1f.png">


### Checklist

Delete any items that are not applicable to this PR.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [x] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [x] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [x] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)
This commit is contained in:
Vitalii Dmyterko 2023-04-25 19:57:30 +01:00 • committed by GitHub
parent 609228bc95
commit 5fb93a1ea2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 1105 additions and 191 deletions
Download patch file Download diff file Expand all files Collapse all files

21
x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/query_attributes.ts
View file

@ -9,8 +9,27 @@ import * as t from 'io-ts';
import {
LimitedSizeArray,
PositiveIntegerGreaterThanZero,
enumeration,
} from '@kbn/securitysolution-io-ts-types';
/**
* describes how alerts will be generated for documents with missing suppress by fields
*/
export enum AlertSuppressionMissingFieldsStrategy {
// per each document a separate alert will be created
DoNotSuppress = 'doNotSuppress',
// only alert will be created per suppress by bucket
Suppress = 'suppress',
}
export type AlertSuppressionMissingFields = t.TypeOf<typeof AlertSuppressionMissingFields>;
export const AlertSuppressionMissingFields = enumeration(
'AlertSuppressionMissingFields',
AlertSuppressionMissingFieldsStrategy
);
export const DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY =
AlertSuppressionMissingFieldsStrategy.Suppress;
export const AlertSuppressionGroupBy = LimitedSizeArray({
codec: t.string,
minSize: 1,
@ -41,6 +60,7 @@ export const AlertSuppression = t.intersection([
t.exact(
t.partial({
duration: AlertSuppressionDuration,
missing_fields_strategy: AlertSuppressionMissingFields,
})
),
]);
@ -55,6 +75,7 @@ export const AlertSuppressionCamel = t.intersection([
t.exact(
t.partial({
duration: AlertSuppressionDuration,
missingFieldsStrategy: AlertSuppressionMissingFields,
})
),
]);

4
x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts
View file

@ -48,6 +48,7 @@ import {
GroupByOptions,
} from '../../../../detections/pages/detection_engine/rules/types';
import type { RuleCreateProps } from '../../../../../common/detection_engine/rule_schema';
import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/rule_schema';
import { stepActionsDefaultValue } from '../../../../detections/components/rules/step_rule_actions';
export const getTimeTypeValue = (time: string): { unit: Unit; value: number } => {
@ -447,6 +448,9 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep
ruleFields.groupByRadioSelection === GroupByOptions.PerTimePeriod
? ruleFields.groupByDuration
: undefined,
missing_fields_strategy:
ruleFields.suppressionMissingFields ||
DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY,
},
}
: {}),

60
x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx
View file

@ -15,7 +15,6 @@ import {
EuiIcon,
EuiToolTip,
EuiFlexGrid,
EuiBetaBadge,
} from '@elastic/eui';
import { ALERT_RISK_SCORE } from '@kbn/rule-data-utils';
@ -37,7 +36,6 @@ import type {
RequiredFieldArray,
Threshold,
} from '../../../../../common/detection_engine/rule_schema';
import { minimumLicenseForSuppression } from '../../../../../common/detection_engine/rule_schema';
import * as i18n from './translations';
import type { BuildQueryBarDescription, BuildThreatDescription, ListItems } from './types';
@ -50,8 +48,9 @@ import type {
import { GroupByOptions } from '../../../pages/detection_engine/rules/types';
import { defaultToEmptyTag } from '../../../../common/components/empty_value';
import { ThreatEuiFlexGroup } from './threat_description';
import { TechnicalPreviewBadge } from './technical_preview_badge';
import type { LicenseService } from '../../../../../common/license';
import { AlertSuppressionMissingFieldsStrategy } from '../../../../../common/detection_engine/rule_schema';
const NoteDescriptionContainer = styled(EuiFlexItem)`
height: 105px;
overflow-y: hidden;
@ -535,21 +534,7 @@ export const buildAlertSuppressionDescription = (
</EuiFlexGroup>
);
const title = (
<>
{label}
<EuiBetaBadge
label={i18n.ALERT_SUPPRESSION_TECHNICAL_PREVIEW}
style={{ verticalAlign: 'middle', marginLeft: '8px' }}
size="s"
/>
{!license.isAtLeast(minimumLicenseForSuppression) && (
<EuiToolTip position="top" content={i18n.ALERT_SUPPRESSION_INSUFFICIENT_LICENSE}>
<EuiIcon type={'warning'} size="l" color="#BD271E" style={{ marginLeft: '8px' }} />
</EuiToolTip>
)}
</>
);
const title = <TechnicalPreviewBadge label={label} license={license} />;
return [
{
title,
@ -569,21 +554,30 @@ export const buildAlertSuppressionWindowDescription = (
? `${value.value}${value.unit}`
: i18n.ALERT_SUPPRESSION_PER_RULE_EXECUTION;
const title = (
<>
{label}
<EuiBetaBadge
label={i18n.ALERT_SUPPRESSION_TECHNICAL_PREVIEW}
style={{ verticalAlign: 'middle', marginLeft: '8px' }}
size="s"
/>
{!license.isAtLeast(minimumLicenseForSuppression) && (
<EuiToolTip position="top" content={i18n.ALERT_SUPPRESSION_INSUFFICIENT_LICENSE}>
<EuiIcon type={'warning'} size="l" color="#BD271E" style={{ marginLeft: '8px' }} />
</EuiToolTip>
)}
</>
);
const title = <TechnicalPreviewBadge label={label} license={license} />;
return [
{
title,
description,
},
];
};
export const buildAlertSuppressionMissingFieldsDescription = (
label: string,
value: AlertSuppressionMissingFieldsStrategy,
license: LicenseService
): ListItems[] => {
if (isEmpty(value)) {
return [];
}
const description =
value === AlertSuppressionMissingFieldsStrategy.Suppress
? i18n.ALERT_SUPPRESSION_SUPPRESS_ON_MISSING_FIELDS
: i18n.ALERT_SUPPRESSION_DO_NOT_SUPPRESS_ON_MISSING_FIELDS;
const title = <TechnicalPreviewBadge label={label} license={license} />;
return [
{
title,

8
x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx
View file

@ -46,6 +46,7 @@ import {
buildRequiredFieldsDescription,
buildAlertSuppressionDescription,
buildAlertSuppressionWindowDescription,
buildAlertSuppressionMissingFieldsDescription,
} from './helpers';
import { buildMlJobsDescription } from './build_ml_jobs_description';
import { buildActionsDescription } from './actions_description';
@ -216,6 +217,13 @@ export const getDescriptionItem = (
} else {
return [];
}
} else if (field === 'suppressionMissingFields') {
if (get('groupByFields', data).length > 0) {
const value = get(field, data);
return buildAlertSuppressionMissingFieldsDescription(label, value, license);
} else {
return [];
}
} else if (field === 'eqlOptions') {
const eqlOptions: EqlOptionsSelected = get(field, data);
return buildEqlOptionsDescription(eqlOptions);

35
x-pack/plugins/security_solution/public/detections/components/rules/description_step/technical_preview_badge.tsx Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React from 'react';
import { EuiIcon, EuiToolTip, EuiBetaBadge } from '@elastic/eui';
import type { LicenseService } from '../../../../../common/license';
import { minimumLicenseForSuppression } from '../../../../../common/detection_engine/rule_schema';
import * as i18n from './translations';
interface TechnicalPreviewBadgeProps {
label: string;
license: LicenseService;
}
export const TechnicalPreviewBadge = ({ label, license }: TechnicalPreviewBadgeProps) => (
<>
{label}
<EuiBetaBadge
label={i18n.ALERT_SUPPRESSION_TECHNICAL_PREVIEW}
style={{ verticalAlign: 'middle', marginLeft: '8px' }}
size="s"
/>
{!license.isAtLeast(minimumLicenseForSuppression) && (
<EuiToolTip position="top" content={i18n.ALERT_SUPPRESSION_INSUFFICIENT_LICENSE}>
<EuiIcon type={'warning'} size="l" color="#BD271E" style={{ marginLeft: '8px' }} />
</EuiToolTip>
)}
</>
);

14
x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.ts
View file

@ -147,3 +147,17 @@ export const ALERT_SUPPRESSION_PER_RULE_EXECUTION = i18n.translate(
defaultMessage: 'One rule execution',
}
);
export const ALERT_SUPPRESSION_SUPPRESS_ON_MISSING_FIELDS = i18n.translate(
'xpack.securitySolution.detectionEngine.ruleDescription.alertSuppressionSuppressOnMissingFieldsDescription',
{
defaultMessage: 'Suppress on missing field value',
}
);
export const ALERT_SUPPRESSION_DO_NOT_SUPPRESS_ON_MISSING_FIELDS = i18n.translate(
'xpack.securitySolution.detectionEngine.ruleDescription.alertSuppressionDoNotSuppressOnMissingFieldsDescription',
{
defaultMessage: 'Do not suppress',
}
);

128
x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx
View file

@ -7,6 +7,7 @@
import type { EuiButtonGroupOptionProps } from '@elastic/eui';
import {
EuiAccordion,
EuiButtonEmpty,
EuiFlexGroup,
EuiFlexItem,
@ -88,7 +89,10 @@ import { defaultCustomQuery } from '../../../pages/detection_engine/rules/utils'
import { getIsRulePreviewDisabled } from '../rule_preview/helpers';
import { GroupByFields } from '../group_by_fields';
import { useLicense } from '../../../../common/hooks/use_license';
import { minimumLicenseForSuppression } from '../../../../../common/detection_engine/rule_schema';
import {
minimumLicenseForSuppression,
AlertSuppressionMissingFieldsStrategy,
} from '../../../../../common/detection_engine/rule_schema';
import { DurationInput } from '../duration_input';
const CommonUseField = getUseField({ component: Field });
@ -179,6 +183,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
'groupByRadioSelection',
'groupByDuration.value',
'groupByDuration.unit',
'suppressionMissingFields',
],
onChange: (data: DefineStepRule) => {
if (onRuleDataChange) {
@ -561,6 +566,34 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
[license, groupByFields]
);
const AlertsSuppressionMissingFields = useCallback(
({ suppressionMissingFields }) => (
<EuiRadioGroup
disabled={
!license.isAtLeast(minimumLicenseForSuppression) ||
groupByFields == null ||
groupByFields.length === 0
}
idSelected={suppressionMissingFields.value}
options={[
{
id: AlertSuppressionMissingFieldsStrategy.Suppress,
label: i18n.ALERT_SUPPRESSION_MISSING_FIELDS_SUPPRESS_OPTION,
},
{
id: AlertSuppressionMissingFieldsStrategy.DoNotSuppress,
label: i18n.ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS_OPTION,
},
]}
onChange={(id: string) => {
suppressionMissingFields.setValue(id);
}}
data-test-subj="suppressionMissingFieldsOptions"
/>
),
[license, groupByFields]
);
const dataViewIndexPatternToggleButtonOptions: EuiButtonGroupOptionProps[] = useMemo(
() => [
{
@ -868,41 +901,68 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
</>
)}
<RuleTypeEuiFormRow
$isVisible={isQueryRule(ruleType)}
data-test-subj="alertSuppressionInput"
<EuiSpacer size="l" />
<EuiAccordion
data-test-subj="alertSuppressionAccordion"
id="alertSuppressionAccordion"
buttonContent={i18n.ALERT_SUPPRESSION_ACCORDION_BUTTON}
>
<UseField
path="groupByFields"
component={GroupByFields}
componentProps={{
browserFields: termsAggregationFields,
isDisabled:
!license.isAtLeast(minimumLicenseForSuppression) &&
initialState.groupByFields.length === 0,
}}
/>
</RuleTypeEuiFormRow>
<RuleTypeEuiFormRow
$isVisible={isQueryRule(ruleType)}
data-test-subj="alertSuppressionDuration"
>
<UseMultiFields
fields={{
groupByRadioSelection: {
path: 'groupByRadioSelection',
},
groupByDurationValue: {
path: 'groupByDuration.value',
},
groupByDurationUnit: {
path: 'groupByDuration.unit',
},
}}
<EuiSpacer size="l" />
<RuleTypeEuiFormRow
$isVisible={isQueryRule(ruleType)}
data-test-subj="alertSuppressionInput"
>
{GroupByChildren}
</UseMultiFields>
</RuleTypeEuiFormRow>
<UseField
path="groupByFields"
component={GroupByFields}
componentProps={{
browserFields: termsAggregationFields,
isDisabled:
!license.isAtLeast(minimumLicenseForSuppression) &&
initialState.groupByFields.length === 0,
}}
/>
</RuleTypeEuiFormRow>
<RuleTypeEuiFormRow
$isVisible={isQueryRule(ruleType)}
data-test-subj="alertSuppressionDuration"
>
<UseMultiFields
fields={{
groupByRadioSelection: {
path: 'groupByRadioSelection',
},
groupByDurationValue: {
path: 'groupByDuration.value',
},
groupByDurationUnit: {
path: 'groupByDuration.unit',
},
}}
>
{GroupByChildren}
</UseMultiFields>
</RuleTypeEuiFormRow>
<RuleTypeEuiFormRow
$isVisible={isQueryRule(ruleType)}
data-test-subj="alertSuppressionMissingFields"
label={i18n.ALERT_SUPPRESSION_MISSING_FIELDS_FORM_ROW_LABEL}
>
<UseMultiFields
fields={{
suppressionMissingFields: {
path: 'suppressionMissingFields',
},
}}
>
{AlertsSuppressionMissingFields}
</UseMultiFields>
</RuleTypeEuiFormRow>
</EuiAccordion>
<EuiSpacer size="l" />
<RuleTypeEuiFormRow $isVisible={isMlRule(ruleType)} fullWidth>
<>

8
x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx
View file

@ -622,6 +622,14 @@ export const schema: FormSchema<DefineStepRule> = {
value: {},
unit: {},
},
suppressionMissingFields: {
label: i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.suppressionMissingFieldsLabel',
{
defaultMessage: 'If “Suppress by” field does not exist',
}
),
},
newTermsFields: {
type: FIELD_TYPES.COMBO_BOX,
label: i18n.translate(

28
x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx
View file

@ -149,3 +149,31 @@ export const RULE_PREVIEW_TITLE = i18n.translate(
defaultMessage: 'Rule Preview',
}
);
export const ALERT_SUPPRESSION_ACCORDION_BUTTON = i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.alertSuppressionAccordionButtonLabel',
{
defaultMessage: 'Suppression Configuration',
}
);
export const ALERT_SUPPRESSION_MISSING_FIELDS_FORM_ROW_LABEL = i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.alertSuppressionMissingFieldsLabel',
{
defaultMessage: 'If “Suppress by” field does not exist',
}
);
export const ALERT_SUPPRESSION_MISSING_FIELDS_SUPPRESS_OPTION = i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.alertSuppressionMissingFieldsSuppressLabel',
{
defaultMessage: 'Suppress on missing field value',
}
);
export const ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS_OPTION = i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.alertSuppressionMissingFieldsDoNotSuppressLabel',
{
defaultMessage: 'Do not suppress',
}
);

112
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx
View file

@ -25,6 +25,7 @@ import {
mockRule,
} from '../../../../detection_engine/rule_management_ui/components/rules_table/__mocks__/mock';
import { FilterStateStore } from '@kbn/es-query';
import { AlertSuppressionMissingFieldsStrategy } from '../../../../../common/detection_engine/rule_schema';
import type { Rule } from '../../../../detection_engine/rule_management/logic';
import type {
@ -124,6 +125,7 @@ describe('rule helpers', () => {
groupByRadioSelection: 'per-rule-execution',
newTermsFields: ['host.name'],
historyWindowSize: '7d',
suppressionMissingFields: expect.any(String),
};
const aboutRuleStepData: AboutStepRule = {
@ -224,13 +226,8 @@ describe('rule helpers', () => {
describe('getDefineStepsData', () => {
test('returns with saved_id if value exists on rule', () => {
const result: DefineStepRule = getDefineStepsData(mockRule('test-id'));
const expected = {
const expected = expect.objectContaining({
ruleType: 'saved_query',
anomalyThreshold: 50,
dataSourceType: 'indexPatterns',
dataViewId: undefined,
machineLearningJobId: [],
index: ['auditbeat-*'],
queryBar: {
query: {
query: '',
@ -239,41 +236,8 @@ describe('rule helpers', () => {
filters: [],
saved_id: "Garrett's IP",
},
relatedIntegrations: [],
requiredFields: [],
threshold: {
field: [],
value: '100',
},
threatIndex: [],
threatMapping: [],
threatQueryBar: {
query: {
query: '',
language: '',
},
filters: [],
saved_id: null,
},
timeline: {
id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2',
title: 'Untitled timeline',
},
eqlOptions: {
timestampField: undefined,
eventCategoryField: undefined,
tiebreakerField: undefined,
},
groupByFields: [],
groupByDuration: {
value: 5,
unit: 'm',
},
groupByRadioSelection: 'per-rule-execution',
newTermsFields: [],
historyWindowSize: '7d',
shouldLoadQueryDynamically: true,
};
});
expect(result).toEqual(expected);
});
@ -284,13 +248,8 @@ describe('rule helpers', () => {
};
delete mockedRule.saved_id;
const result: DefineStepRule = getDefineStepsData(mockedRule);
const expected = {
const expected = expect.objectContaining({
ruleType: 'saved_query',
anomalyThreshold: 50,
dataSourceType: 'indexPatterns',
dataViewId: undefined,
machineLearningJobId: [],
index: ['auditbeat-*'],
queryBar: {
query: {
query: '',
@ -299,41 +258,8 @@ describe('rule helpers', () => {
filters: [],
saved_id: null,
},
relatedIntegrations: [],
requiredFields: [],
threshold: {
field: [],
value: '100',
},
threatIndex: [],
threatMapping: [],
threatQueryBar: {
query: {
query: '',
language: '',
},
filters: [],
saved_id: null,
},
timeline: {
id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2',
title: 'Untitled timeline',
},
eqlOptions: {
timestampField: undefined,
eventCategoryField: undefined,
tiebreakerField: undefined,
},
groupByFields: [],
groupByDuration: {
value: 5,
unit: 'm',
},
groupByRadioSelection: 'per-rule-execution',
newTermsFields: [],
historyWindowSize: '7d',
shouldLoadQueryDynamically: false,
};
});
expect(result).toEqual(expected);
});
@ -347,6 +273,32 @@ describe('rule helpers', () => {
expect(result.timeline.id).toBeNull();
expect(result.timeline.title).toBeNull();
});
describe('suppression on missing fields', () => {
test('returns default suppress value in suppress strategy is missing', () => {
const result: DefineStepRule = getDefineStepsData(mockRule('test-id'));
const expected = expect.objectContaining({
suppressionMissingFields: AlertSuppressionMissingFieldsStrategy.Suppress,
});
expect(result).toEqual(expected);
});
test('returns suppress value if rule is configured with missing_fields_strategy', () => {
const result: DefineStepRule = getDefineStepsData({
...mockRule('test-id'),
alert_suppression: {
group_by: [],
missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.DoNotSuppress,
},
});
const expected = expect.objectContaining({
suppressionMissingFields: AlertSuppressionMissingFieldsStrategy.DoNotSuppress,
});
expect(result).toEqual(expected);
});
});
});
describe('getHumanizedDuration', () => {

3
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx
View file

@ -23,6 +23,7 @@ import type { Filter } from '@kbn/es-query';
import type { ActionVariables } from '@kbn/triggers-actions-ui-plugin/public';
import type { ResponseAction } from '../../../../../common/detection_engine/rule_response_actions/schemas';
import { normalizeThresholdField } from '../../../../../common/detection_engine/utils';
import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/rule_schema';
import type { RuleAlertAction } from '../../../../../common/detection_engine/types';
import { assertUnreachable } from '../../../../../common/utility_types';
import {
@ -139,6 +140,8 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({
? GroupByOptions.PerTimePeriod
: GroupByOptions.PerRuleExecution,
groupByDuration: rule.alert_suppression?.duration ?? { value: 5, unit: 'm' },
suppressionMissingFields:
rule.alert_suppression?.missing_fields_strategy ?? DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY,
});
const convertHistoryStartToSize = (relativeTime: string) => {

2
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts
View file

@ -33,6 +33,7 @@ import type {
RuleNameOverride,
SetupGuide,
TimestampOverride,
AlertSuppressionMissingFields,
} from '../../../../../common/detection_engine/rule_schema';
import type { SortOrder } from '../../../../../common/detection_engine/schemas/common';
import type { EqlOptionsSelected } from '../../../../../common/search_strategy';
@ -176,6 +177,7 @@ export interface DefineStepRule {
groupByFields: string[];
groupByRadioSelection: GroupByOptions;
groupByDuration: Duration;
suppressionMissingFields?: AlertSuppressionMissingFields;
}
export interface Duration {

2
x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts
View file

@ -370,6 +370,7 @@ export const convertAlertSuppressionToCamel = (
? {
groupBy: input.group_by,
duration: input.duration,
missingFieldsStrategy: input.missing_fields_strategy,
}
: undefined;
@ -380,5 +381,6 @@ export const convertAlertSuppressionToSnake = (
? {
group_by: input.groupBy,
duration: input.duration,
missing_fields_strategy: input.missingFieldsStrategy,
}
: undefined;

1
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/__snapshots__/build_group_by_field_aggregation.test.ts.snap generated
View file

@ -35,7 +35,6 @@ Object {
"host.name": Object {
"terms": Object {
"field": "host.name",
"missing_bucket": true,
},
},
},

2
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/__snapshots__/group_and_bulk_create.test.ts.snap generated
View file

@ -28,6 +28,7 @@ Array [
},
},
],
"must_not": Array [],
},
},
Object {
@ -53,6 +54,7 @@ Array [
},
},
],
"must_not": Array [],
},
},
],

33
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/build_group_by_field_aggregation.test.ts
View file

@ -16,7 +16,40 @@ describe('build_group_by_field_aggregation', () => {
groupByFields,
maxSignals,
aggregatableTimestampField: 'kibana.combined_timestamp',
missingBucket: false,
});
expect(agg).toMatchSnapshot();
});
it('should include missing bucket configuration for aggregation if configured', () => {
const groupByFields = ['host.name'];
const maxSignals = 100;
const agg = buildGroupByFieldAggregation({
groupByFields,
maxSignals,
aggregatableTimestampField: 'kibana.combined_timestamp',
missingBucket: true,
});
expect(agg.eventGroups.composite.sources[0]['host.name'].terms).toEqual({
field: 'host.name',
missing_bucket: true,
missing_order: 'last',
});
});
it('should not include missing bucket configuration for aggregation if not configured', () => {
const groupByFields = ['host.name'];
const maxSignals = 100;
const agg = buildGroupByFieldAggregation({
groupByFields,
maxSignals,
aggregatableTimestampField: 'kibana.combined_timestamp',
missingBucket: false,
});
expect(agg.eventGroups.composite.sources[0]['host.name'].terms).toEqual({
field: 'host.name',
});
});
});

18
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/build_group_by_field_aggregation.ts
View file

@ -5,16 +5,30 @@
* 2.0.
*/
import type { ESSearchResponse } from '@kbn/es-types';
import type { SignalSource } from '../../types';
export type EventGroupingMultiBucketAggregationResult = ESSearchResponse<
SignalSource,
{
body: {
aggregations: ReturnType<typeof buildGroupByFieldAggregation>;
};
}
>;
interface GetGroupByFieldAggregationArgs {
groupByFields: string[];
maxSignals: number;
aggregatableTimestampField: string;
missingBucket: boolean;
}
export const buildGroupByFieldAggregation = ({
groupByFields,
maxSignals,
aggregatableTimestampField,
missingBucket,
}: GetGroupByFieldAggregationArgs) => ({
eventGroups: {
composite: {
@ -22,7 +36,9 @@ export const buildGroupByFieldAggregation = ({
[field]: {
terms: {
field,
missing_bucket: true,
...(missingBucket
? { missing_bucket: missingBucket, missing_order: 'last' as const }
: {}),
},
},
})),

90
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/bulk_create_unsuppressed_alerts.ts Normal file
View file

@ -0,0 +1,90 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
import type { RunOpts, SearchAfterAndBulkCreateReturnType, RuleServices } from '../../types';
import type { UnifiedQueryRuleParams } from '../../../rule_schema';
import type { BuildReasonMessage } from '../../utils/reason_formatters';
import { searchAfterAndBulkCreate } from '../../utils/search_after_bulk_create';
import type { ITelemetryEventsSender } from '../../../../telemetry/sender';
type BulkCreateUnsuppressedAlerts = (params: {
runOpts: RunOpts<UnifiedQueryRuleParams>;
size: number;
groupByFields: string[];
buildReasonMessage: BuildReasonMessage;
services: RuleServices;
filter: QueryDslQueryContainer;
eventsTelemetry: ITelemetryEventsSender | undefined;
}) => Promise<SearchAfterAndBulkCreateReturnType>;
/**
* searches and bulk creates unsuppressed alerts if any exists
* @param param0
* @returns
*/
export const bulkCreateUnsuppressedAlerts: BulkCreateUnsuppressedAlerts = async ({
size,
groupByFields,
buildReasonMessage,
runOpts,
filter,
services,
eventsTelemetry,
}) => {
const bulkCreatedResult = await searchAfterAndBulkCreate({
tuple: { ...runOpts.tuple, maxSignals: size },
exceptionsList: runOpts.unprocessedExceptions,
services,
listClient: runOpts.listClient,
ruleExecutionLogger: runOpts.ruleExecutionLogger,
eventsTelemetry,
inputIndexPattern: runOpts.inputIndex,
pageSize: runOpts.searchAfterSize,
filter,
buildReasonMessage,
bulkCreate: runOpts.bulkCreate,
wrapHits: runOpts.wrapHits,
runtimeMappings: runOpts.runtimeMappings,
primaryTimestamp: runOpts.primaryTimestamp,
secondaryTimestamp: runOpts.secondaryTimestamp,
additionalFilters: buildMissingFieldsFilter(groupByFields),
});
return bulkCreatedResult;
};
/**
* builds filter that returns only docs with at least one missing field from a list of groupByFields fields
* @param groupByFields
* @returns - Array<{@link QueryDslQueryContainer}>
*/
const buildMissingFieldsFilter = (groupByFields: string[]): QueryDslQueryContainer[] => {
if (groupByFields.length === 0) {
return [];
}
return [
{
bool: {
should: groupByFields.map((field) => ({
bool: {
must_not: [
{
exists: {
field,
},
},
],
},
})),
},
},
];
};

99
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts
View file

@ -9,24 +9,28 @@ import type moment from 'moment';
import type * as estypes from '@elastic/elasticsearch/lib/api/types';
import type { ESSearchResponse } from '@kbn/es-types';
import { withSecuritySpan } from '../../../../../utils/with_security_span';
import { buildTimeRangeFilter } from '../../utils/build_events_query';
import type {
RuleServices,
RunOpts,
SearchAfterAndBulkCreateReturnType,
SignalSource,
} from '../../types';
import { addToSearchAfterReturn, getUnprocessedExceptionsWarnings } from '../../utils/utils';
import type { SuppressionBuckets } from './wrap_suppressed_alerts';
import type { RuleServices, RunOpts, SearchAfterAndBulkCreateReturnType } from '../../types';
import {
addToSearchAfterReturn,
getUnprocessedExceptionsWarnings,
mergeReturns,
} from '../../utils/utils';
import type { SuppressionBucket } from './wrap_suppressed_alerts';
import { wrapSuppressedAlerts } from './wrap_suppressed_alerts';
import { buildGroupByFieldAggregation } from './build_group_by_field_aggregation';
import type { EventGroupingMultiBucketAggregationResult } from './build_group_by_field_aggregation';
import { singleSearchAfter } from '../../utils/single_search_after';
import { bulkCreateWithSuppression } from './bulk_create_with_suppression';
import type { UnifiedQueryRuleParams } from '../../../rule_schema';
import type { BuildReasonMessage } from '../../utils/reason_formatters';
import {
AlertSuppressionMissingFieldsStrategy,
DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY,
} from '../../../../../../common/detection_engine/rule_schema';
import { bulkCreateUnsuppressedAlerts } from './bulk_create_unsuppressed_alerts';
import type { ITelemetryEventsSender } from '../../../../telemetry/sender';
export interface BucketHistory {
key: Record<string, string | number | null>;
@ -41,6 +45,7 @@ export interface GroupAndBulkCreateParams {
buildReasonMessage: BuildReasonMessage;
bucketHistory?: BucketHistory[];
groupByFields: string[];
eventsTelemetry: ITelemetryEventsSender | undefined;
}
export interface GroupAndBulkCreateReturnType extends SearchAfterAndBulkCreateReturnType {
@ -49,15 +54,6 @@ export interface GroupAndBulkCreateReturnType extends SearchAfterAndBulkCreateRe
};
}
type EventGroupingMultiBucketAggregationResult = ESSearchResponse<
SignalSource,
{
body: {
aggregations: ReturnType<typeof buildGroupByFieldAggregation>;
};
}
>;
/**
* Builds a filter that excludes documents from existing buckets.
*/
@ -80,22 +76,21 @@ export const buildBucketHistoryFilter = ({
bool: {
must_not: bucketHistory.map((bucket) => ({
bool: {
must_not: Object.entries(bucket.key)
.filter(([_, value]) => value == null)
.map(([field, _]) => ({
exists: {
field,
},
})),
filter: [
...Object.entries(bucket.key).map(([field, value]) =>
value != null
? {
term: {
[field]: value,
},
}
: {
must_not: {
exists: {
field,
},
},
}
),
...Object.entries(bucket.key)
.filter(([_, value]) => value != null)
.map(([field, value]) => ({
term: {
[field]: value,
},
})),
buildTimeRangeFilter({
to: bucket.endDate,
from: from.toISOString(),
@ -128,6 +123,7 @@ export const groupAndBulkCreate = async ({
buildReasonMessage,
bucketHistory,
groupByFields,
eventsTelemetry,
}: GroupAndBulkCreateParams): Promise<GroupAndBulkCreateReturnType> => {
return withSecuritySpan('groupAndBulkCreate', async () => {
const tuple = runOpts.tuple;
@ -137,7 +133,7 @@ export const groupAndBulkCreate = async ({
fromDate: tuple.from.toDate(),
});
const toReturn: GroupAndBulkCreateReturnType = {
let toReturn: GroupAndBulkCreateReturnType = {
success: true,
warning: false,
searchAfterTimes: [],
@ -170,13 +166,20 @@ export const groupAndBulkCreate = async ({
from: tuple.from,
});
// if we do not suppress alerts for docs with missing values, we will create aggregation for null missing buckets
const suppressOnMissingFields =
(runOpts.completeRule.ruleParams.alertSuppression?.missingFieldsStrategy ??
DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) ===
AlertSuppressionMissingFieldsStrategy.Suppress;
const groupingAggregation = buildGroupByFieldAggregation({
groupByFields,
maxSignals: tuple.maxSignals,
aggregatableTimestampField: runOpts.aggregatableTimestampField,
missingBucket: suppressOnMissingFields,
});
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
const eventsSearchParams = {
aggregations: groupingAggregation,
searchAfterSortIds: undefined,
index: runOpts.inputIndex,
@ -190,7 +193,11 @@ export const groupAndBulkCreate = async ({
secondaryTimestamp: runOpts.secondaryTimestamp,
runtimeMappings: runOpts.runtimeMappings,
additionalFilters: bucketHistoryFilter,
});
};
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter(
eventsSearchParams
);
toReturn.searchAfterTimes.push(searchDuration);
toReturn.errors.push(...searchErrors);
@ -202,11 +209,26 @@ export const groupAndBulkCreate = async ({
const buckets = eventsByGroupResponseWithAggs.aggregations.eventGroups.buckets;
// we can create only as many unsuppressed alerts, as total number of alerts(suppressed and unsuppressed) does not exceeds maxSignals
const maxUnsuppressedCount = tuple.maxSignals - buckets.length;
if (suppressOnMissingFields === false && maxUnsuppressedCount > 0) {
const unsuppressedResult = await bulkCreateUnsuppressedAlerts({
groupByFields,
size: maxUnsuppressedCount,
runOpts,
buildReasonMessage,
eventsTelemetry,
filter,
services,
});
toReturn = { ...toReturn, ...mergeReturns([toReturn, unsuppressedResult]) };
}
if (buckets.length === 0) {
return toReturn;
}
const suppressionBuckets: SuppressionBuckets[] = buckets.map((bucket) => ({
const suppressionBuckets: SuppressionBucket[] = buckets.map((bucket) => ({
event: bucket.topHits.hits.hits[0],
count: bucket.doc_count,
start: bucket.min_timestamp.value_as_string
@ -243,6 +265,7 @@ export const groupAndBulkCreate = async ({
alertTimestampOverride: runOpts.alertTimestampOverride,
});
addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult });
runOpts.ruleExecutionLogger.debug(`created ${bulkCreateResult.createdItemsCount} signals`);
} else {
const bulkCreateResult = await runOpts.bulkCreate(wrappedAlerts);
addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult });

4
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts
View file

@ -26,7 +26,7 @@ import type { SignalSource } from '../../types';
import { buildBulkBody } from '../../factories/utils/build_bulk_body';
import type { BuildReasonMessage } from '../../utils/reason_formatters';
export interface SuppressionBuckets {
export interface SuppressionBucket {
event: estypes.SearchHit<SignalSource>;
count: number;
start: Date;
@ -57,7 +57,7 @@ export const wrapSuppressedAlerts = ({
ruleExecutionLogger,
publicBaseUrl,
}: {
suppressionBuckets: SuppressionBuckets[];
suppressionBuckets: SuppressionBucket[];
spaceId: string;
completeRule: CompleteRule<RuleParams>;
mergeStrategy: ConfigType['alertMergeStrategy'];

1
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts
View file

@ -74,6 +74,7 @@ export const queryExecutor = async ({
buildReasonMessage: buildReasonMessageForQueryAlert,
bucketHistory,
groupByFields: ruleParams.alertSuppression.groupBy,
eventsTelemetry,
})
: {
...(await searchAfterAndBulkCreate({

1
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts
View file

@ -374,6 +374,7 @@ export interface SearchAfterAndBulkCreateParams {
runtimeMappings: estypes.MappingRuntimeFields | undefined;
primaryTimestamp: string;
secondaryTimestamp?: string;
additionalFilters?: estypes.QueryDslQueryContainer[];
}
export interface SearchAfterAndBulkCreateReturnType {

2
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.ts
View file

@ -44,6 +44,7 @@ export const searchAfterAndBulkCreate = async ({
runtimeMappings,
primaryTimestamp,
secondaryTimestamp,
additionalFilters,
}: SearchAfterAndBulkCreateParams): Promise<SearchAfterAndBulkCreateReturnType> => {
return withSecuritySpan('searchAfterAndBulkCreate', async () => {
let toReturn = createSearchAfterReturnType();
@ -80,6 +81,7 @@ export const searchAfterAndBulkCreate = async ({
secondaryTimestamp,
trackTotalHits,
sortOrder,
additionalFilters,
});
mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]);
toReturn = mergeReturns([

2
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/single_search_after.ts
View file

@ -18,7 +18,7 @@ import type { TimestampOverride } from '../../../../../common/detection_engine/r
import { withSecuritySpan } from '../../../../utils/with_security_span';
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
interface SingleSearchAfterParams {
export interface SingleSearchAfterParams {
aggregations?: Record<string, estypes.AggregationsAggregationContainer>;
searchAfterSortIds: estypes.SortResults | undefined;
index: string[];

615
x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts
View file

@ -24,7 +24,10 @@ import { flattenWithPrefix } from '@kbn/securitysolution-rules';
import { orderBy } from 'lodash';
import { v4 as uuidv4 } from 'uuid';
import { QueryRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema';
import {
QueryRuleCreateProps,
AlertSuppressionMissingFieldsStrategy,
} from '@kbn/security-solution-plugin/common/detection_engine/rule_schema';
import { RuleExecutionStatus } from '@kbn/security-solution-plugin/common/detection_engine/rule_monitoring';
import { Ancestor } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/types';
import {
@ -1434,6 +1437,616 @@ export default ({ getService }: FtrProviderContext) => {
});
});
});
// when missing_fields_strategy set to "doNotSuppress", each document with missing grouped by field
// will generate a separate alert
describe('unsuppressed alerts', () => {
const { indexListOfDocuments, indexGeneratedDocuments } = dataGeneratorFactory({
es,
index: 'ecs_compliant',
log,
});
before(async () => {
await esArchiver.load(
'x-pack/test/functional/es_archives/security_solution/ecs_compliant'
);
});
after(async () => {
await esArchiver.unload(
'x-pack/test/functional/es_archives/security_solution/ecs_compliant'
);
});
it('should create unsuppressed alerts for single host.name field when some of the documents have missing values', async () => {
const id = uuidv4();
const timestamp = '2020-10-28T06:00:00.000Z';
const firstDoc = {
id,
'@timestamp': timestamp,
agent: {
name: 'agent-1',
},
};
const missingFieldDoc = {
id,
'@timestamp': timestamp,
};
await indexListOfDocuments([
firstDoc,
firstDoc,
missingFieldDoc,
missingFieldDoc,
missingFieldDoc,
]);
const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
query: `id:${id}`,
alert_suppression: {
group_by: ['agent.name'],
missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.DoNotSuppress,
},
from: 'now-1h',
interval: '1h',
};
const { previewId } = await previewRule({
supertest,
rule,
timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
});
const previewAlerts = await getPreviewAlerts({
es,
previewId,
size: 10,
sort: ['agent.name'],
});
expect(previewAlerts.length).to.eql(4);
// first alert should be suppressed
expect(previewAlerts[0]._source).to.eql({
...previewAlerts[0]._source,
[ALERT_SUPPRESSION_TERMS]: [
{
field: 'agent.name',
value: 'agent-1',
},
],
[ALERT_SUPPRESSION_START]: timestamp,
[ALERT_SUPPRESSION_END]: timestamp,
[ALERT_ORIGINAL_TIME]: timestamp,
[ALERT_SUPPRESSION_DOCS_COUNT]: 1,
});
// alert is not suppressed and do not have suppress properties
expect(previewAlerts[1]._source).to.have.property('id', id);
expect(previewAlerts[1]._source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
expect(previewAlerts[1]._source).not.to.have.property(ALERT_SUPPRESSION_END);
expect(previewAlerts[1]._source).not.to.have.property(ALERT_SUPPRESSION_TERMS);
expect(previewAlerts[1]._source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
// rest of alerts are not suppressed and do not have suppress properties
previewAlerts.slice(2).forEach((previewAlert) => {
const source = previewAlert._source;
expect(source).to.have.property('id', id);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
expect(source).not.to.have.property(ALERT_SUPPRESSION_END);
expect(source).not.to.have.property(ALERT_SUPPRESSION_TERMS);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
});
});
it('should create unsuppressed alerts for single host.name field when all the documents have missing values', async () => {
const id = uuidv4();
const timestamp = '2020-10-28T06:00:00.000Z';
const missingFieldDoc = {
id,
'@timestamp': timestamp,
};
await indexListOfDocuments([
missingFieldDoc,
missingFieldDoc,
missingFieldDoc,
missingFieldDoc,
]);
const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
query: `id:${id}`,
alert_suppression: {
group_by: ['agent.name'],
missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.DoNotSuppress,
},
from: 'now-1h',
interval: '1h',
};
const { previewId } = await previewRule({
supertest,
rule,
timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
});
const previewAlerts = await getPreviewAlerts({
es,
previewId,
size: 10,
sort: ['agent.name'],
});
expect(previewAlerts.length).to.eql(4);
// alert is not suppressed and do not have suppress properties
expect(previewAlerts[1]._source).to.have.property('id', id);
expect(previewAlerts[1]._source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
expect(previewAlerts[1]._source).not.to.have.property(ALERT_SUPPRESSION_END);
expect(previewAlerts[1]._source).not.to.have.property(ALERT_SUPPRESSION_TERMS);
expect(previewAlerts[1]._source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
// rest of alerts are not suppressed and do not have suppress properties
previewAlerts.slice(2).forEach((previewAlert) => {
const source = previewAlert._source;
expect(source).to.have.property('id', id);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
expect(source).not.to.have.property(ALERT_SUPPRESSION_END);
expect(source).not.to.have.property(ALERT_SUPPRESSION_TERMS);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
});
});
it('should not create unsuppressed alerts if documents are not missing fields', async () => {
const id = uuidv4();
const timestamp = '2020-10-28T06:00:00.000Z';
const firstDoc = {
id,
'@timestamp': timestamp,
agent: {
name: 'agent-1',
},
};
await indexListOfDocuments([firstDoc, firstDoc]);
const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
query: `id:${id}`,
alert_suppression: {
group_by: ['agent.name'],
missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.DoNotSuppress,
},
from: 'now-1h',
interval: '1h',
};
const { previewId } = await previewRule({
supertest,
rule,
timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
});
const previewAlerts = await getPreviewAlerts({
es,
previewId,
size: 10,
sort: ['agent.name'],
});
expect(previewAlerts.length).to.eql(1);
// first alert should be suppressed
expect(previewAlerts[0]._source).to.eql({
...previewAlerts[0]._source,
[ALERT_SUPPRESSION_TERMS]: [
{
field: 'agent.name',
value: 'agent-1',
},
],
[ALERT_SUPPRESSION_START]: timestamp,
[ALERT_SUPPRESSION_END]: timestamp,
[ALERT_ORIGINAL_TIME]: timestamp,
[ALERT_SUPPRESSION_DOCS_COUNT]: 1,
});
});
it('should create no more than max_signals unsuppressed alerts', async () => {
const id = uuidv4();
const timestamp = '2020-10-28T06:00:00.000Z';
const firstDoc = { id, '@timestamp': timestamp, agent: { name: 'agent-0' } };
await indexGeneratedDocuments({
docsCount: 150,
seed: () => ({
id,
'@timestamp': timestamp,
}),
});
await indexListOfDocuments([firstDoc, firstDoc]);
const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
query: `id:${id}`,
alert_suppression: {
group_by: ['agent.name'],
missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.DoNotSuppress,
},
from: 'now-1h',
interval: '1h',
max_signals: 100,
};
const { previewId } = await previewRule({
supertest,
rule,
timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
});
const previewAlerts = await getPreviewAlerts({
es,
previewId,
size: 200,
sort: ['agent.name'],
});
// alerts number should be still at 100
expect(previewAlerts.length).to.eql(100);
// first alert should be suppressed
expect(previewAlerts[0]._source).to.eql({
...previewAlerts[0]._source,
[ALERT_SUPPRESSION_TERMS]: [
{
field: 'agent.name',
value: 'agent-0',
},
],
[ALERT_SUPPRESSION_START]: timestamp,
[ALERT_SUPPRESSION_END]: timestamp,
[ALERT_ORIGINAL_TIME]: timestamp,
[ALERT_SUPPRESSION_DOCS_COUNT]: 1,
});
// rest of alerts are not suppressed and do not have suppress properties
previewAlerts.slice(1).forEach((previewAlert) => {
const source = previewAlert._source;
expect(source).to.have.property('id', id);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
expect(source).not.to.have.property(ALERT_SUPPRESSION_END);
expect(source).not.to.have.property(ALERT_SUPPRESSION_TERMS);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
});
});
it('should create unsuppressed alerts for multiple fields when ony some of the documents have missing values', async () => {
const id = uuidv4();
const timestamp = '2020-10-28T06:00:00.000Z';
const firstDoc = {
id,
'@timestamp': timestamp,
agent: { name: 'agent-0', version: 'filebeat' },
};
const secondDoc = {
id,
'@timestamp': timestamp,
agent: { name: 'agent-0', version: 'auditbeat' },
};
const thirdDoc = {
id,
'@timestamp': timestamp,
agent: { name: 'agent-1', version: 'filebeat' },
};
const missingFieldDoc1 = {
id,
'@timestamp': timestamp,
agent: { name: 'agent-2' },
};
const missingFieldDoc2 = {
id,
'@timestamp': timestamp,
agent: { version: 'filebeat' },
};
const missingAllFieldsDoc = {
id,
'@timestamp': timestamp,
};
await indexListOfDocuments([
firstDoc,
secondDoc,
secondDoc,
thirdDoc,
thirdDoc,
thirdDoc,
missingFieldDoc1,
missingFieldDoc2,
missingFieldDoc2,
missingAllFieldsDoc,
missingAllFieldsDoc,
]);
const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
query: `id:${id}`,
alert_suppression: {
group_by: ['agent.name', 'agent.version'],
missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.DoNotSuppress,
},
from: 'now-1h',
interval: '1h',
};
const { previewId } = await previewRule({
supertest,
rule,
timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
});
const previewAlerts = await getPreviewAlerts({
es,
previewId,
size: 10,
sort: ['agent.name', 'agent.version'],
});
// total 8 alerts = 3 suppressed alerts + 5 unsuppressed (from docs with at least one missing field)
expect(previewAlerts.length).to.eql(8);
// first 3 alerts should be suppressed
expect(previewAlerts[0]._source).to.eql({
...previewAlerts[0]._source,
[ALERT_SUPPRESSION_TERMS]: [
{ field: 'agent.name', value: 'agent-0' },
{ field: 'agent.version', value: 'auditbeat' },
],
[ALERT_SUPPRESSION_DOCS_COUNT]: 1,
});
expect(previewAlerts[1]._source).to.eql({
...previewAlerts[1]._source,
[ALERT_SUPPRESSION_TERMS]: [
{ field: 'agent.name', value: 'agent-0' },
{ field: 'agent.version', value: 'filebeat' },
],
[ALERT_SUPPRESSION_DOCS_COUNT]: 0,
});
expect(previewAlerts[2]._source).to.eql({
...previewAlerts[2]._source,
[ALERT_SUPPRESSION_TERMS]: [
{ field: 'agent.name', value: 'agent-1' },
{ field: 'agent.version', value: 'filebeat' },
],
[ALERT_SUPPRESSION_DOCS_COUNT]: 2,
});
// rest of alerts are not suppressed and do not have suppress properties
previewAlerts.slice(3).forEach((previewAlert) => {
const source = previewAlert._source;
expect(source).to.have.property('id', id);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
expect(source).not.to.have.property(ALERT_SUPPRESSION_END);
expect(source).not.to.have.property(ALERT_SUPPRESSION_TERMS);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
});
});
it('should create unsuppressed alerts for multiple fields when all the documents have missing values', async () => {
const id = uuidv4();
const timestamp = '2020-10-28T06:00:00.000Z';
const missingFieldDoc1 = { id, '@timestamp': timestamp, agent: { name: 'agent-2' } };
const missingFieldDoc2 = { id, '@timestamp': timestamp, agent: { version: 'filebeat' } };
const missingAllFieldsDoc = { id, '@timestamp': timestamp };
await indexListOfDocuments([
missingFieldDoc1,
missingFieldDoc2,
missingFieldDoc2,
missingAllFieldsDoc,
missingAllFieldsDoc,
missingAllFieldsDoc,
]);
const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
query: `id:${id}`,
alert_suppression: {
group_by: ['agent.name', 'agent.version'],
missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.DoNotSuppress,
},
from: 'now-1h',
interval: '1h',
};
const { previewId } = await previewRule({
supertest,
rule,
timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
});
const previewAlerts = await getPreviewAlerts({
es,
previewId,
size: 10,
sort: ['agent.name', 'agent.version'],
});
// total 6 alerts = 6 unsuppressed (from docs with at least one missing field)
expect(previewAlerts.length).to.eql(6);
// all alerts are not suppressed and do not have suppress properties
previewAlerts.forEach((previewAlert) => {
const source = previewAlert._source;
expect(source).to.have.property('id', id);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
expect(source).not.to.have.property(ALERT_SUPPRESSION_END);
expect(source).not.to.have.property(ALERT_SUPPRESSION_TERMS);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
});
});
// following 2 tests created to show the difference in 2 modes, using the same data and using suppression time window
// rule will be executing 2 times and will create alert during both executions, that will be suppressed according to time window
describe('with a suppression time window', async () => {
let id: string;
const timestamp = '2020-10-28T06:00:00.000Z';
const laterTimestamp = '2020-10-28T07:00:00.000Z';
beforeEach(async () => {
id = uuidv4();
// we have 3 kind of documents here:
// 1. agent.name: 2 docs with agent-1 in both rule runs
// 2. agent.name: 2 docs with agent-2 in second rule run only
// 3. agent.name: 3 docs with undefined in both rule runs
// if suppress on missing = true - we'll get 3 suppressed alerts
// if suppress on missing = false - we'll get 2 suppressed alerts and 3 unsuppressed for missing agent.name
const firstDoc = { id, '@timestamp': timestamp, agent: { name: 'agent-1' } };
const secondDoc = { id, '@timestamp': laterTimestamp, agent: { name: 'agent-1' } };
const thirdDoc = { id, '@timestamp': laterTimestamp, agent: { name: 'agent-2' } };
const missingFieldDoc1 = { id, '@timestamp': laterTimestamp };
const missingFieldDoc2 = { id, '@timestamp': laterTimestamp };
await indexListOfDocuments([
firstDoc,
secondDoc,
thirdDoc,
thirdDoc,
missingFieldDoc1,
missingFieldDoc1,
missingFieldDoc2,
]);
});
it('should create suppressed alerts for single host.name when rule configure with suppress', async () => {
const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
query: `id:${id}`,
alert_suppression: {
group_by: ['agent.name'],
duration: {
value: 300,
unit: 'm',
},
missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.Suppress,
},
from: 'now-1h',
interval: '1h',
timestamp_override: 'event.ingested',
};
// checking first default suppress mode
const { previewId, logs } = await previewRule({
supertest,
rule,
timeframeEnd: new Date('2020-10-28T07:30:00.000Z'),
invocationCount: 2,
});
const previewAlerts = await getPreviewAlerts({
es,
previewId,
size: 10,
sort: ['agent.name', ALERT_ORIGINAL_TIME],
});
expect(previewAlerts.length).to.eql(3);
expect(previewAlerts[0]._source).to.eql({
...previewAlerts[0]._source,
[ALERT_SUPPRESSION_TERMS]: [
{
field: 'agent.name',
value: 'agent-1',
},
],
[ALERT_SUPPRESSION_DOCS_COUNT]: 1,
});
expect(previewAlerts[1]._source).to.eql({
...previewAlerts[1]._source,
[ALERT_SUPPRESSION_TERMS]: [
{
field: 'agent.name',
value: 'agent-2',
},
],
[ALERT_SUPPRESSION_DOCS_COUNT]: 1,
});
expect(previewAlerts[2]._source).to.eql({
...previewAlerts[2]._source,
[ALERT_SUPPRESSION_TERMS]: [
{
field: 'agent.name',
value: null,
},
],
[ALERT_SUPPRESSION_DOCS_COUNT]: 2,
});
for (const logEntry of logs) {
expect(logEntry.errors.length).to.eql(0);
}
});
it('should create unsuppressed alerts for single host.name', async () => {
const rule: QueryRuleCreateProps = {
...getRuleForSignalTesting(['ecs_compliant']),
query: `id:${id}`,
alert_suppression: {
group_by: ['agent.name'],
duration: {
value: 300,
unit: 'm',
},
missing_fields_strategy: AlertSuppressionMissingFieldsStrategy.DoNotSuppress,
},
from: 'now-1h',
interval: '1h',
timestamp_override: 'event.ingested',
};
const { previewId, logs } = await previewRule({
supertest,
rule,
timeframeEnd: new Date('2020-10-28T07:30:00.000Z'),
invocationCount: 2,
});
const previewAlerts = await getPreviewAlerts({
es,
previewId,
size: 10,
sort: ['agent.name', ALERT_ORIGINAL_TIME],
});
expect(previewAlerts.length).to.eql(5);
// first alerts expected to be suppressed
expect(previewAlerts[0]._source).to.eql({
...previewAlerts[0]._source,
[ALERT_SUPPRESSION_TERMS]: [
{
field: 'agent.name',
value: 'agent-1',
},
],
[ALERT_SUPPRESSION_DOCS_COUNT]: 1,
});
expect(previewAlerts[1]._source).to.eql({
...previewAlerts[1]._source,
[ALERT_SUPPRESSION_TERMS]: [
{
field: 'agent.name',
value: 'agent-2',
},
],
[ALERT_SUPPRESSION_DOCS_COUNT]: 1,
});
// third alert is not suppressed and do not have suppress properties
expect(previewAlerts[2]._source).to.have.property('id', id);
expect(previewAlerts[2]._source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
expect(previewAlerts[2]._source).not.to.have.property(ALERT_SUPPRESSION_END);
expect(previewAlerts[2]._source).not.to.have.property(ALERT_SUPPRESSION_TERMS);
expect(previewAlerts[2]._source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
// rest of alerts are not suppressed and do not have suppress properties
previewAlerts.slice(3).forEach((previewAlert) => {
const source = previewAlert._source;
expect(source).to.have.property('id', id);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
expect(source).not.to.have.property(ALERT_SUPPRESSION_END);
expect(source).not.to.have.property(ALERT_SUPPRESSION_TERMS);
expect(source).not.to.have.property(ALERT_SUPPRESSION_DOCS_COUNT);
});
for (const logEntry of logs) {
expect(logEntry.errors.length).to.eql(0);
}
});
});
});
});
describe('with exceptions', async () => {

3
x-pack/test/functional/es_archives/security_solution/ecs_compliant/mappings.json
View file

@ -15,6 +15,9 @@
"name": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"type": {
"type": "long"
}