github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[AO] Save group information in AAD for the new threshold rule (#164087)

Browse source 
Closes #161758

## Summary

In this PR, I am saving the groupings information for the new threshold
in AAD in a similar format as the security team does, you can check the
format in the following screenshots. (Please check this
[RFC](https://docs.google.com/document/d/1DlykydM8Hk7-VAPOcuoUXp0L_qSi2jCZabJkPdO44tQ/edit#heading=h.2b1v1tr0ep8m)
for more information)

### Alert as data document


![image](ce4d5000-3799-4dd7-9a04-d012f1cc5aca)

### Groupings action variable


![image](5a4aaff1-b9c5-44e8-86e5-9fa397b6af62)

### Alert table


![image](cfe1aaf1-475c-4d04-8726-b064c0905d55)

It is also possible to search based on these new variables:


f07b39c2-52e8-4f50-b713-577da7ab1c42
This commit is contained in:
Maryam Saeidi 2023-10-02 15:42:35 +02:00 • committed by GitHub
parent 8fd6dbed55
commit 41e54e7208
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 171 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

18
packages/kbn-alerts-as-data-utils/src/field_maps/legacy_experimental_field_map.ts
View file

@ -11,6 +11,9 @@ import {
ALERT_EVALUATION_THRESHOLD,
ALERT_EVALUATION_VALUE,
ALERT_EVALUATION_VALUES,
ALERT_GROUP,
ALERT_GROUP_FIELD,
ALERT_GROUP_VALUE,
} from '@kbn/rule-data-utils';
export const legacyExperimentalFieldMap = {
@ -27,6 +30,21 @@ export const legacyExperimentalFieldMap = {
required: false,
array: true,
},
[ALERT_GROUP]: {
type: 'object',
array: true,
required: false,
},
[ALERT_GROUP_FIELD]: {
type: 'keyword',
array: false,
required: false,
},
[ALERT_GROUP_VALUE]: {
type: 'keyword',
array: false,
required: false,
},
} as const;
export type ExperimentalRuleFieldMap = typeof legacyExperimentalFieldMap;

6
packages/kbn-alerts-as-data-utils/src/schemas/generated/observability_apm_schema.ts
View file

@ -84,6 +84,12 @@ const ObservabilityApmAlertOptional = rt.partial({
value: schemaStringOrNumber,
values: schemaStringOrNumberArray,
}),
group: rt.array(
rt.partial({
field: schemaString,
value: schemaString,
})
),
}),
}),
processor: rt.partial({

6
packages/kbn-alerts-as-data-utils/src/schemas/generated/observability_logs_schema.ts
View file

@ -78,6 +78,12 @@ const ObservabilityLogsAlertOptional = rt.partial({
value: schemaStringOrNumber,
values: schemaStringOrNumberArray,
}),
group: rt.array(
rt.partial({
field: schemaString,
value: schemaString,
})
),
}),
}),
});

6
packages/kbn-alerts-as-data-utils/src/schemas/generated/observability_metrics_schema.ts
View file

@ -78,6 +78,12 @@ const ObservabilityMetricsAlertOptional = rt.partial({
value: schemaStringOrNumber,
values: schemaStringOrNumberArray,
}),
group: rt.array(
rt.partial({
field: schemaString,
value: schemaString,
})
),
}),
}),
});

6
packages/kbn-alerts-as-data-utils/src/schemas/generated/observability_slo_schema.ts
View file

@ -77,6 +77,12 @@ const ObservabilitySloAlertOptional = rt.partial({
value: schemaStringOrNumber,
values: schemaStringOrNumberArray,
}),
group: rt.array(
rt.partial({
field: schemaString,
value: schemaString,
})
),
}),
}),
slo: rt.partial({

6
packages/kbn-alerts-as-data-utils/src/schemas/generated/observability_uptime_schema.ts
View file

@ -89,6 +89,12 @@ const ObservabilityUptimeAlertOptional = rt.partial({
value: schemaStringOrNumber,
values: schemaStringOrNumberArray,
}),
group: rt.array(
rt.partial({
field: schemaString,
value: schemaString,
})
),
}),
}),
monitor: rt.partial({

9
packages/kbn-rule-data-utils/src/technical_field_names.ts
View file

@ -88,6 +88,9 @@ const ALERT_EVALUATION_THRESHOLD = `${ALERT_NAMESPACE}.evaluation.threshold` as
const ALERT_EVALUATION_VALUE = `${ALERT_NAMESPACE}.evaluation.value` as const;
const ALERT_CONTEXT = `${ALERT_NAMESPACE}.context` as const;
const ALERT_EVALUATION_VALUES = `${ALERT_NAMESPACE}.evaluation.values` as const;
const ALERT_GROUP = `${ALERT_NAMESPACE}.group` as const;
const ALERT_GROUP_FIELD = `${ALERT_GROUP}.field` as const;
const ALERT_GROUP_VALUE = `${ALERT_GROUP}.value` as const;
// Fields pertaining to the rule associated with the alert
const ALERT_RULE_EXCEPTIONS_LIST = `${ALERT_RULE_NAMESPACE}.exceptions_list` as const;
@ -129,6 +132,9 @@ const fields = {
ALERT_EVALUATION_THRESHOLD,
ALERT_EVALUATION_VALUE,
ALERT_EVALUATION_VALUES,
ALERT_GROUP,
ALERT_GROUP_FIELD,
ALERT_GROUP_VALUE,
ALERT_FLAPPING,
ALERT_MAINTENANCE_WINDOW_IDS,
ALERT_INSTANCE_ID,
@ -200,6 +206,9 @@ export {
ALERT_EVALUATION_VALUE,
ALERT_CONTEXT,
ALERT_EVALUATION_VALUES,
ALERT_GROUP,
ALERT_GROUP_FIELD,
ALERT_GROUP_VALUE,
ALERT_RULE_EXCEPTIONS_LIST,
ALERT_RULE_NAMESPACE_FIELD,
ALERT_THREAT_FRAMEWORK,

8
x-pack/plugins/observability/server/lib/rules/custom_threshold/custom_threshold_executor.test.ts
View file

@ -389,8 +389,12 @@ describe('The metric threshold alert type', () => {
},
]);
await execute(Comparator.GT, [0.75]);
expect(mostRecentAction(instanceIdA).action.group).toEqual({ groupByField: 'a' });
expect(mostRecentAction(instanceIdB).action.group).toEqual({ groupByField: 'b' });
expect(mostRecentAction(instanceIdA).action.group).toEqual([
{ field: 'groupByField', value: 'a' },
]);
expect(mostRecentAction(instanceIdB).action.group).toEqual([
{ field: 'groupByField', value: 'b' },
]);
});
test('persists previous groups that go missing, until the groupBy param changes', async () => {
setEvaluationResults([

28
x-pack/plugins/observability/server/lib/rules/custom_threshold/custom_threshold_executor.ts
View file

@ -8,7 +8,12 @@
import { isEqual } from 'lodash';
import { TypeOf } from '@kbn/config-schema';
import { i18n } from '@kbn/i18n';
import { ALERT_ACTION_GROUP, ALERT_EVALUATION_VALUES, ALERT_REASON } from '@kbn/rule-data-utils';
import {
ALERT_ACTION_GROUP,
ALERT_EVALUATION_VALUES,
ALERT_REASON,
ALERT_GROUP,
} from '@kbn/rule-data-utils';
import { LocatorPublic } from '@kbn/share-plugin/common';
import {
ActionGroupIdsOf,
@ -37,7 +42,7 @@ import {
hasAdditionalContext,
validGroupByForContext,
flattenAdditionalContext,
getGroupByObject,
getFormattedGroupBy,
} from './utils';
import { EvaluatedRuleParams, evaluateRule } from './lib/evaluate_rule';
@ -80,12 +85,18 @@ type MetricThresholdAlert = Alert<
MetricThresholdAllowedActionGroups
>;
export type Group = Array<{
field: string;
value: string;
}>;
type MetricThresholdAlertFactory = (
id: string,
reason: string,
actionGroup: MetricThresholdActionGroup,
additionalContext?: AdditionalContext | null,
evaluationValues?: Array<number | null>
evaluationValues?: Array<number | null>,
group?: Group
) => MetricThresholdAlert;
export const createMetricThresholdExecutor = ({
@ -139,7 +150,8 @@ export const createMetricThresholdExecutor = ({
reason,
actionGroup,
additionalContext,
evaluationValues
evaluationValues,
group
) =>
alertWithLifecycle({
id,
@ -147,6 +159,7 @@ export const createMetricThresholdExecutor = ({
[ALERT_REASON]: reason,
[ALERT_ACTION_GROUP]: actionGroup,
[ALERT_EVALUATION_VALUES]: evaluationValues,
[ALERT_GROUP]: group,
...flattenAdditionalContext(additionalContext),
},
});
@ -198,7 +211,7 @@ export const createMetricThresholdExecutor = ({
}
}
const groupByKeysObjectMapping = getGroupByObject(params.groupBy, resultGroupSet);
const groupByKeysObjectMapping = getFormattedGroupBy(params.groupBy, resultGroupSet);
const groups = [...resultGroupSet];
const nextMissingGroups = new Set<MissingGroupsRecord>();
const hasGroups = !isEqual(groups, [UNGROUPED_FACTORY_KEY]);
@ -292,7 +305,8 @@ export const createMetricThresholdExecutor = ({
reason,
actionGroupId,
additionalContext,
evaluationValues
evaluationValues,
groupByKeysObjectMapping[group]
);
const alertUuid = getAlertUuid(group);
const indexedStartedAt = getAlertStartedDate(group) ?? startedAt.toISOString();
@ -325,7 +339,7 @@ export const createMetricThresholdExecutor = ({
const { getRecoveredAlerts } = services.alertFactory.done();
const recoveredAlerts = getRecoveredAlerts();
const groupByKeysObjectForRecovered = getGroupByObject(
const groupByKeysObjectForRecovered = getFormattedGroupBy(
params.groupBy,
new Set<string>(recoveredAlerts.map((recoveredAlert) => recoveredAlert.getId()))
);

2
x-pack/plugins/observability/server/lib/rules/custom_threshold/register_custom_threshold_rule_type.ts
View file

@ -147,7 +147,7 @@ export function thresholdRuleType(
doesSetRecoveryContext: true,
actionVariables: {
context: [
{ name: 'groupings', description: groupByKeysActionVariableDescription },
{ name: 'group', description: groupByKeysActionVariableDescription },
{
name: 'alertDetailsUrl',
description: alertDetailUrlActionVariableDescription,

47
x-pack/plugins/observability/server/lib/rules/custom_threshold/utils.test.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { flattenObject, validateKQLStringFilter } from './utils';
import { flattenObject, getFormattedGroupBy, validateKQLStringFilter } from './utils';
describe('FlattenObject', () => {
it('flattens multi level item', () => {
@ -69,3 +69,48 @@ describe('validateKQLStringFilter', () => {
expect(validateKQLStringFilter(input)).toEqual(output);
});
});
describe('getFormattedGroupBy', () => {
it('should format groupBy correctly for empty input', () => {
expect(getFormattedGroupBy(undefined, new Set<string>())).toEqual({});
});
it('should format groupBy correctly for multiple groups', () => {
expect(
getFormattedGroupBy(
['host.name', 'host.mac', 'tags', 'container.name'],
new Set([
'host-0,00-00-5E-00-53-23,event-0,container-name',
'host-0,00-00-5E-00-53-23,group-0,container-name',
'host-0,00-00-5E-00-53-24,event-0,container-name',
'host-0,00-00-5E-00-53-24,group-0,container-name',
])
)
).toEqual({
'host-0,00-00-5E-00-53-23,event-0,container-name': [
{ field: 'host.name', value: 'host-0' },
{ field: 'host.mac', value: '00-00-5E-00-53-23' },
{ field: 'tags', value: 'event-0' },
{ field: 'container.name', value: 'container-name' },
],
'host-0,00-00-5E-00-53-23,group-0,container-name': [
{ field: 'host.name', value: 'host-0' },
{ field: 'host.mac', value: '00-00-5E-00-53-23' },
{ field: 'tags', value: 'group-0' },
{ field: 'container.name', value: 'container-name' },
],
'host-0,00-00-5E-00-53-24,event-0,container-name': [
{ field: 'host.name', value: 'host-0' },
{ field: 'host.mac', value: '00-00-5E-00-53-24' },
{ field: 'tags', value: 'event-0' },
{ field: 'container.name', value: 'container-name' },
],
'host-0,00-00-5E-00-53-24,group-0,container-name': [
{ field: 'host.name', value: 'host-0' },
{ field: 'host.mac', value: '00-00-5E-00-53-24' },
{ field: 'tags', value: 'group-0' },
{ field: 'container.name', value: 'container-name' },
],
});
});
});

26
x-pack/plugins/observability/server/lib/rules/custom_threshold/utils.ts
View file

@ -16,6 +16,7 @@ import { ES_FIELD_TYPES } from '@kbn/field-types';
import { set } from '@kbn/safer-lodash-set';
import { ParsedExperimentalFields } from '@kbn/rule-registry-plugin/common/parse_experimental_fields';
import { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common';
import { Group } from './custom_threshold_executor';
import { ObservabilityConfig } from '../../..';
import { AlertExecutionDetails } from './types';
@ -227,21 +228,20 @@ export const flattenObject = (obj: AdditionalContext, prefix: string = ''): Addi
return acc;
}, {});
export const getGroupByObject = (
export const getFormattedGroupBy = (
groupBy: string | string[] | undefined,
resultGroupSet: Set<string>
): Record<string, object> => {
const groupByKeysObjectMapping: Record<string, object> = {};
groupSet: Set<string>
): Record<string, Group> => {
const groupByKeysObjectMapping: Record<string, Group> = {};
if (groupBy) {
resultGroupSet.forEach((groupSet) => {
const groupSetKeys = groupSet.split(',');
groupByKeysObjectMapping[groupSet] = unflattenObject(
Array.isArray(groupBy)
? groupBy.reduce((result, group, index) => {
return { ...result, [group]: groupSetKeys[index]?.trim() };
}, {})
: { [groupBy]: groupSet }
);
groupSet.forEach((group) => {
const groupSetKeys = group.split(',');
groupByKeysObjectMapping[group] = Array.isArray(groupBy)
? groupBy.reduce((result: Group, groupByItem, index) => {
result.push({ field: groupByItem, value: groupSetKeys[index]?.trim() });
return result;
}, [])
: [{ field: groupBy, value: group }];
});
}
return groupByKeysObjectMapping;

2
x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts
View file

@ -163,6 +163,8 @@ export default function ({ getService }: FtrProviderContext) {
expect(resp.hits.hits[0]._source).property('event.kind', 'signal');
expect(resp.hits.hits[0]._source).property('event.action', 'open');
expect(resp.hits.hits[0]._source).not.have.property('kibana.alert.group');
expect(resp.hits.hits[0]._source)
.property('kibana.alert.rule.parameters')
.eql({

29
x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts
View file

@ -111,7 +111,7 @@ export default function ({ getService }: FtrProviderContext) {
},
index: DATA_VIEW_ID,
},
groupBy: ['host.name'],
groupBy: ['host.name', 'container.id'],
},
actions: [
{
@ -125,6 +125,7 @@ export default function ({ getService }: FtrProviderContext) {
reason: '{{context.reason}}',
value: '{{context.value}}',
host: '{{context.host}}',
group: '{{context.group}}',
},
],
},
@ -180,7 +181,10 @@ export default function ({ getService }: FtrProviderContext) {
'custom_threshold.fired'
);
expect(resp.hits.hits[0]._source).property('tags').contain('observability');
expect(resp.hits.hits[0]._source).property('kibana.alert.instance.id', 'host-0');
expect(resp.hits.hits[0]._source).property(
'kibana.alert.instance.id',
'host-0,container-0'
);
expect(resp.hits.hits[0]._source).property('kibana.alert.workflow_status', 'open');
expect(resp.hits.hits[0]._source).property('event.kind', 'signal');
expect(resp.hits.hits[0]._source).property('event.action', 'open');
@ -193,6 +197,19 @@ export default function ({ getService }: FtrProviderContext) {
expect(resp.hits.hits[0]._source).property('container.name', 'container-name');
expect(resp.hits.hits[0]._source).not.property('container.cpu');
expect(resp.hits.hits[0]._source)
.property('kibana.alert.group')
.eql([
{
field: 'host.name',
value: 'host-0',
},
{
field: 'container.id',
value: 'container-0',
},
]);
expect(resp.hits.hits[0]._source)
.property('kibana.alert.rule.parameters')
.eql({
@ -209,7 +226,7 @@ export default function ({ getService }: FtrProviderContext) {
alertOnNoData: true,
alertOnGroupDisappear: true,
searchConfiguration: { index: 'data-view-id', query: { query: '', language: 'kuery' } },
groupBy: ['host.name'],
groupBy: ['host.name', 'container.id'],
});
});
@ -221,6 +238,7 @@ export default function ({ getService }: FtrProviderContext) {
reason: string;
value: string;
host: string;
group: string;
}>({
esClient,
indexName: ALERT_ACTION_INDEX,
@ -231,12 +249,15 @@ export default function ({ getService }: FtrProviderContext) {
`https://localhost:5601/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)`
);
expect(resp.hits.hits[0]._source?.reason).eql(
'Custom equation is 0.8 in the last 1 min for host-0. Alert when >= 0.2.'
'Custom equation is 0.8 in the last 1 min for host-0,container-0. Alert when >= 0.2.'
);
expect(resp.hits.hits[0]._source?.value).eql('0.8');
expect(resp.hits.hits[0]._source?.host).eql(
'{"name":"host-0","mac":["00-00-5E-00-53-23","00-00-5E-00-53-24"]}'
);
expect(resp.hits.hits[0]._source?.group).eql(
'{"field":"host.name","value":"host-0"},{"field":"container.id","value":"container-0"}'
);
});
});
});