github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[RAM][Maintenance Window][8.8]Fix window maintenance workflow (#156427)

Browse source 
## Summary

The way that we canceled every notification for our alert life cycle
during an active maintenance window was not close enough to what our
customers were expecting. For our persisted security solution alerts, we
did not have to change the logic because it will always be a new alert.
Therefore, @shanisagiv1, @mdefazio, @JiaweiWu, and @XavierM had a
discussion about this problem and we decided this:

To summarize, we will only keep the notification during a maintenance
window if an alert has been created/active outside of window
maintenance. We created three different scenarios to explain the new
logic and we will make the assumption that our alert has an action per
status change. For you to understand the different scenarios, I created
this legend below:
<img width="223" alt="image"
src="https://user-images.githubusercontent.com/189600/236045974-f4fa379b-db5e-41f8-91a8-2689b9f24dab.png">

### Scenario I
If an alert is active/created before a maintenance window and recovered
inside of the maintenance window then we will send notifications
<img width="463" alt="image"
src="https://user-images.githubusercontent.com/189600/236046473-d04df836-d3e6-42d8-97be-8b4f1544cc1a.png">

### Scenario II
If an alert is active/created and recovered inside of window maintenance
then we will NOT send notifications
<img width="407" alt="image"
src="https://user-images.githubusercontent.com/189600/236046913-c2f77131-9ff1-4864-9dab-89c4c429152e.png">

### Scenario III
if an alert is active/created in a maintenance window and recovered
outside of the maintenance window then we will not send notifications
<img width="496" alt="image"
src="https://user-images.githubusercontent.com/189600/236047613-e63efe52-87fa-419e-9e0e-965b1d10ae18.png">


### Checklist
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Xavier Mouligneau <xavier.mouligneau@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Jiawei Wu 2023-05-04 18:11:26 -06:00 • committed by GitHub
parent a83ab21783
commit ea407983bb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 542 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

1
x-pack/packages/kbn-alerting-state-types/src/alert_instance.ts
View file

@ -36,6 +36,7 @@ const metaSchema = t.partial({
flappingHistory: t.array(t.boolean),
// flapping flag that indicates whether the alert is flapping
flapping: t.boolean,
maintenanceWindowIds: t.array(t.string),
pendingRecoveredCount: t.number,
uuid: t.string,
});

8
x-pack/plugins/alerting/server/alert/alert.test.ts
View file

@ -344,6 +344,7 @@ describe('updateLastScheduledActions()', () => {
group: 'default',
},
flappingHistory: [],
maintenanceWindowIds: [],
},
});
});
@ -357,6 +358,7 @@ describe('updateLastScheduledActions()', () => {
state: {},
meta: {
flappingHistory: [],
maintenanceWindowIds: [],
uuid: expect.any(String),
lastScheduledActions: {
date: new Date().toISOString(),
@ -373,6 +375,7 @@ describe('updateLastScheduledActions()', () => {
const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1', {
meta: {
flappingHistory: [],
maintenanceWindowIds: [],
lastScheduledActions: {
date: new Date(),
group: 'default',
@ -387,6 +390,7 @@ describe('updateLastScheduledActions()', () => {
state: {},
meta: {
flappingHistory: [],
maintenanceWindowIds: [],
uuid: expect.any(String),
lastScheduledActions: {
date: new Date().toISOString(),
@ -484,6 +488,7 @@ describe('toJSON', () => {
group: 'default',
},
flappingHistory: [false, true],
maintenanceWindowIds: [],
flapping: false,
pendingRecoveredCount: 2,
},
@ -548,6 +553,7 @@ describe('toRaw', () => {
meta: {
flappingHistory: [false, true, true],
flapping: false,
maintenanceWindowIds: [],
uuid: expect.any(String),
},
});
@ -570,6 +576,7 @@ describe('setFlappingHistory', () => {
"flappingHistory": Array [
false,
],
"maintenanceWindowIds": Array [],
"uuid": Any<String>,
},
"state": Object {},
@ -602,6 +609,7 @@ describe('setFlapping', () => {
"meta": Object {
"flapping": false,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"uuid": Any<String>,
},
"state": Object {},

11
x-pack/plugins/alerting/server/alert/alert.ts
View file

@ -63,7 +63,7 @@ export class Alert<
this.context = {} as Context;
this.meta = meta;
this.meta.uuid = meta.uuid ?? uuidV4();
this.meta.maintenanceWindowIds = meta.maintenanceWindowIds ?? [];
if (!this.meta.flappingHistory) {
this.meta.flappingHistory = [];
}
@ -229,6 +229,7 @@ export class Alert<
// for a recovered alert, we only care to track the flappingHistory,
// the flapping flag, and the UUID
meta: {
maintenanceWindowIds: this.meta.maintenanceWindowIds,
flappingHistory: this.meta.flappingHistory,
flapping: this.meta.flapping,
uuid: this.meta.uuid,
@ -296,4 +297,12 @@ export class Alert<
get(alert, ALERT_UUID) === this.getId() || get(alert, ALERT_UUID) === this.getUuid()
);
}
setMaintenanceWindowIds(maintenanceWindowIds: string[] = []) {
this.meta.maintenanceWindowIds = maintenanceWindowIds;
}
getMaintenanceWindowIds() {
return this.meta.maintenanceWindowIds ?? [];
}
}

20
x-pack/plugins/alerting/server/alert/create_alert_factory.test.ts
View file

@ -31,6 +31,7 @@ describe('createAlertFactory()', () => {
logger,
maxAlerts: 1000,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
const result = alertFactory.create('1');
expect(result).toMatchObject({
@ -58,6 +59,7 @@ describe('createAlertFactory()', () => {
logger,
maxAlerts: 1000,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
const result = alertFactory.create('1');
expect(result).toMatchObject({
@ -82,6 +84,7 @@ describe('createAlertFactory()', () => {
logger,
maxAlerts: 1000,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
alertFactory.create('1');
expect(alerts).toMatchObject({
@ -103,6 +106,7 @@ describe('createAlertFactory()', () => {
logger,
maxAlerts: 3,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
expect(alertFactory.hasReachedAlertLimit()).toBe(false);
@ -123,6 +127,7 @@ describe('createAlertFactory()', () => {
logger,
maxAlerts: 1000,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
const result = alertFactory.create('1');
expect(result).toMatchObject({
@ -166,6 +171,7 @@ describe('createAlertFactory()', () => {
canSetRecoveryContext: true,
maxAlerts: 1000,
autoRecoverAlerts: true,
maintenanceWindowIds: ['test-id-1'],
});
const result = alertFactory.create('1');
expect(result).toMatchObject({
@ -184,6 +190,11 @@ describe('createAlertFactory()', () => {
const recoveredAlerts = getRecoveredAlertsFn!();
expect(Array.isArray(recoveredAlerts)).toBe(true);
expect(recoveredAlerts.length).toEqual(2);
expect(processAlerts).toHaveBeenLastCalledWith(
expect.objectContaining({
maintenanceWindowIds: ['test-id-1'],
})
);
});
test('returns empty array if no recovered alerts', () => {
@ -194,6 +205,7 @@ describe('createAlertFactory()', () => {
maxAlerts: 1000,
canSetRecoveryContext: true,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
const result = alertFactory.create('1');
expect(result).toMatchObject({
@ -221,6 +233,7 @@ describe('createAlertFactory()', () => {
maxAlerts: 1000,
canSetRecoveryContext: true,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
const result = alertFactory.create('1');
expect(result).toMatchObject({
@ -247,6 +260,7 @@ describe('createAlertFactory()', () => {
maxAlerts: 1000,
canSetRecoveryContext: false,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
const result = alertFactory.create('1');
expect(result).toMatchObject({
@ -275,6 +289,7 @@ describe('createAlertFactory()', () => {
logger,
maxAlerts: 1000,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
const limit = alertFactory.alertLimit.getValue();
@ -293,6 +308,7 @@ describe('createAlertFactory()', () => {
logger,
maxAlerts: 1000,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
const limit = alertFactory.alertLimit.getValue();
@ -308,6 +324,7 @@ describe('createAlertFactory()', () => {
logger,
maxAlerts: 1000,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
const limit = alertFactory.alertLimit.getValue();
@ -324,11 +341,13 @@ describe('createAlertFactory()', () => {
maxAlerts: 1000,
canSetRecoveryContext: true,
autoRecoverAlerts: false,
maintenanceWindowIds: [],
});
const result = alertFactory.create('1');
expect(result).toEqual({
meta: {
flappingHistory: [],
maintenanceWindowIds: [],
uuid: expect.any(String),
},
state: {},
@ -354,6 +373,7 @@ describe('getPublicAlertFactory', () => {
logger,
maxAlerts: 1000,
autoRecoverAlerts: true,
maintenanceWindowIds: [],
});
expect(alertFactory.create).toBeDefined();

3
x-pack/plugins/alerting/server/alert/create_alert_factory.ts
View file

@ -54,6 +54,7 @@ export interface CreateAlertFactoryOpts<
logger: Logger;
maxAlerts: number;
autoRecoverAlerts: boolean;
maintenanceWindowIds: string[];
canSetRecoveryContext?: boolean;
}
@ -66,6 +67,7 @@ export function createAlertFactory<
logger,
maxAlerts,
autoRecoverAlerts,
maintenanceWindowIds,
canSetRecoveryContext = false,
}: CreateAlertFactoryOpts<State, Context>): AlertFactory<State, Context, ActionGroupIds> {
// Keep track of which alerts we started with so we can determine which have recovered
@ -152,6 +154,7 @@ export function createAlertFactory<
autoRecoverAlerts,
// flappingSettings.enabled is false, as we only want to use this function to get the recovered alerts
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds,
});
return Object.keys(currentRecoveredAlerts ?? {}).map(
(alertId: string) => currentRecoveredAlerts[alertId]

18
x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts
View file

@ -124,7 +124,8 @@ describe('Legacy Alerts Client', () => {
'1': testAlert1,
'2': testAlert2,
},
{}
{},
['test-id-1']
);
expect(createAlertFactory).toHaveBeenCalledWith({
@ -136,6 +137,7 @@ describe('Legacy Alerts Client', () => {
maxAlerts: 1000,
canSetRecoveryContext: false,
autoRecoverAlerts: true,
maintenanceWindowIds: ['test-id-1'],
});
});
@ -151,7 +153,8 @@ describe('Legacy Alerts Client', () => {
'1': testAlert1,
'2': testAlert2,
},
{}
{},
[]
);
alertsClient.getExecutorServices();
@ -170,7 +173,8 @@ describe('Legacy Alerts Client', () => {
'1': testAlert1,
'2': testAlert2,
},
{}
{},
[]
);
alertsClient.checkLimitUsage();
@ -189,7 +193,8 @@ describe('Legacy Alerts Client', () => {
'1': testAlert1,
'2': testAlert2,
},
{}
{},
[]
);
alertsClient.hasReachedAlertLimit();
@ -230,7 +235,8 @@ describe('Legacy Alerts Client', () => {
'1': testAlert1,
'2': testAlert2,
},
{}
{},
[]
);
alertsClient.processAndLogAlerts({
@ -257,6 +263,7 @@ describe('Legacy Alerts Client', () => {
alertLimit: 1000,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: ['window-id1', 'window-id2'],
});
expect(getAlertsForNotification).toHaveBeenCalledWith(
@ -289,7 +296,6 @@ describe('Legacy Alerts Client', () => {
ruleRunMetricsStore,
canSetRecoveryContext: false,
shouldPersistAlerts: true,
maintenanceWindowIds: ['window-id1', 'window-id2'],
});
expect(alertsClient.getProcessedAlerts('active')).toEqual({

8
x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts
View file

@ -75,7 +75,8 @@ export class LegacyAlertsClient<
public initialize(
activeAlertsFromState: Record<string, RawAlertInstance>,
recoveredAlertsFromState: Record<string, RawAlertInstance>
recoveredAlertsFromState: Record<string, RawAlertInstance>,
maintenanceWindowIds: string[]
) {
for (const id in activeAlertsFromState) {
if (activeAlertsFromState.hasOwnProperty(id)) {
@ -107,6 +108,7 @@ export class LegacyAlertsClient<
maxAlerts: this.options.maxAlerts,
autoRecoverAlerts: this.options.ruleType.autoRecoverAlerts ?? true,
canSetRecoveryContext: this.options.ruleType.doesSetRecoveryContext ?? false,
maintenanceWindowIds,
});
}
@ -125,7 +127,7 @@ export class LegacyAlertsClient<
ruleRunMetricsStore: RuleRunMetricsStore;
flappingSettings: RulesSettingsFlappingProperties;
notifyWhen: RuleNotifyWhenType | null;
maintenanceWindowIds?: string[];
maintenanceWindowIds: string[];
}) {
const {
newAlerts: processedAlertsNew,
@ -143,6 +145,7 @@ export class LegacyAlertsClient<
? this.options.ruleType.autoRecoverAlerts
: true,
flappingSettings,
maintenanceWindowIds,
});
const { trimmedAlertsRecovered, earlyRecoveredAlerts } = trimRecoveredAlerts(
@ -178,7 +181,6 @@ export class LegacyAlertsClient<
ruleRunMetricsStore,
canSetRecoveryContext: this.options.ruleType.doesSetRecoveryContext ?? false,
shouldPersistAlerts: shouldLogAndScheduleActionsForAlerts,
maintenanceWindowIds,
});
}

44
x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts
View file

@ -38,6 +38,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": true,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": "uuid-1",
},
@ -51,6 +52,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": true,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": "uuid-1",
},
@ -60,6 +62,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": false,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": "uuid-2",
},
@ -105,6 +108,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": true,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 1,
"uuid": Any<String>,
},
@ -123,18 +127,19 @@ describe('getAlertsForNotification', () => {
]
`);
expect(alertsWithAnyUUID(currentActiveAlerts)).toMatchInlineSnapshot(`
Object {
"3": Object {
"meta": Object {
"flapping": true,
"flappingHistory": Array [],
"pendingRecoveredCount": 1,
"uuid": Any<String>,
},
"state": Object {},
},
}
`);
Object {
"3": Object {
"meta": Object {
"flapping": true,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 1,
"uuid": Any<String>,
},
"state": Object {},
},
}
`);
expect(Object.values(currentActiveAlerts).map((a) => a.getScheduledActionOptions()))
.toMatchInlineSnapshot(`
Array [
@ -151,6 +156,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": true,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": Any<String>,
},
@ -160,6 +166,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": false,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"uuid": Any<String>,
},
"state": Object {},
@ -172,6 +179,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": true,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": Any<String>,
},
@ -181,6 +189,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": false,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"uuid": Any<String>,
},
"state": Object {},
@ -231,6 +240,7 @@ describe('getAlertsForNotification', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": Any<String>,
},
@ -244,6 +254,7 @@ describe('getAlertsForNotification', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": Any<String>,
},
@ -257,6 +268,7 @@ describe('getAlertsForNotification', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": Any<String>,
},
@ -274,6 +286,7 @@ describe('getAlertsForNotification', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": Any<String>,
},
@ -287,6 +300,7 @@ describe('getAlertsForNotification', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": Any<String>,
},
@ -300,6 +314,7 @@ describe('getAlertsForNotification', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": Any<String>,
},
@ -345,6 +360,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": true,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 1,
"uuid": Any<String>,
},
@ -372,6 +388,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": true,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": Any<String>,
},
@ -381,6 +398,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": false,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"uuid": Any<String>,
},
"state": Object {},
@ -393,6 +411,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": true,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"pendingRecoveredCount": 0,
"uuid": Any<String>,
},
@ -402,6 +421,7 @@ describe('getAlertsForNotification', () => {
"meta": Object {
"flapping": false,
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"uuid": Any<String>,
},
"state": Object {},

211
x-pack/plugins/alerting/server/lib/process_alerts.test.ts
View file

@ -12,6 +12,8 @@ import { Alert } from '../alert';
import { AlertInstanceState, AlertInstanceContext } from '../types';
import { DEFAULT_FLAPPING_SETTINGS, DISABLE_FLAPPING_SETTINGS } from '../../common/rules_settings';
const maintenanceWindowIds = ['test-id-1', 'test-id-2'];
describe('processAlerts', () => {
let clock: sinon.SinonFakeTimers;
@ -58,6 +60,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(newAlerts).toEqual({ '1': newAlert });
@ -96,6 +99,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(newAlerts).toEqual({ '1': newAlert1, '2': newAlert2 });
@ -112,6 +116,46 @@ describe('processAlerts', () => {
expect(newAlert1State.end).not.toBeDefined();
expect(newAlert2State.end).not.toBeDefined();
});
test('sets maintenance window IDs in new alert state', () => {
const newAlert1 = new Alert<AlertInstanceState, AlertInstanceContext>('1');
const newAlert2 = new Alert<AlertInstanceState, AlertInstanceContext>('2');
const existingAlert1 = new Alert<AlertInstanceState, AlertInstanceContext>('3');
const existingAlert2 = new Alert<AlertInstanceState, AlertInstanceContext>('4');
const existingAlerts = {
'3': existingAlert1,
'4': existingAlert2,
};
const updatedAlerts = {
...cloneDeep(existingAlerts),
'1': newAlert1,
'2': newAlert2,
};
updatedAlerts['1'].scheduleActions('default' as never, { foo: '1' });
updatedAlerts['2'].scheduleActions('default' as never, { foo: '1' });
updatedAlerts['3'].scheduleActions('default' as never, { foo: '1' });
updatedAlerts['4'].scheduleActions('default' as never, { foo: '2' });
expect(newAlert1.getState()).toStrictEqual({});
expect(newAlert2.getState()).toStrictEqual({});
const { newAlerts } = processAlerts({
alerts: updatedAlerts,
existingAlerts,
previouslyRecoveredAlerts: {},
hasReachedAlertLimit: false,
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds,
});
expect(newAlerts['1'].getMaintenanceWindowIds()).toEqual(maintenanceWindowIds);
expect(newAlerts['2'].getMaintenanceWindowIds()).toEqual(maintenanceWindowIds);
});
});
describe('activeAlerts', () => {
@ -142,6 +186,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toEqual({
@ -180,6 +225,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toEqual({
@ -228,6 +274,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toEqual({
@ -286,6 +333,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toEqual({
@ -347,6 +395,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(
@ -365,6 +414,37 @@ describe('processAlerts', () => {
expect(previouslyRecoveredAlert1State.end).not.toBeDefined();
expect(previouslyRecoveredAlert2State.end).not.toBeDefined();
});
test('should not set maintenance window IDs for active alerts', () => {
const newAlert = new Alert<AlertInstanceState, AlertInstanceContext>('1');
const existingAlert1 = new Alert<AlertInstanceState, AlertInstanceContext>('2');
const existingAlerts = {
'2': existingAlert1,
};
existingAlerts['2'].replaceState({ start: '1969-12-30T00:00:00.000Z', duration: 33000 });
const updatedAlerts = {
...cloneDeep(existingAlerts),
'1': newAlert,
};
updatedAlerts['1'].scheduleActions('default' as never, { foo: '1' });
updatedAlerts['2'].scheduleActions('default' as never, { foo: '1' });
const { activeAlerts } = processAlerts({
alerts: updatedAlerts,
existingAlerts,
previouslyRecoveredAlerts: {},
hasReachedAlertLimit: false,
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds,
});
expect(activeAlerts['2'].getMaintenanceWindowIds()).toEqual([]);
});
});
describe('recoveredAlerts', () => {
@ -390,6 +470,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(recoveredAlerts).toEqual({ '2': updatedAlerts['2'] });
@ -418,6 +499,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(recoveredAlerts).toEqual({});
@ -448,6 +530,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(recoveredAlerts).toEqual({ '2': updatedAlerts['2'], '3': updatedAlerts['3'] });
@ -487,6 +570,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(recoveredAlerts).toEqual({ '2': updatedAlerts['2'], '3': updatedAlerts['3'] });
@ -526,6 +610,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(recoveredAlerts).toEqual(updatedAlerts);
@ -556,10 +641,39 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: false,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(recoveredAlerts).toEqual({});
});
test('should not set maintenance window IDs for recovered alerts', () => {
const activeAlert = new Alert<AlertInstanceState, AlertInstanceContext>('1');
const recoveredAlert1 = new Alert<AlertInstanceState, AlertInstanceContext>('2');
const existingAlerts = {
'1': activeAlert,
'2': recoveredAlert1,
};
existingAlerts['2'].replaceState({ start: '1969-12-30T00:00:00.000Z', duration: 33000 });
const updatedAlerts = cloneDeep(existingAlerts);
updatedAlerts['1'].scheduleActions('default' as never, { foo: '1' });
const { recoveredAlerts } = processAlerts({
alerts: updatedAlerts,
existingAlerts,
previouslyRecoveredAlerts: {},
hasReachedAlertLimit: false,
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds,
});
expect(recoveredAlerts['2'].getMaintenanceWindowIds()).toEqual([]);
});
});
describe('when hasReachedAlertLimit is true', () => {
@ -602,6 +716,7 @@ describe('processAlerts', () => {
alertLimit: 7,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(recoveredAlerts).toEqual({});
@ -638,6 +753,7 @@ describe('processAlerts', () => {
alertLimit: 7,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toEqual({
@ -698,6 +814,7 @@ describe('processAlerts', () => {
alertLimit: MAX_ALERTS,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(Object.keys(activeAlerts).length).toEqual(MAX_ALERTS);
@ -715,6 +832,68 @@ describe('processAlerts', () => {
'7': newAlert7,
});
});
test('should set maintenance window IDs for new alerts when reached alert limit', () => {
const MAX_ALERTS = 7;
const existingAlert1 = new Alert<AlertInstanceState, AlertInstanceContext>('1');
const existingAlert2 = new Alert<AlertInstanceState, AlertInstanceContext>('2');
const existingAlert3 = new Alert<AlertInstanceState, AlertInstanceContext>('3');
const existingAlert4 = new Alert<AlertInstanceState, AlertInstanceContext>('4');
const existingAlert5 = new Alert<AlertInstanceState, AlertInstanceContext>('5');
const newAlert6 = new Alert<AlertInstanceState, AlertInstanceContext>('6');
const newAlert7 = new Alert<AlertInstanceState, AlertInstanceContext>('7');
const newAlert8 = new Alert<AlertInstanceState, AlertInstanceContext>('8');
const newAlert9 = new Alert<AlertInstanceState, AlertInstanceContext>('9');
const newAlert10 = new Alert<AlertInstanceState, AlertInstanceContext>('10');
const existingAlerts = {
'1': existingAlert1,
'2': existingAlert2,
'3': existingAlert3,
'4': existingAlert4,
'5': existingAlert5,
};
const updatedAlerts = {
...cloneDeep(existingAlerts),
'6': newAlert6,
'7': newAlert7,
'8': newAlert8,
'9': newAlert9,
'10': newAlert10,
};
updatedAlerts['1'].scheduleActions('default' as never, { foo: '1' });
updatedAlerts['2'].scheduleActions('default' as never, { foo: '1' });
updatedAlerts['3'].scheduleActions('default' as never, { foo: '2' });
updatedAlerts['4'].scheduleActions('default' as never, { foo: '2' });
// intentionally not scheduling actions for alert "5"
updatedAlerts['6'].scheduleActions('default' as never, { foo: '2' });
updatedAlerts['7'].scheduleActions('default' as never, { foo: '2' });
updatedAlerts['8'].scheduleActions('default' as never, { foo: '2' });
updatedAlerts['9'].scheduleActions('default' as never, { foo: '2' });
updatedAlerts['10'].scheduleActions('default' as never, { foo: '2' });
const { activeAlerts, newAlerts } = processAlerts({
alerts: updatedAlerts,
existingAlerts,
previouslyRecoveredAlerts: {},
hasReachedAlertLimit: true,
alertLimit: MAX_ALERTS,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds,
});
expect(Object.keys(activeAlerts).length).toEqual(MAX_ALERTS);
expect(newAlerts['6'].getMaintenanceWindowIds()).toEqual(maintenanceWindowIds);
expect(newAlerts['7'].getMaintenanceWindowIds()).toEqual(maintenanceWindowIds);
expect(activeAlerts['1'].getMaintenanceWindowIds()).toEqual([]);
expect(activeAlerts['2'].getMaintenanceWindowIds()).toEqual([]);
expect(activeAlerts['3'].getMaintenanceWindowIds()).toEqual([]);
expect(activeAlerts['4'].getMaintenanceWindowIds()).toEqual([]);
expect(activeAlerts['5'].getMaintenanceWindowIds()).toEqual([]);
});
});
describe('updating flappingHistory', () => {
@ -734,6 +913,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toMatchInlineSnapshot(`
@ -743,6 +923,7 @@ describe('processAlerts', () => {
"flappingHistory": Array [
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {
@ -759,6 +940,7 @@ describe('processAlerts', () => {
"flappingHistory": Array [
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {
@ -787,6 +969,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toMatchInlineSnapshot(`
@ -797,6 +980,7 @@ describe('processAlerts', () => {
false,
false,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {},
@ -827,6 +1011,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toMatchInlineSnapshot(`
@ -837,6 +1022,7 @@ describe('processAlerts', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {
@ -854,6 +1040,7 @@ describe('processAlerts', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {
@ -885,6 +1072,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toMatchInlineSnapshot(`Object {}`);
@ -897,6 +1085,7 @@ describe('processAlerts', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {},
@ -920,6 +1109,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toMatchInlineSnapshot(`Object {}`);
@ -932,6 +1122,7 @@ describe('processAlerts', () => {
false,
false,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {},
@ -965,6 +1156,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toMatchInlineSnapshot(`
@ -972,6 +1164,7 @@ describe('processAlerts', () => {
"1": Object {
"meta": Object {
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {
@ -984,6 +1177,7 @@ describe('processAlerts', () => {
"flappingHistory": Array [
false,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-2",
},
"state": Object {},
@ -995,6 +1189,7 @@ describe('processAlerts', () => {
"1": Object {
"meta": Object {
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {
@ -1011,6 +1206,7 @@ describe('processAlerts', () => {
"flappingHistory": Array [
false,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-3",
},
"state": Object {},
@ -1036,6 +1232,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toMatchInlineSnapshot(`
@ -1046,6 +1243,7 @@ describe('processAlerts', () => {
false,
false,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {},
@ -1076,6 +1274,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toMatchInlineSnapshot(`
@ -1086,6 +1285,7 @@ describe('processAlerts', () => {
false,
false,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {},
@ -1096,6 +1296,7 @@ describe('processAlerts', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-2",
},
"state": Object {
@ -1113,6 +1314,7 @@ describe('processAlerts', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-2",
},
"state": Object {
@ -1145,6 +1347,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toMatchInlineSnapshot(`
@ -1155,6 +1358,7 @@ describe('processAlerts', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {
@ -1167,6 +1371,7 @@ describe('processAlerts', () => {
"flappingHistory": Array [
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-2",
},
"state": Object {
@ -1184,6 +1389,7 @@ describe('processAlerts', () => {
false,
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {
@ -1196,6 +1402,7 @@ describe('processAlerts', () => {
"flappingHistory": Array [
true,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-2",
},
"state": Object {
@ -1228,6 +1435,7 @@ describe('processAlerts', () => {
alertLimit: 10,
autoRecoverAlerts: true,
flappingSettings: DISABLE_FLAPPING_SETTINGS,
maintenanceWindowIds: [],
});
expect(activeAlerts).toMatchInlineSnapshot(`
@ -1237,6 +1445,7 @@ describe('processAlerts', () => {
"flappingHistory": Array [
false,
],
"maintenanceWindowIds": Array [],
"uuid": "uuid-1",
},
"state": Object {},
@ -1244,6 +1453,7 @@ describe('processAlerts', () => {
"2": Object {
"meta": Object {
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"uuid": "uuid-2",
},
"state": Object {
@ -1258,6 +1468,7 @@ describe('processAlerts', () => {
"2": Object {
"meta": Object {
"flappingHistory": Array [],
"maintenanceWindowIds": Array [],
"uuid": "uuid-2",
},
"state": Object {

17
x-pack/plugins/alerting/server/lib/process_alerts.ts
View file

@ -23,6 +23,7 @@ interface ProcessAlertsOpts<
alertLimit: number;
autoRecoverAlerts: boolean;
flappingSettings: RulesSettingsFlappingProperties;
maintenanceWindowIds: string[];
}
interface ProcessAlertsResult<
State extends AlertInstanceState,
@ -50,6 +51,7 @@ export function processAlerts<
alertLimit,
autoRecoverAlerts,
flappingSettings,
maintenanceWindowIds,
}: ProcessAlertsOpts<State, Context>): ProcessAlertsResult<
State,
Context,
@ -62,14 +64,16 @@ export function processAlerts<
existingAlerts,
previouslyRecoveredAlerts,
alertLimit,
flappingSettings
flappingSettings,
maintenanceWindowIds
)
: processAlertsHelper(
alerts,
existingAlerts,
previouslyRecoveredAlerts,
autoRecoverAlerts,
flappingSettings
flappingSettings,
maintenanceWindowIds
);
}
@ -83,7 +87,8 @@ function processAlertsHelper<
existingAlerts: Record<string, Alert<State, Context>>,
previouslyRecoveredAlerts: Record<string, Alert<State, Context>>,
autoRecoverAlerts: boolean,
flappingSettings: RulesSettingsFlappingProperties
flappingSettings: RulesSettingsFlappingProperties,
maintenanceWindowIds: string[]
): ProcessAlertsResult<State, Context, ActionGroupIds, RecoveryActionGroupId> {
const existingAlertIds = new Set(Object.keys(existingAlerts));
const previouslyRecoveredAlertsIds = new Set(Object.keys(previouslyRecoveredAlerts));
@ -114,6 +119,7 @@ function processAlertsHelper<
}
updateAlertFlappingHistory(flappingSettings, newAlerts[id], true);
}
newAlerts[id].setMaintenanceWindowIds(maintenanceWindowIds);
} else {
// this alert did exist in previous run
// calculate duration to date for active alerts
@ -175,7 +181,8 @@ function processAlertsLimitReached<
existingAlerts: Record<string, Alert<State, Context>>,
previouslyRecoveredAlerts: Record<string, Alert<State, Context>>,
alertLimit: number,
flappingSettings: RulesSettingsFlappingProperties
flappingSettings: RulesSettingsFlappingProperties,
maintenanceWindowIds: string[]
): ProcessAlertsResult<State, Context, ActionGroupIds, RecoveryActionGroupId> {
const existingAlertIds = new Set(Object.keys(existingAlerts));
const previouslyRecoveredAlertsIds = new Set(Object.keys(previouslyRecoveredAlerts));
@ -244,6 +251,8 @@ function processAlertsLimitReached<
updateAlertFlappingHistory(flappingSettings, newAlerts[id], true);
}
newAlerts[id].setMaintenanceWindowIds(maintenanceWindowIds);
if (!hasCapacityForNewAlerts()) {
break;
}

2
x-pack/plugins/alerting/server/maintenance_window_client/maintenance_window_client.ts
View file

@ -71,6 +71,6 @@ export class MaintenanceWindowClient {
archive(this.context, params);
public finish = (params: FinishParams): Promise<MaintenanceWindow> =>
finish(this.context, params);
public getActiveMaintenanceWindows = (params: ActiveParams): Promise<MaintenanceWindow[]> =>
public getActiveMaintenanceWindows = (params?: ActiveParams): Promise<MaintenanceWindow[]> =>
getActiveMaintenanceWindows(this.context, params);
}

4
x-pack/plugins/alerting/server/maintenance_window_client/methods/get_active_maintenance_windows.ts
View file

@ -34,10 +34,10 @@ export interface ActiveParams {
export async function getActiveMaintenanceWindows(
context: MaintenanceWindowClientContext,
params: ActiveParams
params?: ActiveParams
): Promise<MaintenanceWindow[]> {
const { savedObjectsClient, logger } = context;
const { start, interval } = params;
const { start, interval } = params || {};
const startDate = start ? new Date(start) : new Date();
const duration = interval ? parseDuration(interval) : 0;

88
x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts
View file

@ -140,6 +140,7 @@ const generateAlert = ({
scheduleActions = true,
throttledActions = {},
lastScheduledActionsGroup = 'default',
maintenanceWindowIds,
}: {
id: number;
group?: ActiveActionGroup | 'recovered';
@ -148,12 +149,14 @@ const generateAlert = ({
scheduleActions?: boolean;
throttledActions?: ThrottledActions;
lastScheduledActionsGroup?: string;
maintenanceWindowIds?: string[];
}) => {
const alert = new Alert<AlertInstanceState, AlertInstanceContext, 'default' | 'other-group'>(
String(id),
{
state: state || { test: true },
meta: {
maintenanceWindowIds,
lastScheduledActions: {
date: new Date(),
group: lastScheduledActionsGroup,
@ -1499,6 +1502,91 @@ describe('Execution Handler', () => {
);
});
test('does not schedule summary actions when there is an active maintenance window', async () => {
getSummarizedAlertsMock.mockResolvedValue({
new: {
count: 2,
data: [
{ ...mockAAD, kibana: { alert: { uuid: '1' } } },
{ ...mockAAD, kibana: { alert: { uuid: '2' } } },
],
},
ongoing: { count: 0, data: [] },
recovered: { count: 0, data: [] },
});
const executionHandler = new ExecutionHandler(
generateExecutionParams({
rule: {
...defaultExecutionParams.rule,
mutedInstanceIds: ['foo'],
actions: [
{
uuid: '1',
id: '1',
group: null,
actionTypeId: 'testActionTypeId',
frequency: {
summary: true,
notifyWhen: 'onActiveAlert',
throttle: null,
},
params: {
message:
'New: {{alerts.new.count}} Ongoing: {{alerts.ongoing.count}} Recovered: {{alerts.recovered.count}}',
},
},
],
},
maintenanceWindowIds: ['test-id-active'],
})
);
await executionHandler.run({
...generateAlert({ id: 1, maintenanceWindowIds: ['test-id-1'] }),
...generateAlert({ id: 2, maintenanceWindowIds: ['test-id-2'] }),
...generateAlert({ id: 3, maintenanceWindowIds: ['test-id-3'] }),
});
expect(actionsClient.bulkEnqueueExecution).not.toHaveBeenCalled();
expect(defaultExecutionParams.logger.debug).toHaveBeenCalledTimes(2);
expect(defaultExecutionParams.logger.debug).toHaveBeenNthCalledWith(
1,
'(1) alert has been filtered out for: testActionTypeId:1'
);
expect(defaultExecutionParams.logger.debug).toHaveBeenNthCalledWith(
2,
'no scheduling of summary actions "1" for rule "1": has active maintenance windows test-id-active.'
);
});
test('does not schedule actions for alerts with maintenance window IDs', async () => {
const executionHandler = new ExecutionHandler(generateExecutionParams());
await executionHandler.run({
...generateAlert({ id: 1, maintenanceWindowIds: ['test-id-1'] }),
...generateAlert({ id: 2, maintenanceWindowIds: ['test-id-2'] }),
...generateAlert({ id: 3, maintenanceWindowIds: ['test-id-3'] }),
});
expect(actionsClient.bulkEnqueueExecution).not.toHaveBeenCalled();
expect(defaultExecutionParams.logger.debug).toHaveBeenCalledTimes(3);
expect(defaultExecutionParams.logger.debug).toHaveBeenNthCalledWith(
1,
'no scheduling of actions "1" for rule "1": has active maintenance windows test-id-1.'
);
expect(defaultExecutionParams.logger.debug).toHaveBeenNthCalledWith(
2,
'no scheduling of actions "1" for rule "1": has active maintenance windows test-id-2.'
);
expect(defaultExecutionParams.logger.debug).toHaveBeenNthCalledWith(
3,
'no scheduling of actions "1" for rule "1": has active maintenance windows test-id-3.'
);
});
describe('rule url', () => {
const ruleWithUrl = {
...rule,

24
x-pack/plugins/alerting/server/task_runner/execution_handler.ts
View file

@ -92,6 +92,7 @@ export class ExecutionHandler<
private ruleTypeActionGroups?: Map<ActionGroupIds | RecoveryActionGroupId, string>;
private mutedAlertIdsSet: Set<string> = new Set();
private previousStartedAt: Date | null;
private maintenanceWindowIds: string[] = [];
constructor({
rule,
@ -107,6 +108,7 @@ export class ExecutionHandler<
ruleLabel,
previousStartedAt,
actionsClient,
maintenanceWindowIds,
}: ExecutionHandlerOptions<
Params,
ExtractedParams,
@ -134,6 +136,7 @@ export class ExecutionHandler<
);
this.previousStartedAt = previousStartedAt;
this.mutedAlertIdsSet = new Set(rule.mutedInstanceIds);
this.maintenanceWindowIds = maintenanceWindowIds ?? [];
}
public async run(
@ -509,7 +512,16 @@ export class ExecutionHandler<
}
}
if (isSummaryAction(action)) {
// By doing that we are not cancelling the summary action but just waiting
// for the window maintenance to be over before sending the summary action
if (isSummaryAction(action) && this.maintenanceWindowIds.length > 0) {
this.logger.debug(
`no scheduling of summary actions "${action.id}" for rule "${
this.taskInstance.params.alertId
}": has active maintenance windows ${this.maintenanceWindowIds.join()}.`
);
continue;
} else if (isSummaryAction(action)) {
if (summarizedAlerts && summarizedAlerts.all.count !== 0) {
executables.push({ action, summarizedAlerts });
}
@ -520,6 +532,16 @@ export class ExecutionHandler<
if (alert.isFilteredOut(summarizedAlerts)) {
continue;
}
if (alert.getMaintenanceWindowIds().length > 0) {
this.logger.debug(
`no scheduling of actions "${action.id}" for rule "${
this.taskInstance.params.alertId
}": has active maintenance windows ${alert.getMaintenanceWindowIds().join()}.`
);
continue;
}
const actionGroup = this.getActionGroup(alert);
if (!this.ruleTypeActionGroups!.has(actionGroup)) {

5
x-pack/plugins/alerting/server/task_runner/fixtures.ts
View file

@ -241,7 +241,7 @@ export const generateAlertOpts = ({
group,
state,
id,
maintenanceWindowIds = [],
maintenanceWindowIds,
}: GeneratorParams = {}) => {
id = id ?? '1';
let message: string = '';
@ -264,7 +264,7 @@ export const generateAlertOpts = ({
state,
...(group ? { group } : {}),
flapping: false,
maintenanceWindowIds,
...(maintenanceWindowIds ? { maintenanceWindowIds } : {}),
};
};
@ -374,6 +374,7 @@ export const generateAlertInstance = (
},
flappingHistory,
flapping: false,
maintenanceWindowIds: [],
pendingRecoveredCount: 0,
},
state: {

18
x-pack/plugins/alerting/server/task_runner/log_alerts.test.ts
View file

@ -365,8 +365,6 @@ describe('logAlerts', () => {
test('should correctly set maintenance window in ruleRunMetricsStore and call alertingEventLogger.logAlert', () => {
jest.clearAllMocks();
const MAINTENANCE_WINDOW_IDS = ['window-id-1', 'window-id-2'];
logAlerts({
logger,
alertingEventLogger,
@ -374,18 +372,21 @@ describe('logAlerts', () => {
'4': new Alert<{}, {}, DefaultActionGroupId>('4'),
},
activeAlerts: {
'1': new Alert<{}, {}, DefaultActionGroupId>('1'),
'1': new Alert<{}, {}, DefaultActionGroupId>('1', {
meta: { maintenanceWindowIds: ['window-id-1'] },
}),
'4': new Alert<{}, {}, DefaultActionGroupId>('4'),
},
recoveredAlerts: {
'7': new Alert<{}, {}, DefaultActionGroupId>('7'),
'8': new Alert<{}, {}, DefaultActionGroupId>('8'),
'8': new Alert<{}, {}, DefaultActionGroupId>('8', {
meta: { maintenanceWindowIds: ['window-id-8'] },
}),
},
ruleLogPrefix: `test-rule-type-id:123: 'test rule'`,
ruleRunMetricsStore,
canSetRecoveryContext: false,
shouldPersistAlerts: true,
maintenanceWindowIds: MAINTENANCE_WINDOW_IDS,
});
expect(ruleRunMetricsStore.getNumberOfNewAlerts()).toEqual(1);
@ -402,7 +403,6 @@ describe('logAlerts', () => {
flapping: false,
group: undefined,
uuid: expect.any(String),
maintenanceWindowIds: MAINTENANCE_WINDOW_IDS,
});
expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(2, {
action: 'recovered-instance',
@ -412,7 +412,7 @@ describe('logAlerts', () => {
flapping: false,
group: undefined,
uuid: expect.any(String),
maintenanceWindowIds: MAINTENANCE_WINDOW_IDS,
maintenanceWindowIds: ['window-id-8'],
});
expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(3, {
action: 'new-instance',
@ -422,7 +422,6 @@ describe('logAlerts', () => {
flapping: false,
group: undefined,
uuid: expect.any(String),
maintenanceWindowIds: MAINTENANCE_WINDOW_IDS,
});
expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(4, {
action: 'active-instance',
@ -432,7 +431,7 @@ describe('logAlerts', () => {
flapping: false,
group: undefined,
uuid: expect.any(String),
maintenanceWindowIds: MAINTENANCE_WINDOW_IDS,
maintenanceWindowIds: ['window-id-1'],
});
expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(5, {
action: 'active-instance',
@ -442,7 +441,6 @@ describe('logAlerts', () => {
flapping: false,
group: undefined,
uuid: expect.any(String),
maintenanceWindowIds: MAINTENANCE_WINDOW_IDS,
});
});
});

11
x-pack/plugins/alerting/server/task_runner/log_alerts.ts
View file

@ -28,7 +28,6 @@ export interface LogAlertsParams<
ruleRunMetricsStore: RuleRunMetricsStore;
canSetRecoveryContext: boolean;
shouldPersistAlerts: boolean;
maintenanceWindowIds?: string[];
}
export function logAlerts<
@ -46,7 +45,6 @@ export function logAlerts<
ruleRunMetricsStore,
canSetRecoveryContext,
shouldPersistAlerts,
maintenanceWindowIds,
}: LogAlertsParams<State, Context, ActionGroupIds, RecoveryActionGroupId>) {
const newAlertIds = Object.keys(newAlerts);
const activeAlertIds = Object.keys(activeAlerts);
@ -97,6 +95,7 @@ export function logAlerts<
const { group: actionGroup } = alert.getLastScheduledActions() ?? {};
const uuid = alert.getUuid();
const state = recoveredAlerts[id].getState();
const maintenanceWindowIds = alert.getMaintenanceWindowIds();
const message = `${ruleLogPrefix} alert '${id}' has recovered`;
alertingEventLogger.logAlert({
action: EVENT_LOG_ACTIONS.recoveredInstance,
@ -106,7 +105,7 @@ export function logAlerts<
message,
state,
flapping: recoveredAlerts[id].getFlapping(),
maintenanceWindowIds,
...(maintenanceWindowIds.length ? { maintenanceWindowIds } : {}),
});
}
@ -115,6 +114,7 @@ export function logAlerts<
const { actionGroup } = alert.getScheduledActionOptions() ?? {};
const state = alert.getState();
const uuid = alert.getUuid();
const maintenanceWindowIds = alert.getMaintenanceWindowIds();
const message = `${ruleLogPrefix} created new alert: '${id}'`;
alertingEventLogger.logAlert({
action: EVENT_LOG_ACTIONS.newInstance,
@ -124,7 +124,7 @@ export function logAlerts<
message,
state,
flapping: activeAlerts[id].getFlapping(),
maintenanceWindowIds,
...(maintenanceWindowIds.length ? { maintenanceWindowIds } : {}),
});
}
@ -133,6 +133,7 @@ export function logAlerts<
const { actionGroup } = alert.getScheduledActionOptions() ?? {};
const state = alert.getState();
const uuid = alert.getUuid();
const maintenanceWindowIds = alert.getMaintenanceWindowIds();
const message = `${ruleLogPrefix} active alert: '${id}' in actionGroup: '${actionGroup}'`;
alertingEventLogger.logAlert({
action: EVENT_LOG_ACTIONS.activeInstance,
@ -142,7 +143,7 @@ export function logAlerts<
message,
state,
flapping: activeAlerts[id].getFlapping(),
maintenanceWindowIds,
...(maintenanceWindowIds.length ? { maintenanceWindowIds } : {}),
});
}
}

30
x-pack/plugins/alerting/server/task_runner/task_runner.test.ts
View file

@ -651,33 +651,6 @@ describe('Task Runner', () => {
await taskRunner.run();
expect(actionsClient.ephemeralEnqueuedExecution).toHaveBeenCalledTimes(0);
expect(logger.debug).toHaveBeenCalledTimes(7);
expect(logger.debug).nthCalledWith(1, 'executing rule test:1 at 1970-01-01T00:00:00.000Z');
expect(logger.debug).nthCalledWith(
2,
`rule test:1: '${RULE_NAME}' has 1 active alerts: [{\"instanceId\":\"1\",\"actionGroup\":\"default\"}]`
);
expect(logger.debug).nthCalledWith(
3,
`no scheduling of actions for rule test:1: '${RULE_NAME}': has active maintenance windows test-id-1,test-id-2.`
);
expect(logger.debug).nthCalledWith(
4,
'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"}'
);
expect(logger.debug).nthCalledWith(
5,
'ruleRunStatus for test:1: {"outcome":"succeeded","outcomeOrder":0,"outcomeMsg":null,"warning":null,"alertsCount":{"active":1,"new":1,"recovered":0,"ignored":0}}'
);
expect(logger.debug).nthCalledWith(
6,
'ruleRunMetrics for test:1: {"numSearches":3,"totalSearchDurationMs":23423,"esSearchDurationMs":33,"numberOfTriggeredActions":0,"numberOfGeneratedActions":0,"numberOfActiveAlerts":1,"numberOfRecoveredAlerts":0,"numberOfNewAlerts":1,"hasReachedAlertLimit":false,"triggeredActionsStatus":"complete"}'
);
expect(logger.debug).nthCalledWith(
7,
'Updating rule task for test rule with id 1 - {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"active"} - {"outcome":"succeeded","outcomeOrder":0,"outcomeMsg":null,"warning":null,"alertsCount":{"active":1,"new":1,"recovered":0,"ignored":0}}'
);
const maintenanceWindowIds = ['test-id-1', 'test-id-2'];
testAlertingEventLogCalls({
@ -2767,6 +2740,7 @@ describe('Task Runner', () => {
group: 'default',
},
flappingHistory: [true],
maintenanceWindowIds: [],
flapping: false,
pendingRecoveredCount: 0,
},
@ -2934,6 +2908,7 @@ describe('Task Runner', () => {
group: 'default',
},
flappingHistory: [true],
maintenanceWindowIds: [],
flapping: false,
pendingRecoveredCount: 0,
},
@ -2950,6 +2925,7 @@ describe('Task Runner', () => {
group: 'default',
},
flappingHistory: [true],
maintenanceWindowIds: [],
flapping: false,
pendingRecoveredCount: 0,
},

15
x-pack/plugins/alerting/server/task_runner/task_runner.ts
View file

@ -320,9 +320,7 @@ export class TaskRunner<
let activeMaintenanceWindows: MaintenanceWindow[] = [];
try {
activeMaintenanceWindows = await maintenanceWindowClient.getActiveMaintenanceWindows({
interval: rule.schedule.interval,
});
activeMaintenanceWindows = await maintenanceWindowClient.getActiveMaintenanceWindows();
} catch (err) {
this.logger.error(
`error getting active maintenance window for ${ruleTypeId}:${ruleId} ${err.message}`
@ -339,7 +337,11 @@ export class TaskRunner<
const { updatedRuleTypeState } = await this.timer.runWithTimer(
TaskRunnerTimerSpan.RuleTypeRun,
async () => {
this.legacyAlertsClient.initialize(alertRawInstances, alertRecoveredRawInstances);
this.legacyAlertsClient.initialize(
alertRawInstances,
alertRecoveredRawInstances,
maintenanceWindowIds
);
const checkHasReachedAlertLimit = () => {
const reachedLimit = this.legacyAlertsClient.hasReachedAlertLimit();
@ -483,6 +485,7 @@ export class TaskRunner<
previousStartedAt: previousStartedAt ? new Date(previousStartedAt) : null,
alertingEventLogger: this.alertingEventLogger,
actionsClient: await this.context.actionsPlugin.getActionsClientWithRequest(fakeRequest),
maintenanceWindowIds,
});
let executionHandlerRunResult: RunResult = { throttledSummaryActions: {} };
@ -492,10 +495,6 @@ export class TaskRunner<
if (isRuleSnoozed(rule)) {
this.logger.debug(`no scheduling of actions for rule ${ruleLabel}: rule is snoozed.`);
} else if (maintenanceWindowIds.length) {
this.logger.debug(
`no scheduling of actions for rule ${ruleLabel}: has active maintenance windows ${maintenanceWindowIds}.`
);
} else if (!this.shouldLogAndScheduleActionsForAlerts()) {
this.logger.debug(
`no scheduling of actions for rule ${ruleLabel}: rule execution has been cancelled.`

1
x-pack/plugins/alerting/server/task_runner/types.ts
View file

@ -87,6 +87,7 @@ export interface ExecutionHandlerOptions<
ruleLabel: string;
previousStartedAt: Date | null;
actionsClient: PublicMethodsOf<ActionsClient>;
maintenanceWindowIds?: string[];
}
export interface Executable<

64
x-pack/plugins/rule_registry/server/utils/create_get_summarized_alerts_fn.test.ts
View file

@ -12,6 +12,7 @@ import {
ALERT_ACTION_GROUP,
ALERT_END,
ALERT_INSTANCE_ID,
ALERT_MAINTENANCE_WINDOW_IDS,
ALERT_RULE_EXECUTION_UUID,
ALERT_RULE_UUID,
ALERT_START,
@ -200,6 +201,15 @@ describe('createGetSummarizedAlertsFn', () => {
[ALERT_RULE_UUID]: 'rule-id',
},
},
{
bool: {
must_not: {
exists: {
field: ALERT_MAINTENANCE_WINDOW_IDS,
},
},
},
},
{
term: {
[EVENT_ACTION]: 'open',
@ -236,6 +246,15 @@ describe('createGetSummarizedAlertsFn', () => {
[ALERT_RULE_UUID]: 'rule-id',
},
},
{
bool: {
must_not: {
exists: {
field: ALERT_MAINTENANCE_WINDOW_IDS,
},
},
},
},
{
term: {
[EVENT_ACTION]: 'active',
@ -272,6 +291,15 @@ describe('createGetSummarizedAlertsFn', () => {
[ALERT_RULE_UUID]: 'rule-id',
},
},
{
bool: {
must_not: {
exists: {
field: ALERT_MAINTENANCE_WINDOW_IDS,
},
},
},
},
{
term: {
[EVENT_ACTION]: 'close',
@ -934,6 +962,15 @@ describe('createGetSummarizedAlertsFn', () => {
[ALERT_RULE_UUID]: 'rule-id',
},
},
{
bool: {
must_not: {
exists: {
field: ALERT_MAINTENANCE_WINDOW_IDS,
},
},
},
},
{
bool: {
must_not: {
@ -2127,6 +2164,15 @@ describe('createGetSummarizedAlertsFn', () => {
[ALERT_RULE_UUID]: 'rule-id',
},
},
{
bool: {
must_not: {
exists: {
field: ALERT_MAINTENANCE_WINDOW_IDS,
},
},
},
},
{
term: {
[EVENT_ACTION]: 'open',
@ -2213,6 +2259,15 @@ describe('createGetSummarizedAlertsFn', () => {
[ALERT_RULE_UUID]: 'rule-id',
},
},
{
bool: {
must_not: {
exists: {
field: ALERT_MAINTENANCE_WINDOW_IDS,
},
},
},
},
{
term: {
[EVENT_ACTION]: 'active',
@ -2299,6 +2354,15 @@ describe('createGetSummarizedAlertsFn', () => {
[ALERT_RULE_UUID]: 'rule-id',
},
},
{
bool: {
must_not: {
exists: {
field: ALERT_MAINTENANCE_WINDOW_IDS,
},
},
},
},
{
term: {
[EVENT_ACTION]: 'close',

10
x-pack/plugins/rule_registry/server/utils/create_get_summarized_alerts_fn.ts
View file

@ -17,6 +17,7 @@ import {
EVENT_ACTION,
TIMESTAMP,
ALERT_INSTANCE_ID,
ALERT_MAINTENANCE_WINDOW_IDS,
} from '@kbn/rule-data-utils';
import {
QueryDslQueryContainer,
@ -292,6 +293,15 @@ const getQueryByExecutionUuid = ({
[ALERT_RULE_UUID]: ruleId,
},
},
{
bool: {
must_not: {
exists: {
field: ALERT_MAINTENANCE_WINDOW_IDS,
},
},
},
},
];
if (action) {
filter.push({

8
x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts
View file

@ -991,7 +991,7 @@ describe('createLifecycleExecutor', () => {
);
});
it('updates documents with maintenance window ids for repeatedly firing alerts', async () => {
it('does not update documents with maintenance window ids for repeatedly firing alerts', async () => {
const logger = loggerMock.create();
const ruleDataClientMock = createRuleDataClientMock();
ruleDataClientMock.getReader().search.mockResolvedValue({
@ -1094,7 +1094,6 @@ describe('createLifecycleExecutor', () => {
labels: { LABEL_0_KEY: 'LABEL_0_VALUE' },
[EVENT_ACTION]: 'active',
[EVENT_KIND]: 'signal',
[ALERT_MAINTENANCE_WINDOW_IDS]: maintenanceWindowIds,
}),
{ index: { _id: 'TEST_ALERT_1_UUID' } },
expect.objectContaining({
@ -1103,7 +1102,6 @@ describe('createLifecycleExecutor', () => {
[ALERT_STATUS]: ALERT_STATUS_ACTIVE,
[EVENT_ACTION]: 'active',
[EVENT_KIND]: 'signal',
[ALERT_MAINTENANCE_WINDOW_IDS]: maintenanceWindowIds,
}),
],
})
@ -1121,7 +1119,7 @@ describe('createLifecycleExecutor', () => {
);
});
it('updates document with maintenance window ids for recovered alerts', async () => {
it('does not update documents with maintenance window ids for recovered alerts', async () => {
const logger = loggerMock.create();
const ruleDataClientMock = createRuleDataClientMock();
ruleDataClientMock.getReader().search.mockResolvedValue({
@ -1220,7 +1218,6 @@ describe('createLifecycleExecutor', () => {
[TAGS]: ['source-tag1', 'source-tag2', 'rule-tag1', 'rule-tag2'],
[EVENT_ACTION]: 'close',
[EVENT_KIND]: 'signal',
[ALERT_MAINTENANCE_WINDOW_IDS]: maintenanceWindowIds,
}),
{ index: { _id: 'TEST_ALERT_1_UUID' } },
expect.objectContaining({
@ -1229,7 +1226,6 @@ describe('createLifecycleExecutor', () => {
[EVENT_ACTION]: 'active',
[EVENT_KIND]: 'signal',
[TAGS]: ['source-tag3', 'source-tag4', 'rule-tag1', 'rule-tag2'],
[ALERT_MAINTENANCE_WINDOW_IDS]: maintenanceWindowIds,
}),
]),
})

2
x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts
View file

@ -301,7 +301,7 @@ export const createLifecycleExecutor =
[VERSION]: ruleDataClient.kibanaVersion,
[ALERT_FLAPPING]: flapping,
...(isRecovered ? { [ALERT_END]: commonRuleFields[TIMESTAMP] } : {}),
...(maintenanceWindowIds?.length
...(isNew && maintenanceWindowIds?.length
? { [ALERT_MAINTENANCE_WINDOW_IDS]: maintenanceWindowIds }
: {}),
};

4
x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts
View file

@ -12,6 +12,7 @@ import { chunk, partition } from 'lodash';
import {
ALERT_INSTANCE_ID,
ALERT_LAST_DETECTED,
ALERT_MAINTENANCE_WINDOW_IDS,
ALERT_NAMESPACE,
ALERT_START,
ALERT_SUPPRESSION_DOCS_COUNT,
@ -49,6 +50,9 @@ const augmentAlerts = <T>({
[ALERT_START]: currentTimeOverride ?? new Date(),
[ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(),
[VERSION]: kibanaVersion,
...(options?.maintenanceWindowIds?.length
? { [ALERT_MAINTENANCE_WINDOW_IDS]: options.maintenanceWindowIds }
: {}),
...commonRuleFields,
...alert._source,
},

18
x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts
View file

@ -15,7 +15,6 @@ import {
AlertConsumers,
ALERT_REASON,
ALERT_INSTANCE_ID,
ALERT_MAINTENANCE_WINDOW_IDS,
} from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names';
import {
createLifecycleExecutor,
@ -382,7 +381,7 @@ export default function createGetSummarizedAlertsTest({ getService }: FtrProvide
expect(get(summarizedAlertsExcludingId2.new.data[0], ALERT_INSTANCE_ID)).to.eql(id1);
});
it('should return new, ongoing, and recovered alerts if there are active maintenance windows', async () => {
it('should not trigger new, ongoing, and recovered alerts if there are active maintenance windows', async () => {
const id = 'host-01';
const maintenanceWindowIds = ['test-id-1', 'test-id-2'];
@ -466,12 +465,9 @@ export default function createGetSummarizedAlertsTest({ getService }: FtrProvide
spaceId: 'default',
excludedAlertInstanceIds: [],
});
expect(execution1SummarizedAlerts.new.count).to.eql(1);
expect(execution1SummarizedAlerts.new.count).to.eql(0);
expect(execution1SummarizedAlerts.ongoing.count).to.eql(0);
expect(execution1SummarizedAlerts.recovered.count).to.eql(0);
expect(get(execution1SummarizedAlerts.new.data[0], ALERT_MAINTENANCE_WINDOW_IDS)).to.eql(
maintenanceWindowIds
);
// Execute again to update the existing alert
const execution2Uuid = uuidv4();
@ -489,11 +485,8 @@ export default function createGetSummarizedAlertsTest({ getService }: FtrProvide
excludedAlertInstanceIds: [],
});
expect(execution2SummarizedAlerts.new.count).to.eql(0);
expect(execution2SummarizedAlerts.ongoing.count).to.eql(1);
expect(execution2SummarizedAlerts.ongoing.count).to.eql(0);
expect(execution2SummarizedAlerts.recovered.count).to.eql(0);
expect(get(execution2SummarizedAlerts.ongoing.data[0], ALERT_MAINTENANCE_WINDOW_IDS)).to.eql(
maintenanceWindowIds
);
// Execute again to recover the alert
const execution3Uuid = uuidv4();
@ -512,10 +505,7 @@ export default function createGetSummarizedAlertsTest({ getService }: FtrProvide
});
expect(execution3SummarizedAlerts.new.count).to.eql(0);
expect(execution3SummarizedAlerts.ongoing.count).to.eql(0);
expect(execution3SummarizedAlerts.recovered.count).to.eql(1);
expect(
get(execution3SummarizedAlerts.recovered.data[0], ALERT_MAINTENANCE_WINDOW_IDS)
).to.eql(maintenanceWindowIds);
expect(execution3SummarizedAlerts.recovered.count).to.eql(0);
});
});
}