github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

Onboard Synthetics Monitor Status rule type with FAAD (#186214)

Browse source 
Towards: https://github.com/elastic/kibana/issues/169867

This PR onboards the Synthetics Monitor Status rule type with FAAD.

### To verify
I can't get the rule to alert, so I modified the status check to report
the monitor as down. If you know of an easier way pls let me know 🙂

1. Create a [monitor](http://localhost:5601/app/synthetics/monitors), by
default creating a monitor creates a rule.
2. Click on the monitor and grab the id and locationId from the url
3. Go to [the status check
code](https://github.com/elastic/kibana/blob/main/x-pack/plugins/observability_solution/synthetics/server/queries/query_monitor_status.ts#L208)
and replace the object that is returned with the following using the id
and locationId you got from the monitor.
```
{
    up: 0,
    down: 1,
    pending: 0,
    upConfigs: {},
    pendingConfigs: {},
    downConfigs: {
      '${id}-${locationId}': {
        configId: '${id}',
        monitorQueryId: '${id}',
        status: 'down',
        locationId: '${locationId}',
        ping: {
          '@timestamp': new Date().toISOString(),
          state: {
            id: 'test-state',
          },
          monitor: {
            name: 'test-monitor',
          },
          observer: {
            name: 'test-monitor',
          },
        } as any,
        timestamp: new Date().toISOString(),
      },
    },
    enabledMonitorQueryIds: ['${id}'],
  };
```
5. Your rule should create an alert and should saved it in
`.internal.alerts-observability.uptime.alerts-default-000001`
Example:
```
GET .internal.alerts-*/_search
```
6. Recover repeating step 3 using
```
{
    up: 1,
    down: 0,
    pending: 0,
    downConfigs: {},
    pendingConfigs: {},
    upConfigs: {
      '${id}-${locationId}': {
        configId: '${id}',
        monitorQueryId: '${id}',
        status: 'down',
        locationId: '${locationId}',
        ping: {
          '@timestamp': new Date().toISOString(),
          state: {
            id: 'test-state',
          },
          monitor: {
            name: 'test-monitor',
          },
          observer: {
            name: 'test-monitor',
          },
        } as any,
        timestamp: new Date().toISOString(),
      },
    },
    enabledMonitorQueryIds: ['${id}'],
  };
```
8. The alert should be recovered and the AAD in the above index should
be updated `kibana.alert.status: recovered`.
This commit is contained in:
Alexi Doak 2024-06-20 06:56:17 -07:00 • committed by GitHub
parent 85f12800bb
commit 0468b8f46d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 206 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

215
x-pack/plugins/observability_solution/synthetics/server/alert_rules/common.test.ts
View file

@ -4,7 +4,6 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
import { IBasePath } from '@kbn/core/server';
import { updateState, setRecoveredAlertsContext } from './common';
import { SyntheticsCommonState } from '../../common/runtime_types/alert_rules/common';
@ -186,8 +185,6 @@ describe('updateState', () => {
});
describe('setRecoveredAlertsContext', () => {
const { alertFactory } = alertsMock.createRuleExecutorServices();
const { getRecoveredAlerts } = alertFactory.done();
const alertUuid = 'alert-id';
const location = 'US Central';
const configId = '12345';
@ -195,7 +192,6 @@ describe('setRecoveredAlertsContext', () => {
const basePath = {
publicBaseUrl: 'https://localhost:5601',
} as IBasePath;
const getAlertUuid = () => alertUuid;
const upConfigs = {
[idWithLocation]: {
@ -219,17 +215,26 @@ describe('setRecoveredAlertsContext', () => {
};
it('sets context correctly when monitor is deleted', () => {
const setContext = jest.fn();
getRecoveredAlerts.mockReturnValue([
{
getId: () => alertUuid,
getState: () => ({
idWithLocation,
monitorName: 'test-monitor',
}),
setContext,
},
]);
const alertsClientMock = {
report: jest.fn(),
getAlertLimitValue: jest.fn().mockReturnValue(10),
setAlertLimitReached: jest.fn(),
getRecoveredAlerts: jest.fn().mockReturnValue([
{
alert: {
getId: () => alertUuid,
getState: () => ({
idWithLocation,
monitorName: 'test-monitor',
}),
setContext: jest.fn(),
getUuid: () => alertUuid,
},
},
]),
setAlertData: jest.fn(),
isTrackedAlert: jest.fn(),
};
const staleDownConfigs = {
[idWithLocation]: {
configId,
@ -250,45 +255,56 @@ describe('setRecoveredAlertsContext', () => {
},
};
setRecoveredAlertsContext({
alertFactory,
alertsClient: alertsClientMock,
basePath,
getAlertUuid,
spaceId: 'default',
staleDownConfigs,
upConfigs: {},
dateFormat,
tz: 'UTC',
});
expect(setContext).toBeCalledWith({
checkedAt: 'Feb 26, 2023 @ 00:00:00.000',
configId: '12345',
idWithLocation,
linkMessage: '',
alertDetailsUrl: 'https://localhost:5601/app/observability/alerts/alert-id',
monitorName: 'test-monitor',
recoveryReason: 'the monitor has been deleted',
recoveryStatus: 'has been deleted',
monitorUrl: '(unavailable)',
monitorUrlLabel: 'URL',
reason:
'Monitor "test-monitor" from Unnamed-location is recovered. Checked at February 25, 2023 7:00 PM.',
stateId: '123456',
status: 'recovered',
expect(alertsClientMock.setAlertData).toBeCalledWith({
id: 'alert-id',
context: {
checkedAt: 'Feb 26, 2023 @ 00:00:00.000',
configId: '12345',
idWithLocation,
linkMessage: '',
alertDetailsUrl: 'https://localhost:5601/app/observability/alerts/alert-id',
monitorName: 'test-monitor',
recoveryReason: 'the monitor has been deleted',
recoveryStatus: 'has been deleted',
monitorUrl: '(unavailable)',
monitorUrlLabel: 'URL',
reason:
'Monitor "test-monitor" from Unnamed-location is recovered. Checked at February 25, 2023 7:00 PM.',
stateId: '123456',
status: 'recovered',
},
});
});
it('sets context correctly when location is removed', () => {
const setContext = jest.fn();
getRecoveredAlerts.mockReturnValue([
{
getId: () => alertUuid,
getState: () => ({
idWithLocation,
monitorName: 'test-monitor',
}),
setContext,
},
]);
const alertsClientMock = {
report: jest.fn(),
getAlertLimitValue: jest.fn().mockReturnValue(10),
setAlertLimitReached: jest.fn(),
getRecoveredAlerts: jest.fn().mockReturnValue([
{
alert: {
getId: () => alertUuid,
getState: () => ({
idWithLocation,
monitorName: 'test-monitor',
}),
setContext: jest.fn(),
getUuid: () => alertUuid,
},
},
]),
setAlertData: jest.fn(),
isTrackedAlert: jest.fn(),
};
const staleDownConfigs = {
[idWithLocation]: {
configId,
@ -309,47 +325,58 @@ describe('setRecoveredAlertsContext', () => {
},
};
setRecoveredAlertsContext({
alertFactory,
alertsClient: alertsClientMock,
basePath,
getAlertUuid,
spaceId: 'default',
staleDownConfigs,
upConfigs: {},
dateFormat,
tz: 'UTC',
});
expect(setContext).toBeCalledWith({
configId: '12345',
checkedAt: 'Feb 26, 2023 @ 00:00:00.000',
monitorUrl: '(unavailable)',
reason:
'Monitor "test-monitor" from Unnamed-location is recovered. Checked at February 25, 2023 7:00 PM.',
idWithLocation,
linkMessage: '',
alertDetailsUrl: 'https://localhost:5601/app/observability/alerts/alert-id',
monitorName: 'test-monitor',
recoveryReason: 'this location has been removed from the monitor',
recoveryStatus: 'has recovered',
stateId: '123456',
status: 'recovered',
monitorUrlLabel: 'URL',
expect(alertsClientMock.setAlertData).toBeCalledWith({
id: 'alert-id',
context: {
configId: '12345',
checkedAt: 'Feb 26, 2023 @ 00:00:00.000',
monitorUrl: '(unavailable)',
reason:
'Monitor "test-monitor" from Unnamed-location is recovered. Checked at February 25, 2023 7:00 PM.',
idWithLocation,
linkMessage: '',
alertDetailsUrl: 'https://localhost:5601/app/observability/alerts/alert-id',
monitorName: 'test-monitor',
recoveryReason: 'this location has been removed from the monitor',
recoveryStatus: 'has recovered',
stateId: '123456',
status: 'recovered',
monitorUrlLabel: 'URL',
},
});
});
it('sets context correctly when monitor is up', () => {
const setContext = jest.fn();
getRecoveredAlerts.mockReturnValue([
{
getId: () => alertUuid,
getState: () => ({
idWithLocation,
monitorName: 'test-monitor',
locationId: 'us_west',
configId: '12345-67891',
}),
setContext,
},
]);
const alertsClientMock = {
report: jest.fn(),
getAlertLimitValue: jest.fn().mockReturnValue(10),
setAlertLimitReached: jest.fn(),
getRecoveredAlerts: jest.fn().mockReturnValue([
{
alert: {
getId: () => alertUuid,
getState: () => ({
idWithLocation,
monitorName: 'test-monitor',
locationId: 'us_west',
configId: '12345-67891',
}),
setContext: jest.fn(),
getUuid: () => alertUuid,
},
},
]),
setAlertData: jest.fn(),
isTrackedAlert: jest.fn(),
};
const staleDownConfigs = {
[idWithLocation]: {
configId,
@ -370,33 +397,35 @@ describe('setRecoveredAlertsContext', () => {
},
};
setRecoveredAlertsContext({
alertFactory,
alertsClient: alertsClientMock,
basePath,
getAlertUuid,
spaceId: 'default',
staleDownConfigs,
upConfigs,
dateFormat,
tz: 'UTC',
});
expect(setContext).toBeCalledWith({
configId: '12345-67891',
idWithLocation,
alertDetailsUrl: 'https://localhost:5601/app/observability/alerts/alert-id',
monitorName: 'test-monitor',
status: 'up',
recoveryReason:
'the monitor is now up again. It ran successfully at Feb 26, 2023 @ 00:00:00.000',
recoveryStatus: 'is now up',
locationId: 'us_west',
checkedAt: 'Feb 26, 2023 @ 00:00:00.000',
linkMessage:
'- Link: https://localhost:5601/app/synthetics/monitor/12345-67891/errors/123456?locationId=us_west',
monitorUrl: '(unavailable)',
monitorUrlLabel: 'URL',
reason:
'Monitor "test-monitor" from Unnamed-location is recovered. Checked at February 25, 2023 7:00 PM.',
stateId: null,
expect(alertsClientMock.setAlertData).toBeCalledWith({
id: 'alert-id',
context: {
configId: '12345-67891',
idWithLocation,
alertDetailsUrl: 'https://localhost:5601/app/observability/alerts/alert-id',
monitorName: 'test-monitor',
status: 'up',
recoveryReason:
'the monitor is now up again. It ran successfully at Feb 26, 2023 @ 00:00:00.000',
recoveryStatus: 'is now up',
locationId: 'us_west',
checkedAt: 'Feb 26, 2023 @ 00:00:00.000',
linkMessage:
'- Link: https://localhost:5601/app/synthetics/monitor/12345-67891/errors/123456?locationId=us_west',
monitorUrl: '(unavailable)',
monitorUrlLabel: 'URL',
reason:
'Monitor "test-monitor" from Unnamed-location is recovered. Checked at February 25, 2023 7:00 PM.',
stateId: null,
},
});
});
});

42
x-pack/plugins/observability_solution/synthetics/server/alert_rules/common.ts
View file

@ -8,13 +8,22 @@ import moment, { Moment } from 'moment';
import { isRight } from 'fp-ts/lib/Either';
import Mustache from 'mustache';
import { IBasePath } from '@kbn/core/server';
import { IRuleTypeAlerts, RuleExecutorServices } from '@kbn/alerting-plugin/server';
import {
IRuleTypeAlerts,
ActionGroupIdsOf,
AlertInstanceContext as AlertContext,
AlertInstanceState as AlertState,
} from '@kbn/alerting-plugin/server';
import { addSpaceIdToPath } from '@kbn/spaces-plugin/common';
import { i18n } from '@kbn/i18n';
import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils';
import { legacyExperimentalFieldMap, ObservabilityUptimeAlert } from '@kbn/alerts-as-data-utils';
import { PublicAlertsClient } from '@kbn/alerting-plugin/server/alerts_client/types';
import { combineFiltersAndUserSearch, stringifyKueries } from '../../common/lib';
import { SYNTHETICS_RULE_TYPES_ALERT_CONTEXT } from '../../common/constants/synthetics_alerts';
import {
MonitorStatusActionGroup,
SYNTHETICS_RULE_TYPES_ALERT_CONTEXT,
} from '../../common/constants/synthetics_alerts';
import { uptimeRuleFieldMap } from '../../common/rules/uptime_rule_field_map';
import {
getUptimeIndexPattern,
@ -26,7 +35,6 @@ import { getMonitorSummary } from './status_rule/message_utils';
import {
SyntheticsCommonState,
SyntheticsCommonStateCodec,
SyntheticsMonitorStatusAlertState,
} from '../../common/runtime_types/alert_rules/common';
import { getSyntheticsErrorRouteFromMonitorId } from '../../common/utils/get_synthetics_monitor_url';
import { ALERT_DETAILS_URL, RECOVERY_REASON } from './action_variables';
@ -154,30 +162,33 @@ export const getErrorDuration = (startedAt: Moment, endsAt: Moment) => {
};
export const setRecoveredAlertsContext = ({
alertFactory,
alertsClient,
basePath,
getAlertUuid,
spaceId,
staleDownConfigs,
upConfigs,
dateFormat,
tz,
}: {
alertFactory: RuleExecutorServices['alertFactory'];
alertsClient: PublicAlertsClient<
ObservabilityUptimeAlert,
AlertState,
AlertContext,
ActionGroupIdsOf<MonitorStatusActionGroup>
>;
basePath?: IBasePath;
getAlertUuid?: (alertId: string) => string | null;
spaceId?: string;
staleDownConfigs: AlertOverviewStatus['staleDownConfigs'];
upConfigs: AlertOverviewStatus['upConfigs'];
dateFormat: string;
tz: string;
}) => {
const { getRecoveredAlerts } = alertFactory.done();
for (const alert of getRecoveredAlerts()) {
const recoveredAlertId = alert.getId();
const alertUuid = getAlertUuid?.(recoveredAlertId) || undefined;
const recoveredAlerts = alertsClient.getRecoveredAlerts() ?? [];
for (const recoveredAlert of recoveredAlerts) {
const recoveredAlertId = recoveredAlert.alert.getId();
const alertUuid = recoveredAlert.alert.getUuid();
const state = alert.getState() as SyntheticsCommonState & SyntheticsMonitorStatusAlertState;
const state = recoveredAlert.alert.getState();
let recoveryReason = '';
let recoveryStatus = i18n.translate(
@ -279,7 +290,7 @@ export const setRecoveredAlertsContext = ({
}
}
alert.setContext({
const context = {
...state,
...(monitorSummary ? monitorSummary : {}),
lastErrorMessage,
@ -290,7 +301,8 @@ export const setRecoveredAlertsContext = ({
...(basePath && spaceId && alertUuid
? { [ALERT_DETAILS_URL]: getAlertDetailsUrl(basePath, spaceId, alertUuid) }
: {}),
});
};
alertsClient.setAlertData({ id: recoveredAlertId, context });
}
};

93
x-pack/plugins/observability_solution/synthetics/server/alert_rules/status_rule/monitor_status_rule.ts
View file

@ -8,9 +8,16 @@
import { DEFAULT_APP_CATEGORIES } from '@kbn/core/server';
import { isEmpty } from 'lodash';
import { ActionGroupIdsOf } from '@kbn/alerting-plugin/common';
import { GetViewInAppRelativeUrlFnOpts } from '@kbn/alerting-plugin/server';
import { PluginSetupContract } from '@kbn/alerting-plugin/server';
import {
GetViewInAppRelativeUrlFnOpts,
AlertInstanceContext as AlertContext,
RuleExecutorOptions,
AlertsClientError,
IRuleTypeAlerts,
} from '@kbn/alerting-plugin/server';
import { observabilityPaths } from '@kbn/observability-plugin/common';
import { createLifecycleRuleTypeFactory, IRuleDataClient } from '@kbn/rule-registry-plugin/server';
import { ObservabilityUptimeAlert } from '@kbn/alerts-as-data-utils';
import { SyntheticsPluginsSetupDependencies, SyntheticsServerSetup } from '../../types';
import { DOWN_LABEL, getMonitorAlertDocument, getMonitorSummary } from './message_utils';
import {
@ -19,7 +26,7 @@ import {
} from '../../../common/runtime_types/alert_rules/common';
import { OverviewStatus } from '../../../common/runtime_types';
import { StatusRuleExecutor } from './status_rule_executor';
import { StatusRulePramsSchema } from '../../../common/rules/status_rule';
import { StatusRulePramsSchema, StatusRuleParams } from '../../../common/rules/status_rule';
import {
MONITOR_STATUS,
SYNTHETICS_ALERT_RULE_TYPES,
@ -37,20 +44,26 @@ import { ALERT_DETAILS_URL, getActionVariables, VIEW_IN_APP_URL } from '../actio
import { STATUS_RULE_NAME } from '../translations';
import { SyntheticsMonitorClient } from '../../synthetics_service/synthetics_monitor/synthetics_monitor_client';
export type ActionGroupIds = ActionGroupIdsOf<typeof MONITOR_STATUS>;
type MonitorStatusRuleTypeParams = StatusRuleParams;
type MonitorStatusActionGroups = ActionGroupIdsOf<typeof MONITOR_STATUS>;
type MonitorStatusRuleTypeState = SyntheticsCommonState;
type MonitorStatusAlertState = SyntheticsMonitorStatusAlertState;
type MonitorStatusAlertContext = AlertContext;
type MonitorStatusAlert = ObservabilityUptimeAlert;
export const registerSyntheticsStatusCheckRule = (
server: SyntheticsServerSetup,
plugins: SyntheticsPluginsSetupDependencies,
syntheticsMonitorClient: SyntheticsMonitorClient,
ruleDataClient: IRuleDataClient
alerting: PluginSetupContract
) => {
const createLifecycleRuleType = createLifecycleRuleTypeFactory({
ruleDataClient,
logger: server.logger,
});
if (!alerting) {
throw new Error(
'Cannot register the synthetics monitor status rule type. The alerting plugin needs to be enabled.'
);
}
return createLifecycleRuleType({
alerting.registerType({
id: SYNTHETICS_ALERT_RULE_TYPES.MONITOR_STATUS,
category: DEFAULT_APP_CATEGORIES.observability.id,
producer: 'uptime',
@ -64,19 +77,22 @@ export const registerSyntheticsStatusCheckRule = (
isExportable: true,
minimumLicenseRequired: 'basic',
doesSetRecoveryContext: true,
async executor({ state, params, services, spaceId, previousStartedAt }) {
const ruleState = state as SyntheticsCommonState;
executor: async (
options: RuleExecutorOptions<
MonitorStatusRuleTypeParams,
MonitorStatusRuleTypeState,
MonitorStatusAlertState,
MonitorStatusAlertContext,
MonitorStatusActionGroups,
MonitorStatusAlert
>
) => {
const { state: ruleState, params, services, spaceId, previousStartedAt, startedAt } = options;
const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services;
if (!alertsClient) {
throw new AlertsClientError();
}
const { basePath } = server;
const {
alertFactory,
getAlertUuid,
savedObjectsClient,
scopedClusterClient,
alertWithLifecycle,
uiSettingsClient,
} = services;
const dateFormat = await uiSettingsClient.get('dateFormat');
const timezone = await uiSettingsClient.get('dateFormat:tz');
const tz = timezone === 'Browser' ? 'UTC' : timezone;
@ -106,13 +122,11 @@ export const registerSyntheticsStatusCheckRule = (
tz
);
const alert = alertWithLifecycle({
const { uuid, start } = alertsClient.report({
id: alertId,
fields: getMonitorAlertDocument(monitorSummary),
actionGroup: MONITOR_STATUS.id,
});
const alertUuid = getAlertUuid(alertId);
const alertState = alert.getState() as SyntheticsMonitorStatusAlertState;
const errorStartedAt: string = alertState.errorStartedAt || ping['@timestamp'];
const errorStartedAt = start ?? startedAt.toISOString();
let relativeViewInAppUrl = '';
if (monitorSummary.stateId) {
@ -123,31 +137,29 @@ export const registerSyntheticsStatusCheckRule = (
});
}
const payload = getMonitorAlertDocument(monitorSummary);
const context = {
...monitorSummary,
idWithLocation,
errorStartedAt,
linkMessage: monitorSummary.stateId
? getFullViewInAppMessage(basePath, spaceId, relativeViewInAppUrl)
: '',
[VIEW_IN_APP_URL]: getViewInAppUrl(basePath, spaceId, relativeViewInAppUrl),
[ALERT_DETAILS_URL]: getAlertDetailsUrl(basePath, spaceId, uuid),
};
alert.replaceState({
...updateState(ruleState, true),
...context,
idWithLocation,
});
alert.scheduleActions(MONITOR_STATUS.id, {
...context,
[ALERT_DETAILS_URL]: getAlertDetailsUrl(basePath, spaceId, alertUuid),
alertsClient.setAlertData({
id: alertId,
payload,
context,
});
});
setRecoveredAlertsContext({
alertFactory,
alertsClient,
basePath,
getAlertUuid,
spaceId,
staleDownConfigs,
upConfigs,
@ -159,7 +171,10 @@ export const registerSyntheticsStatusCheckRule = (
state: updateState(ruleState, !isEmpty(downConfigs), { downConfigs }),
};
},
alerts: UptimeRuleTypeAlertDefinition,
alerts: {
...UptimeRuleTypeAlertDefinition,
shouldWrite: true,
} as IRuleTypeAlerts<MonitorStatusAlert>,
getViewInAppRelativeUrl: ({ rule }: GetViewInAppRelativeUrlFnOpts<{}>) =>
observabilityPaths.ruleDetails(rule.id),
});

15
x-pack/plugins/observability_solution/synthetics/server/server.ts
View file

@ -138,18 +138,9 @@ export const initSyntheticsServer = (
}
});
const {
alerting: { registerType },
} = plugins;
const { alerting } = plugins;
const statusAlert = registerSyntheticsStatusCheckRule(
server,
plugins,
syntheticsMonitorClient,
ruleDataClient
);
registerType(statusAlert);
registerSyntheticsStatusCheckRule(server, plugins, syntheticsMonitorClient, alerting);
const tlsRule = registerSyntheticsTLSCheckRule(
server,
@ -158,5 +149,5 @@ export const initSyntheticsServer = (
ruleDataClient
);
registerType(tlsRule);
alerting.registerType(tlsRule);
};