mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 17:59:23 -04:00
Onboard Error Count Threshold rule type with FAAD (#179275)
Towards: https://github.com/elastic/kibana/issues/169867 This PR onboards the Error Count Threshold rule type with FAAD. ### To verify 1. Run the following script to generate APM data: ``` node scripts/synthtrace many_errors.ts --local --live ``` 2. Create an error count threshold rule. Example: ``` POST kbn:/api/alerting/rule { "params": { "threshold": 25, "windowSize": 5, "windowUnit": "m", "environment": "ENVIRONMENT_ALL" }, "consumer": "alerts", "schedule": { "interval": "1m" }, "tags": [], "name": "testinggg", "rule_type_id": "apm.error_rate", "notify_when": "onActionGroupChange", "actions": [] } ``` 3. Your rule should create an alert and should saved it in `.internal.alerts-observability.apm.alerts-default-000001` Example: ``` GET .internal.alerts-*/_search ``` 4. Recover the alert by setting `threshold: 10000` 5. The alert should be recovered and the AAD in the above index should be updated `kibana.alert.status: recovered`.
This commit is contained in:
Alexi Doak
GitHub
parent
2993eaea22
commit
3712fa8978
2 changed files with 751 additions and 429 deletions
|
|
@ -35,12 +35,11 @@ describe('Error count alert', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
await executor({ params });
|
await executor({ params });
|
||||||
expect(services.alertFactory.create).not.toBeCalled();
|
expect(services.alertsClient.report).not.toBeCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('sends alerts with service name and environment for those that exceeded the threshold', async () => {
|
it('sends alerts with service name and environment for those that exceeded the threshold', async () => {
|
||||||
const { services, dependencies, executor, scheduleActions } =
|
const { services, dependencies, executor } = createRuleTypeMocks();
|
||||||
createRuleTypeMocks();
|
|
||||||
|
|
||||||
registerErrorCountRuleType(dependencies);
|
registerErrorCountRuleType(dependencies);
|
||||||
|
|
||||||
|
|
@ -129,55 +128,100 @@ describe('Error count alert', () => {
|
||||||
total: 1,
|
total: 1,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
services.alertsClient.report.mockReturnValue({ uuid: 'test-uuid' });
|
||||||
|
|
||||||
await executor({ params });
|
await executor({ params });
|
||||||
['foo_env-foo', 'foo_env-foo-2', 'bar_env-bar'].forEach((instanceName) =>
|
['foo_env-foo', 'foo_env-foo-2', 'bar_env-bar'].forEach((instanceName) =>
|
||||||
expect(services.alertFactory.create).toHaveBeenCalledWith(instanceName)
|
expect(services.alertsClient.report).toHaveBeenCalledWith({
|
||||||
|
actionGroup: 'threshold_met',
|
||||||
|
id: instanceName,
|
||||||
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledTimes(3);
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledTimes(3);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo',
|
||||||
triggerValue: 5,
|
errorGroupingKey: undefined,
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 5 in the last 5 mins for service: foo, env: env-foo. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 5,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 5,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo-2',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo-2',
|
||||||
triggerValue: 4,
|
errorGroupingKey: undefined,
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo-2',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 4,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo-2',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo-2',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 4,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo-2',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'bar',
|
context: {
|
||||||
environment: 'env-bar',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
reason:
|
environment: 'env-bar',
|
||||||
'Error count is 3 in the last 5 mins for service: bar, env: env-bar. Alert when > 2.',
|
errorGroupingKey: undefined,
|
||||||
threshold: 2,
|
interval: '5 mins',
|
||||||
triggerValue: 3,
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'bar',
|
||||||
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 3,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
||||||
|
},
|
||||||
|
id: 'bar_env-bar',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 3,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-bar',
|
||||||
|
'service.name': 'bar',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('sends alert when rule is configured with group by on transaction.name', async () => {
|
it('sends alert when rule is configured with group by on transaction.name', async () => {
|
||||||
const { services, dependencies, executor, scheduleActions } =
|
const { services, dependencies, executor } = createRuleTypeMocks();
|
||||||
createRuleTypeMocks();
|
|
||||||
|
|
||||||
registerErrorCountRuleType(dependencies);
|
registerErrorCountRuleType(dependencies);
|
||||||
|
|
||||||
|
|
@ -227,6 +271,7 @@ describe('Error count alert', () => {
|
||||||
total: 1,
|
total: 1,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
services.alertsClient.report.mockReturnValue({ uuid: 'test-uuid' });
|
||||||
|
|
||||||
await executor({ params });
|
await executor({ params });
|
||||||
[
|
[
|
||||||
|
|
@ -234,55 +279,102 @@ describe('Error count alert', () => {
|
||||||
'foo_env-foo-2_tx-name-foo-2',
|
'foo_env-foo-2_tx-name-foo-2',
|
||||||
'bar_env-bar_tx-name-bar',
|
'bar_env-bar_tx-name-bar',
|
||||||
].forEach((instanceName) =>
|
].forEach((instanceName) =>
|
||||||
expect(services.alertFactory.create).toHaveBeenCalledWith(instanceName)
|
expect(services.alertsClient.report).toHaveBeenCalledWith({
|
||||||
|
actionGroup: 'threshold_met',
|
||||||
|
id: instanceName,
|
||||||
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledTimes(3);
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledTimes(3);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo',
|
||||||
triggerValue: 5,
|
errorGroupingKey: undefined,
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 5 in the last 5 mins for service: foo, env: env-foo, name: tx-name-foo. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo, name: tx-name-foo. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
transactionName: 'tx-name-foo',
|
||||||
transactionName: 'tx-name-foo',
|
triggerValue: 5,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo_tx-name-foo',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 5,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo, name: tx-name-foo. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo',
|
||||||
|
'service.name': 'foo',
|
||||||
|
'transaction.name': 'tx-name-foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo-2',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo-2',
|
||||||
triggerValue: 4,
|
errorGroupingKey: undefined,
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2, name: tx-name-foo-2. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2, name: tx-name-foo-2. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo-2',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
transactionName: 'tx-name-foo-2',
|
||||||
transactionName: 'tx-name-foo-2',
|
triggerValue: 4,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo-2',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo-2_tx-name-foo-2',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 4,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2, name: tx-name-foo-2. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo-2',
|
||||||
|
'service.name': 'foo',
|
||||||
|
'transaction.name': 'tx-name-foo-2',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'bar',
|
context: {
|
||||||
environment: 'env-bar',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
reason:
|
environment: 'env-bar',
|
||||||
'Error count is 3 in the last 5 mins for service: bar, env: env-bar, name: tx-name-bar. Alert when > 2.',
|
errorGroupingKey: undefined,
|
||||||
threshold: 2,
|
interval: '5 mins',
|
||||||
triggerValue: 3,
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar, name: tx-name-bar. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'bar',
|
||||||
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
transactionName: 'tx-name-bar',
|
||||||
transactionName: 'tx-name-bar',
|
triggerValue: 3,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
||||||
|
},
|
||||||
|
id: 'bar_env-bar_tx-name-bar',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 3,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar, name: tx-name-bar. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-bar',
|
||||||
|
'service.name': 'bar',
|
||||||
|
'transaction.name': 'tx-name-bar',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('sends alert when rule is configured with group by on error.grouping_key', async () => {
|
it('sends alert when rule is configured with group by on error.grouping_key', async () => {
|
||||||
const { services, dependencies, executor, scheduleActions } =
|
const { services, dependencies, executor } = createRuleTypeMocks();
|
||||||
createRuleTypeMocks();
|
|
||||||
|
|
||||||
registerErrorCountRuleType(dependencies);
|
registerErrorCountRuleType(dependencies);
|
||||||
|
|
||||||
|
|
@ -332,6 +424,7 @@ describe('Error count alert', () => {
|
||||||
total: 1,
|
total: 1,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
services.alertsClient.report.mockReturnValue({ uuid: 'test-uuid' });
|
||||||
|
|
||||||
await executor({ params });
|
await executor({ params });
|
||||||
[
|
[
|
||||||
|
|
@ -339,55 +432,96 @@ describe('Error count alert', () => {
|
||||||
'foo_env-foo-2_error-key-foo-2',
|
'foo_env-foo-2_error-key-foo-2',
|
||||||
'bar_env-bar_error-key-bar',
|
'bar_env-bar_error-key-bar',
|
||||||
].forEach((instanceName) =>
|
].forEach((instanceName) =>
|
||||||
expect(services.alertFactory.create).toHaveBeenCalledWith(instanceName)
|
expect(services.alertsClient.report).toHaveBeenCalledWith({
|
||||||
|
actionGroup: 'threshold_met',
|
||||||
|
id: instanceName,
|
||||||
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledTimes(3);
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledTimes(3);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo',
|
||||||
triggerValue: 5,
|
errorGroupingKey: 'error-key-foo',
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 5 in the last 5 mins for service: foo, env: env-foo, error key: error-key-foo. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo, error key: error-key-foo. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 5,
|
||||||
errorGroupingKey: 'error-key-foo',
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo_error-key-foo',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': 'error-key-foo',
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 5,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo, error key: error-key-foo. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo-2',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo-2',
|
||||||
triggerValue: 4,
|
errorGroupingKey: 'error-key-foo-2',
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2, error key: error-key-foo-2. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2, error key: error-key-foo-2. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo-2',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 4,
|
||||||
errorGroupingKey: 'error-key-foo-2',
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo-2',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo-2_error-key-foo-2',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': 'error-key-foo-2',
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 4,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2, error key: error-key-foo-2. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo-2',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'bar',
|
context: {
|
||||||
environment: 'env-bar',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
reason:
|
environment: 'env-bar',
|
||||||
'Error count is 3 in the last 5 mins for service: bar, env: env-bar, error key: error-key-bar. Alert when > 2.',
|
errorGroupingKey: 'error-key-bar',
|
||||||
threshold: 2,
|
interval: '5 mins',
|
||||||
triggerValue: 3,
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar, error key: error-key-bar. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'bar',
|
||||||
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 3,
|
||||||
errorGroupingKey: 'error-key-bar',
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
||||||
|
},
|
||||||
|
id: 'bar_env-bar_error-key-bar',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': 'error-key-bar',
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 3,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar, error key: error-key-bar. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-bar',
|
||||||
|
'service.name': 'bar',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('sends alert when rule is configured with preselected group by', async () => {
|
it('sends alert when rule is configured with preselected group by', async () => {
|
||||||
const { services, dependencies, executor, scheduleActions } =
|
const { services, dependencies, executor } = createRuleTypeMocks();
|
||||||
createRuleTypeMocks();
|
|
||||||
|
|
||||||
registerErrorCountRuleType(dependencies);
|
registerErrorCountRuleType(dependencies);
|
||||||
|
|
||||||
|
|
@ -437,55 +571,100 @@ describe('Error count alert', () => {
|
||||||
total: 1,
|
total: 1,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
services.alertsClient.report.mockReturnValue({ uuid: 'test-uuid' });
|
||||||
|
|
||||||
await executor({ params });
|
await executor({ params });
|
||||||
['foo_env-foo', 'foo_env-foo-2', 'bar_env-bar'].forEach((instanceName) =>
|
['foo_env-foo', 'foo_env-foo-2', 'bar_env-bar'].forEach((instanceName) =>
|
||||||
expect(services.alertFactory.create).toHaveBeenCalledWith(instanceName)
|
expect(services.alertsClient.report).toHaveBeenCalledWith({
|
||||||
|
actionGroup: 'threshold_met',
|
||||||
|
id: instanceName,
|
||||||
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledTimes(3);
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledTimes(3);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo',
|
||||||
triggerValue: 5,
|
errorGroupingKey: undefined,
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 5 in the last 5 mins for service: foo, env: env-foo. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 5,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 5,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo-2',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo-2',
|
||||||
triggerValue: 4,
|
errorGroupingKey: undefined,
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo-2',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 4,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo-2',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo-2',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 4,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo-2',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'bar',
|
context: {
|
||||||
environment: 'env-bar',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
reason:
|
environment: 'env-bar',
|
||||||
'Error count is 3 in the last 5 mins for service: bar, env: env-bar. Alert when > 2.',
|
errorGroupingKey: undefined,
|
||||||
threshold: 2,
|
interval: '5 mins',
|
||||||
triggerValue: 3,
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'bar',
|
||||||
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 3,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
||||||
|
},
|
||||||
|
id: 'bar_env-bar',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 3,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-bar',
|
||||||
|
'service.name': 'bar',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('sends alert when service.environment field does not exist in the source', async () => {
|
it('sends alert when service.environment field does not exist in the source', async () => {
|
||||||
const { services, dependencies, executor, scheduleActions } =
|
const { services, dependencies, executor } = createRuleTypeMocks();
|
||||||
createRuleTypeMocks();
|
|
||||||
|
|
||||||
registerErrorCountRuleType(dependencies);
|
registerErrorCountRuleType(dependencies);
|
||||||
|
|
||||||
|
|
@ -535,6 +714,7 @@ describe('Error count alert', () => {
|
||||||
total: 1,
|
total: 1,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
services.alertsClient.report.mockReturnValue({ uuid: 'test-uuid' });
|
||||||
|
|
||||||
await executor({ params });
|
await executor({ params });
|
||||||
[
|
[
|
||||||
|
|
@ -542,52 +722,96 @@ describe('Error count alert', () => {
|
||||||
'foo_ENVIRONMENT_NOT_DEFINED',
|
'foo_ENVIRONMENT_NOT_DEFINED',
|
||||||
'bar_env-bar',
|
'bar_env-bar',
|
||||||
].forEach((instanceName) =>
|
].forEach((instanceName) =>
|
||||||
expect(services.alertFactory.create).toHaveBeenCalledWith(instanceName)
|
expect(services.alertsClient.report).toHaveBeenCalledWith({
|
||||||
|
actionGroup: 'threshold_met',
|
||||||
|
id: instanceName,
|
||||||
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledTimes(3);
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledTimes(3);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'Not defined',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'Not defined',
|
||||||
triggerValue: 5,
|
errorGroupingKey: undefined,
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 5 in the last 5 mins for service: foo, env: Not defined. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 5 in the last 5 mins for service: foo, env: Not defined. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_ALL',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 5,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_ALL',
|
||||||
|
},
|
||||||
|
id: 'foo_ENVIRONMENT_NOT_DEFINED',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 5,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 5 in the last 5 mins for service: foo, env: Not defined. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'ENVIRONMENT_NOT_DEFINED',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'Not defined',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'Not defined',
|
||||||
triggerValue: 4,
|
errorGroupingKey: undefined,
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 4 in the last 5 mins for service: foo, env: Not defined. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 4 in the last 5 mins for service: foo, env: Not defined. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_ALL',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 4,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=ENVIRONMENT_ALL',
|
||||||
|
},
|
||||||
|
id: 'foo_ENVIRONMENT_NOT_DEFINED',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 4,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 4 in the last 5 mins for service: foo, env: Not defined. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'ENVIRONMENT_NOT_DEFINED',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'bar',
|
context: {
|
||||||
environment: 'env-bar',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
reason:
|
environment: 'env-bar',
|
||||||
'Error count is 3 in the last 5 mins for service: bar, env: env-bar. Alert when > 2.',
|
errorGroupingKey: undefined,
|
||||||
threshold: 2,
|
interval: '5 mins',
|
||||||
triggerValue: 3,
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'bar',
|
||||||
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 3,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
||||||
|
},
|
||||||
|
id: 'bar_env-bar',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 3,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-bar',
|
||||||
|
'service.name': 'bar',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('sends alert when rule is configured with group by on error.grouping_key and error.grouping_name', async () => {
|
it('sends alert when rule is configured with group by on error.grouping_key and error.grouping_name', async () => {
|
||||||
const { services, dependencies, executor, scheduleActions } =
|
const { services, dependencies, executor } = createRuleTypeMocks();
|
||||||
createRuleTypeMocks();
|
|
||||||
|
|
||||||
registerErrorCountRuleType(dependencies);
|
registerErrorCountRuleType(dependencies);
|
||||||
|
|
||||||
|
|
@ -642,6 +866,7 @@ describe('Error count alert', () => {
|
||||||
total: 1,
|
total: 1,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
services.alertsClient.report.mockReturnValue({ uuid: 'test-uuid' });
|
||||||
|
|
||||||
await executor({ params });
|
await executor({ params });
|
||||||
[
|
[
|
||||||
|
|
@ -649,58 +874,102 @@ describe('Error count alert', () => {
|
||||||
'foo_env-foo-2_error-key-foo-2_error-name-foo2',
|
'foo_env-foo-2_error-key-foo-2_error-name-foo2',
|
||||||
'bar_env-bar_error-key-bar_error-name-bar',
|
'bar_env-bar_error-key-bar_error-name-bar',
|
||||||
].forEach((instanceName) =>
|
].forEach((instanceName) =>
|
||||||
expect(services.alertFactory.create).toHaveBeenCalledWith(instanceName)
|
expect(services.alertsClient.report).toHaveBeenCalledWith({
|
||||||
|
actionGroup: 'threshold_met',
|
||||||
|
id: instanceName,
|
||||||
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledTimes(3);
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledTimes(3);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo',
|
||||||
triggerValue: 5,
|
errorGroupingKey: 'error-key-foo',
|
||||||
reason:
|
errorGroupingName: 'error-name-foo',
|
||||||
'Error count is 5 in the last 5 mins for service: foo, env: env-foo, error key: error-key-foo, error name: error-name-foo. Alert when > 2.',
|
interval: '5 mins',
|
||||||
interval: '5 mins',
|
reason:
|
||||||
viewInAppUrl:
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo, error key: error-key-foo, error name: error-name-foo. Alert when > 2.',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
serviceName: 'foo',
|
||||||
errorGroupingKey: 'error-key-foo',
|
threshold: 2,
|
||||||
errorGroupingName: 'error-name-foo',
|
triggerValue: 5,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo_error-key-foo_error-name-foo',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': 'error-key-foo',
|
||||||
|
'error.grouping_name': 'error-name-foo',
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 5,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo, error key: error-key-foo, error name: error-name-foo. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo-2',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo-2',
|
||||||
triggerValue: 4,
|
errorGroupingKey: 'error-key-foo-2',
|
||||||
reason:
|
errorGroupingName: 'error-name-foo2',
|
||||||
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2, error key: error-key-foo-2, error name: error-name-foo2. Alert when > 2.',
|
interval: '5 mins',
|
||||||
interval: '5 mins',
|
reason:
|
||||||
viewInAppUrl:
|
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2, error key: error-key-foo-2, error name: error-name-foo2. Alert when > 2.',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo-2',
|
serviceName: 'foo',
|
||||||
errorGroupingKey: 'error-key-foo-2',
|
threshold: 2,
|
||||||
errorGroupingName: 'error-name-foo2',
|
triggerValue: 4,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo-2',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo-2_error-key-foo-2_error-name-foo2',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': 'error-key-foo-2',
|
||||||
|
'error.grouping_name': 'error-name-foo2',
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 4,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 4 in the last 5 mins for service: foo, env: env-foo-2, error key: error-key-foo-2, error name: error-name-foo2. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo-2',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'bar',
|
context: {
|
||||||
environment: 'env-bar',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
reason:
|
environment: 'env-bar',
|
||||||
'Error count is 3 in the last 5 mins for service: bar, env: env-bar, error key: error-key-bar, error name: error-name-bar. Alert when > 2.',
|
errorGroupingKey: 'error-key-bar',
|
||||||
threshold: 2,
|
errorGroupingName: 'error-name-bar',
|
||||||
triggerValue: 3,
|
interval: '5 mins',
|
||||||
interval: '5 mins',
|
reason:
|
||||||
viewInAppUrl:
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar, error key: error-key-bar, error name: error-name-bar. Alert when > 2.',
|
||||||
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
serviceName: 'bar',
|
||||||
errorGroupingKey: 'error-key-bar',
|
threshold: 2,
|
||||||
errorGroupingName: 'error-name-bar',
|
triggerValue: 3,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/bar/errors?environment=env-bar',
|
||||||
|
},
|
||||||
|
id: 'bar_env-bar_error-key-bar_error-name-bar',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': 'error-key-bar',
|
||||||
|
'error.grouping_name': 'error-name-bar',
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 3,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 3 in the last 5 mins for service: bar, env: env-bar, error key: error-key-bar, error name: error-name-bar. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-bar',
|
||||||
|
'service.name': 'bar',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('sends alert when rule is configured with a filter query', async () => {
|
it('sends alert when rule is configured with a filter query', async () => {
|
||||||
const { services, dependencies, executor, scheduleActions } =
|
const { services, dependencies, executor } = createRuleTypeMocks();
|
||||||
createRuleTypeMocks();
|
|
||||||
|
|
||||||
registerErrorCountRuleType(dependencies);
|
registerErrorCountRuleType(dependencies);
|
||||||
|
|
||||||
|
|
@ -745,25 +1014,43 @@ describe('Error count alert', () => {
|
||||||
total: 1,
|
total: 1,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
services.alertsClient.report.mockReturnValue({ uuid: 'test-uuid' });
|
||||||
|
|
||||||
await executor({ params });
|
await executor({ params });
|
||||||
['foo_env-foo'].forEach((instanceName) =>
|
['foo_env-foo'].forEach((instanceName) =>
|
||||||
expect(services.alertFactory.create).toHaveBeenCalledWith(instanceName)
|
expect(services.alertsClient.report).toHaveBeenCalledWith({
|
||||||
|
actionGroup: 'threshold_met',
|
||||||
|
id: instanceName,
|
||||||
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledTimes(1);
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
expect(scheduleActions).toHaveBeenCalledWith('threshold_met', {
|
expect(services.alertsClient.setAlertData).toHaveBeenCalledWith({
|
||||||
serviceName: 'foo',
|
context: {
|
||||||
environment: 'env-foo',
|
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
||||||
threshold: 2,
|
environment: 'env-foo',
|
||||||
triggerValue: 5,
|
errorGroupingKey: undefined,
|
||||||
reason:
|
interval: '5 mins',
|
||||||
'Error count is 5 in the last 5 mins for service: foo, env: env-foo. Alert when > 2.',
|
reason:
|
||||||
interval: '5 mins',
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo. Alert when > 2.',
|
||||||
viewInAppUrl:
|
serviceName: 'foo',
|
||||||
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
threshold: 2,
|
||||||
alertDetailsUrl: 'mockedAlertsLocator > getLocation',
|
triggerValue: 5,
|
||||||
|
viewInAppUrl:
|
||||||
|
'http://localhost:5601/eyr/app/apm/services/foo/errors?environment=env-foo',
|
||||||
|
},
|
||||||
|
id: 'foo_env-foo',
|
||||||
|
payload: {
|
||||||
|
'error.grouping_key': undefined,
|
||||||
|
'kibana.alert.evaluation.threshold': 2,
|
||||||
|
'kibana.alert.evaluation.value': 5,
|
||||||
|
'kibana.alert.reason':
|
||||||
|
'Error count is 5 in the last 5 mins for service: foo, env: env-foo. Alert when > 2.',
|
||||||
|
'processor.event': 'error',
|
||||||
|
'service.environment': 'env-foo',
|
||||||
|
'service.name': 'foo',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,16 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { DEFAULT_APP_CATEGORIES } from '@kbn/core/server';
|
import { DEFAULT_APP_CATEGORIES } from '@kbn/core/server';
|
||||||
import { GetViewInAppRelativeUrlFnOpts } from '@kbn/alerting-plugin/server';
|
import {
|
||||||
|
GetViewInAppRelativeUrlFnOpts,
|
||||||
|
ActionGroupIdsOf,
|
||||||
|
AlertInstanceContext as AlertContext,
|
||||||
|
AlertInstanceState as AlertState,
|
||||||
|
RuleTypeState,
|
||||||
|
RuleExecutorOptions,
|
||||||
|
AlertsClientError,
|
||||||
|
IRuleTypeAlerts,
|
||||||
|
} from '@kbn/alerting-plugin/server';
|
||||||
import {
|
import {
|
||||||
formatDurationFromTimeUnitChar,
|
formatDurationFromTimeUnitChar,
|
||||||
getAlertUrl,
|
getAlertUrl,
|
||||||
|
|
@ -20,7 +29,7 @@ import {
|
||||||
ALERT_REASON,
|
ALERT_REASON,
|
||||||
ApmRuleType,
|
ApmRuleType,
|
||||||
} from '@kbn/rule-data-utils';
|
} from '@kbn/rule-data-utils';
|
||||||
import { createLifecycleRuleTypeFactory } from '@kbn/rule-registry-plugin/server';
|
import { ObservabilityApmAlert } from '@kbn/alerts-as-data-utils';
|
||||||
import {
|
import {
|
||||||
getParsedFilterQuery,
|
getParsedFilterQuery,
|
||||||
termQuery,
|
termQuery,
|
||||||
|
|
@ -38,8 +47,12 @@ import {
|
||||||
APM_SERVER_FEATURE_ID,
|
APM_SERVER_FEATURE_ID,
|
||||||
formatErrorCountReason,
|
formatErrorCountReason,
|
||||||
RULE_TYPES_CONFIG,
|
RULE_TYPES_CONFIG,
|
||||||
|
THRESHOLD_MET_GROUP,
|
||||||
} from '../../../../../common/rules/apm_rule_types';
|
} from '../../../../../common/rules/apm_rule_types';
|
||||||
import { errorCountParamsSchema } from '../../../../../common/rules/schema';
|
import {
|
||||||
|
errorCountParamsSchema,
|
||||||
|
ApmRuleParamsType,
|
||||||
|
} from '../../../../../common/rules/schema';
|
||||||
import { environmentQuery } from '../../../../../common/utils/environment_query';
|
import { environmentQuery } from '../../../../../common/utils/environment_query';
|
||||||
import { getAlertUrlErrorCount } from '../../../../../common/utils/formatters';
|
import { getAlertUrlErrorCount } from '../../../../../common/utils/formatters';
|
||||||
import { apmActionVariables } from '../../action_variables';
|
import { apmActionVariables } from '../../action_variables';
|
||||||
|
|
@ -72,6 +85,13 @@ export const errorCountActionVariables = [
|
||||||
apmActionVariables.viewInAppUrl,
|
apmActionVariables.viewInAppUrl,
|
||||||
];
|
];
|
||||||
|
|
||||||
|
type ErrorCountRuleTypeParams = ApmRuleParamsType[ApmRuleType.ErrorCount];
|
||||||
|
type ErrorCountActionGroups = ActionGroupIdsOf<typeof THRESHOLD_MET_GROUP>;
|
||||||
|
type ErrorCountRuleTypeState = RuleTypeState;
|
||||||
|
type ErrorCountAlertState = AlertState;
|
||||||
|
type ErrorCountAlertContext = AlertContext;
|
||||||
|
type ErrorCountAlert = ObservabilityApmAlert;
|
||||||
|
|
||||||
export function registerErrorCountRuleType({
|
export function registerErrorCountRuleType({
|
||||||
alerting,
|
alerting,
|
||||||
alertsLocator,
|
alertsLocator,
|
||||||
|
|
@ -80,204 +100,219 @@ export function registerErrorCountRuleType({
|
||||||
logger,
|
logger,
|
||||||
ruleDataClient,
|
ruleDataClient,
|
||||||
}: RegisterRuleDependencies) {
|
}: RegisterRuleDependencies) {
|
||||||
const createLifecycleRuleType = createLifecycleRuleTypeFactory({
|
if (!alerting) {
|
||||||
ruleDataClient,
|
throw new Error(
|
||||||
logger,
|
'Cannot register error count rule type. The alerting plugin needs to be enabled.'
|
||||||
});
|
);
|
||||||
|
}
|
||||||
alerting.registerType(
|
alerting.registerType({
|
||||||
createLifecycleRuleType({
|
id: ApmRuleType.ErrorCount,
|
||||||
id: ApmRuleType.ErrorCount,
|
name: ruleTypeConfig.name,
|
||||||
name: ruleTypeConfig.name,
|
actionGroups: ruleTypeConfig.actionGroups,
|
||||||
actionGroups: ruleTypeConfig.actionGroups,
|
defaultActionGroupId: ruleTypeConfig.defaultActionGroupId,
|
||||||
defaultActionGroupId: ruleTypeConfig.defaultActionGroupId,
|
validate: { params: errorCountParamsSchema },
|
||||||
validate: { params: errorCountParamsSchema },
|
schemas: {
|
||||||
schemas: {
|
params: {
|
||||||
params: {
|
type: 'config-schema',
|
||||||
type: 'config-schema',
|
schema: errorCountParamsSchema,
|
||||||
schema: errorCountParamsSchema,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
actionVariables: {
|
},
|
||||||
context: errorCountActionVariables,
|
actionVariables: {
|
||||||
},
|
context: errorCountActionVariables,
|
||||||
category: DEFAULT_APP_CATEGORIES.observability.id,
|
},
|
||||||
producer: APM_SERVER_FEATURE_ID,
|
category: DEFAULT_APP_CATEGORIES.observability.id,
|
||||||
minimumLicenseRequired: 'basic',
|
producer: APM_SERVER_FEATURE_ID,
|
||||||
isExportable: true,
|
minimumLicenseRequired: 'basic',
|
||||||
executor: async ({
|
isExportable: true,
|
||||||
|
executor: async (
|
||||||
|
options: RuleExecutorOptions<
|
||||||
|
ErrorCountRuleTypeParams,
|
||||||
|
ErrorCountRuleTypeState,
|
||||||
|
ErrorCountAlertState,
|
||||||
|
ErrorCountAlertContext,
|
||||||
|
ErrorCountActionGroups,
|
||||||
|
ErrorCountAlert
|
||||||
|
>
|
||||||
|
) => {
|
||||||
|
const {
|
||||||
params: ruleParams,
|
params: ruleParams,
|
||||||
services,
|
services,
|
||||||
spaceId,
|
spaceId,
|
||||||
startedAt,
|
startedAt,
|
||||||
getTimeRange,
|
getTimeRange,
|
||||||
}) => {
|
} = options;
|
||||||
const allGroupByFields = getAllGroupByFields(
|
const { alertsClient, savedObjectsClient, scopedClusterClient } =
|
||||||
ApmRuleType.ErrorCount,
|
services;
|
||||||
ruleParams.groupBy
|
if (!alertsClient) {
|
||||||
);
|
throw new AlertsClientError();
|
||||||
|
}
|
||||||
|
|
||||||
const {
|
const allGroupByFields = getAllGroupByFields(
|
||||||
getAlertUuid,
|
ApmRuleType.ErrorCount,
|
||||||
getAlertStartedDate,
|
ruleParams.groupBy
|
||||||
savedObjectsClient,
|
);
|
||||||
scopedClusterClient,
|
|
||||||
} = services;
|
|
||||||
|
|
||||||
const indices = await getApmIndices(savedObjectsClient);
|
const indices = await getApmIndices(savedObjectsClient);
|
||||||
|
|
||||||
const termFilterQuery = !ruleParams.searchConfiguration?.query?.query
|
const termFilterQuery = !ruleParams.searchConfiguration?.query?.query
|
||||||
? [
|
? [
|
||||||
...termQuery(SERVICE_NAME, ruleParams.serviceName, {
|
...termQuery(SERVICE_NAME, ruleParams.serviceName, {
|
||||||
queryEmptyString: false,
|
queryEmptyString: false,
|
||||||
}),
|
}),
|
||||||
...termQuery(ERROR_GROUP_ID, ruleParams.errorGroupingKey, {
|
...termQuery(ERROR_GROUP_ID, ruleParams.errorGroupingKey, {
|
||||||
queryEmptyString: false,
|
queryEmptyString: false,
|
||||||
}),
|
}),
|
||||||
...environmentQuery(ruleParams.environment),
|
...environmentQuery(ruleParams.environment),
|
||||||
]
|
]
|
||||||
: [];
|
: [];
|
||||||
|
|
||||||
const { dateStart } = getTimeRange(
|
const { dateStart } = getTimeRange(
|
||||||
`${ruleParams.windowSize}${ruleParams.windowUnit}`
|
`${ruleParams.windowSize}${ruleParams.windowUnit}`
|
||||||
);
|
);
|
||||||
|
|
||||||
const searchParams = {
|
const searchParams = {
|
||||||
index: indices.error,
|
index: indices.error,
|
||||||
body: {
|
body: {
|
||||||
track_total_hits: false,
|
track_total_hits: false,
|
||||||
size: 0,
|
size: 0,
|
||||||
query: {
|
query: {
|
||||||
bool: {
|
bool: {
|
||||||
filter: [
|
filter: [
|
||||||
{
|
{
|
||||||
range: {
|
range: {
|
||||||
'@timestamp': {
|
'@timestamp': {
|
||||||
gte: dateStart,
|
gte: dateStart,
|
||||||
},
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{ term: { [PROCESSOR_EVENT]: ProcessorEvent.error } },
|
|
||||||
...termFilterQuery,
|
|
||||||
...getParsedFilterQuery(
|
|
||||||
ruleParams.searchConfiguration?.query?.query as string
|
|
||||||
),
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
aggs: {
|
|
||||||
error_counts: {
|
|
||||||
multi_terms: {
|
|
||||||
terms: getGroupByTerms(allGroupByFields),
|
|
||||||
size: 1000,
|
|
||||||
order: { _count: 'desc' as const },
|
|
||||||
},
|
},
|
||||||
aggs: getServiceGroupFieldsAgg(),
|
{ term: { [PROCESSOR_EVENT]: ProcessorEvent.error } },
|
||||||
},
|
...termFilterQuery,
|
||||||
|
...getParsedFilterQuery(
|
||||||
|
ruleParams.searchConfiguration?.query?.query as string
|
||||||
|
),
|
||||||
|
],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
aggs: {
|
||||||
|
error_counts: {
|
||||||
const response = await alertingEsClient({
|
multi_terms: {
|
||||||
scopedClusterClient,
|
terms: getGroupByTerms(allGroupByFields),
|
||||||
params: searchParams,
|
size: 1000,
|
||||||
});
|
order: { _count: 'desc' as const },
|
||||||
|
|
||||||
const errorCountResults =
|
|
||||||
response.aggregations?.error_counts.buckets.map((bucket) => {
|
|
||||||
const groupByFields = bucket.key.reduce(
|
|
||||||
(obj, bucketKey, bucketIndex) => {
|
|
||||||
obj[allGroupByFields[bucketIndex]] = bucketKey;
|
|
||||||
return obj;
|
|
||||||
},
|
},
|
||||||
{} as Record<string, string>
|
aggs: getServiceGroupFieldsAgg(),
|
||||||
);
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
const bucketKey = bucket.key;
|
const response = await alertingEsClient({
|
||||||
|
scopedClusterClient,
|
||||||
|
params: searchParams,
|
||||||
|
});
|
||||||
|
|
||||||
return {
|
const errorCountResults =
|
||||||
errorCount: bucket.doc_count,
|
response.aggregations?.error_counts.buckets.map((bucket) => {
|
||||||
sourceFields: getServiceGroupFields(bucket),
|
const groupByFields = bucket.key.reduce(
|
||||||
groupByFields,
|
(obj, bucketKey, bucketIndex) => {
|
||||||
bucketKey,
|
obj[allGroupByFields[bucketIndex]] = bucketKey;
|
||||||
};
|
return obj;
|
||||||
}) ?? [];
|
},
|
||||||
|
{} as Record<string, string>
|
||||||
|
);
|
||||||
|
|
||||||
await asyncForEach(
|
const bucketKey = bucket.key;
|
||||||
errorCountResults.filter(
|
|
||||||
(result) => result.errorCount >= ruleParams.threshold
|
|
||||||
),
|
|
||||||
async (result) => {
|
|
||||||
const { errorCount, sourceFields, groupByFields, bucketKey } =
|
|
||||||
result;
|
|
||||||
const alertId = bucketKey.join('_');
|
|
||||||
const alertReason = formatErrorCountReason({
|
|
||||||
threshold: ruleParams.threshold,
|
|
||||||
measured: errorCount,
|
|
||||||
windowSize: ruleParams.windowSize,
|
|
||||||
windowUnit: ruleParams.windowUnit,
|
|
||||||
groupByFields,
|
|
||||||
});
|
|
||||||
|
|
||||||
const alert = services.alertWithLifecycle({
|
return {
|
||||||
id: alertId,
|
errorCount: bucket.doc_count,
|
||||||
fields: {
|
sourceFields: getServiceGroupFields(bucket),
|
||||||
[PROCESSOR_EVENT]: ProcessorEvent.error,
|
groupByFields,
|
||||||
[ALERT_EVALUATION_VALUE]: errorCount,
|
bucketKey,
|
||||||
[ALERT_EVALUATION_THRESHOLD]: ruleParams.threshold,
|
};
|
||||||
[ERROR_GROUP_ID]: ruleParams.errorGroupingKey,
|
}) ?? [];
|
||||||
[ALERT_REASON]: alertReason,
|
|
||||||
...sourceFields,
|
|
||||||
...groupByFields,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
const relativeViewInAppUrl = getAlertUrlErrorCount(
|
await asyncForEach(
|
||||||
groupByFields[SERVICE_NAME],
|
errorCountResults.filter(
|
||||||
getEnvironmentEsField(groupByFields[SERVICE_ENVIRONMENT])?.[
|
(result) => result.errorCount >= ruleParams.threshold
|
||||||
SERVICE_ENVIRONMENT
|
),
|
||||||
]
|
async (result) => {
|
||||||
);
|
const { errorCount, sourceFields, groupByFields, bucketKey } = result;
|
||||||
const viewInAppUrl = addSpaceIdToPath(
|
const alertId = bucketKey.join('_');
|
||||||
basePath.publicBaseUrl,
|
const alertReason = formatErrorCountReason({
|
||||||
spaceId,
|
threshold: ruleParams.threshold,
|
||||||
relativeViewInAppUrl
|
measured: errorCount,
|
||||||
);
|
windowSize: ruleParams.windowSize,
|
||||||
const indexedStartedAt =
|
windowUnit: ruleParams.windowUnit,
|
||||||
getAlertStartedDate(alertId) ?? startedAt.toISOString();
|
groupByFields,
|
||||||
const alertUuid = getAlertUuid(alertId);
|
});
|
||||||
const alertDetailsUrl = await getAlertUrl(
|
|
||||||
alertUuid,
|
|
||||||
spaceId,
|
|
||||||
indexedStartedAt,
|
|
||||||
alertsLocator,
|
|
||||||
basePath.publicBaseUrl
|
|
||||||
);
|
|
||||||
const groupByActionVariables =
|
|
||||||
getGroupByActionVariables(groupByFields);
|
|
||||||
|
|
||||||
alert.scheduleActions(ruleTypeConfig.defaultActionGroupId, {
|
const { uuid, start } = alertsClient.report({
|
||||||
alertDetailsUrl,
|
id: alertId,
|
||||||
interval: formatDurationFromTimeUnitChar(
|
actionGroup: ruleTypeConfig.defaultActionGroupId,
|
||||||
ruleParams.windowSize,
|
});
|
||||||
ruleParams.windowUnit as TimeUnitChar
|
const indexedStartedAt = start ?? startedAt.toISOString();
|
||||||
),
|
|
||||||
reason: alertReason,
|
|
||||||
threshold: ruleParams.threshold,
|
|
||||||
// When group by doesn't include error.grouping_key, the context.error.grouping_key action variable will contain value of the Error Grouping Key filter
|
|
||||||
errorGroupingKey: ruleParams.errorGroupingKey,
|
|
||||||
triggerValue: errorCount,
|
|
||||||
viewInAppUrl,
|
|
||||||
...groupByActionVariables,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
return { state: {} };
|
const relativeViewInAppUrl = getAlertUrlErrorCount(
|
||||||
},
|
groupByFields[SERVICE_NAME],
|
||||||
alerts: ApmRuleTypeAlertDefinition,
|
getEnvironmentEsField(groupByFields[SERVICE_ENVIRONMENT])?.[
|
||||||
getViewInAppRelativeUrl: ({ rule }: GetViewInAppRelativeUrlFnOpts<{}>) =>
|
SERVICE_ENVIRONMENT
|
||||||
observabilityPaths.ruleDetails(rule.id),
|
]
|
||||||
})
|
);
|
||||||
);
|
const viewInAppUrl = addSpaceIdToPath(
|
||||||
|
basePath.publicBaseUrl,
|
||||||
|
spaceId,
|
||||||
|
relativeViewInAppUrl
|
||||||
|
);
|
||||||
|
const alertDetailsUrl = await getAlertUrl(
|
||||||
|
uuid,
|
||||||
|
spaceId,
|
||||||
|
indexedStartedAt,
|
||||||
|
alertsLocator,
|
||||||
|
basePath.publicBaseUrl
|
||||||
|
);
|
||||||
|
const groupByActionVariables =
|
||||||
|
getGroupByActionVariables(groupByFields);
|
||||||
|
|
||||||
|
const payload = {
|
||||||
|
[PROCESSOR_EVENT]: ProcessorEvent.error,
|
||||||
|
[ALERT_EVALUATION_VALUE]: errorCount,
|
||||||
|
[ALERT_EVALUATION_THRESHOLD]: ruleParams.threshold,
|
||||||
|
[ERROR_GROUP_ID]: ruleParams.errorGroupingKey,
|
||||||
|
[ALERT_REASON]: alertReason,
|
||||||
|
...sourceFields,
|
||||||
|
...groupByFields,
|
||||||
|
};
|
||||||
|
|
||||||
|
const context = {
|
||||||
|
alertDetailsUrl,
|
||||||
|
interval: formatDurationFromTimeUnitChar(
|
||||||
|
ruleParams.windowSize,
|
||||||
|
ruleParams.windowUnit as TimeUnitChar
|
||||||
|
),
|
||||||
|
reason: alertReason,
|
||||||
|
threshold: ruleParams.threshold,
|
||||||
|
// When group by doesn't include error.grouping_key, the context.error.grouping_key action variable will contain value of the Error Grouping Key filter
|
||||||
|
errorGroupingKey: ruleParams.errorGroupingKey,
|
||||||
|
triggerValue: errorCount,
|
||||||
|
viewInAppUrl,
|
||||||
|
...groupByActionVariables,
|
||||||
|
};
|
||||||
|
|
||||||
|
alertsClient.setAlertData({
|
||||||
|
id: alertId,
|
||||||
|
payload,
|
||||||
|
context,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
return { state: {} };
|
||||||
|
},
|
||||||
|
alerts: {
|
||||||
|
...ApmRuleTypeAlertDefinition,
|
||||||
|
shouldWrite: true,
|
||||||
|
} as IRuleTypeAlerts<ErrorCountAlert>,
|
||||||
|
getViewInAppRelativeUrl: ({ rule }: GetViewInAppRelativeUrlFnOpts<{}>) =>
|
||||||
|
observabilityPaths.ruleDetails(rule.id),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue