[Response Ops] Fixing issue with empty row in execution log table when rule is running (#129151)

* Excluding execute-start events from exec log

* Fixing checks

* Adding functional tests
This commit is contained in:
Ying Mao 2022-04-01 13:06:23 -04:00 committed by GitHub
parent 0c2ec496d0
commit 8291a2299d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 1090 additions and 984 deletions

View file

@ -68,10 +68,15 @@ interface IExecutionUuidAggBucket extends estypes.AggregationsStringTermsBucketK
};
}
interface ExecutionUuidAggResult<TBucket = IExecutionUuidAggBucket>
export interface ExecutionUuidAggResult<TBucket = IExecutionUuidAggBucket>
extends estypes.AggregationsAggregateBase {
buckets: TBucket[];
}
interface ExcludeExecuteStartAggResult extends estypes.AggregationsAggregateBase {
executionUuid: ExecutionUuidAggResult;
executionUuidCardinality: estypes.AggregationsCardinalityAggregate;
}
export interface IExecutionLogAggOptions {
page: number;
perPage: number;
@ -112,104 +117,119 @@ export function getExecutionLogAggregation({ page, perPage, sort }: IExecutionLo
}
return {
// Get total number of executions
executionUuidCardinality: {
cardinality: {
field: EXECUTION_UUID_FIELD,
},
},
executionUuid: {
// Bucket by execution UUID
terms: {
field: EXECUTION_UUID_FIELD,
size: DEFAULT_MAX_BUCKETS_LIMIT,
order: formatSortForTermSort(sort),
excludeExecuteStart: {
filter: {
bool: {
must_not: [
{
term: {
[ACTION_FIELD]: 'execute-start',
},
},
],
},
},
aggs: {
// Bucket sort to allow paging through executions
executionUuidSorted: {
bucket_sort: {
sort: formatSortForBucketSort(sort),
from: (page - 1) * perPage,
size: perPage,
gap_policy: 'insert_zeros' as estypes.AggregationsGapPolicy,
// Get total number of executions
executionUuidCardinality: {
cardinality: {
field: EXECUTION_UUID_FIELD,
},
},
// Get counts for types of alerts and whether there was an execution timeout
alertCounts: {
filters: {
filters: {
newAlerts: { match: { [ACTION_FIELD]: 'new-instance' } },
activeAlerts: { match: { [ACTION_FIELD]: 'active-instance' } },
recoveredAlerts: { match: { [ACTION_FIELD]: 'recovered-instance' } },
},
executionUuid: {
// Bucket by execution UUID
terms: {
field: EXECUTION_UUID_FIELD,
size: DEFAULT_MAX_BUCKETS_LIMIT,
order: formatSortForTermSort(sort),
},
},
// Filter by action execute doc and get information from this event
actionExecution: {
filter: getProviderAndActionFilter('actions', 'execute'),
aggs: {
actionOutcomes: {
terms: {
field: OUTCOME_FIELD,
size: 2,
// Bucket sort to allow paging through executions
executionUuidSorted: {
bucket_sort: {
sort: formatSortForBucketSort(sort),
from: (page - 1) * perPage,
size: perPage,
gap_policy: 'insert_zeros' as estypes.AggregationsGapPolicy,
},
},
},
},
// Filter by rule execute doc and get information from this event
ruleExecution: {
filter: getProviderAndActionFilter('alerting', 'execute'),
aggs: {
executeStartTime: {
min: {
field: START_FIELD,
},
},
scheduleDelay: {
max: {
field: SCHEDULE_DELAY_FIELD,
},
},
totalSearchDuration: {
max: {
field: TOTAL_SEARCH_DURATION_FIELD,
},
},
esSearchDuration: {
max: {
field: ES_SEARCH_DURATION_FIELD,
},
},
numTriggeredActions: {
max: {
field: NUMBER_OF_TRIGGERED_ACTIONS_FIELD,
},
},
numScheduledActions: {
max: {
field: NUMBER_OF_SCHEDULED_ACTIONS_FIELD,
},
},
executionDuration: {
max: {
field: DURATION_FIELD,
},
},
outcomeAndMessage: {
top_hits: {
size: 1,
_source: {
includes: [OUTCOME_FIELD, MESSAGE_FIELD, ERROR_MESSAGE_FIELD],
// Get counts for types of alerts and whether there was an execution timeout
alertCounts: {
filters: {
filters: {
newAlerts: { match: { [ACTION_FIELD]: 'new-instance' } },
activeAlerts: { match: { [ACTION_FIELD]: 'active-instance' } },
recoveredAlerts: { match: { [ACTION_FIELD]: 'recovered-instance' } },
},
},
},
// Filter by action execute doc and get information from this event
actionExecution: {
filter: getProviderAndActionFilter('actions', 'execute'),
aggs: {
actionOutcomes: {
terms: {
field: OUTCOME_FIELD,
size: 2,
},
},
},
},
// Filter by rule execute doc and get information from this event
ruleExecution: {
filter: getProviderAndActionFilter('alerting', 'execute'),
aggs: {
executeStartTime: {
min: {
field: START_FIELD,
},
},
scheduleDelay: {
max: {
field: SCHEDULE_DELAY_FIELD,
},
},
totalSearchDuration: {
max: {
field: TOTAL_SEARCH_DURATION_FIELD,
},
},
esSearchDuration: {
max: {
field: ES_SEARCH_DURATION_FIELD,
},
},
numTriggeredActions: {
max: {
field: NUMBER_OF_TRIGGERED_ACTIONS_FIELD,
},
},
numScheduledActions: {
max: {
field: NUMBER_OF_SCHEDULED_ACTIONS_FIELD,
},
},
executionDuration: {
max: {
field: DURATION_FIELD,
},
},
outcomeAndMessage: {
top_hits: {
size: 1,
_source: {
includes: [OUTCOME_FIELD, MESSAGE_FIELD, ERROR_MESSAGE_FIELD],
},
},
},
},
},
// If there was a timeout, this filter will return non-zero doc count
timeoutMessage: {
filter: getProviderAndActionFilter('alerting', 'execute-timeout'),
},
},
},
// If there was a timeout, this filter will return non-zero doc count
timeoutMessage: {
filter: getProviderAndActionFilter('alerting', 'execute-timeout'),
},
},
},
};
@ -280,13 +300,14 @@ export function formatExecutionLogResult(
): IExecutionLogResult {
const { aggregations } = results;
if (!aggregations) {
if (!aggregations || !aggregations.excludeExecuteStart) {
return EMPTY_EXECUTION_LOG_RESULT;
}
const total = (aggregations.executionUuidCardinality as estypes.AggregationsCardinalityAggregate)
.value;
const buckets = (aggregations.executionUuid as ExecutionUuidAggResult).buckets;
const aggs = aggregations.excludeExecuteStart as ExcludeExecuteStartAggResult;
const total = aggs.executionUuidCardinality.value;
const buckets = aggs.executionUuid.buckets;
return {
total,

View file

@ -96,185 +96,189 @@ const BaseRuleSavedObject: SavedObject<RawRule> = {
const aggregateResults = {
aggregations: {
executionUuid: {
excludeExecuteStart: {
meta: {},
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{
key: '6705da7d-2635-499d-a6a8-1aee1ae1eac9',
doc_count: 27,
timeoutMessage: {
meta: {},
doc_count: 0,
},
alertCounts: {
meta: {},
buckets: {
activeAlerts: {
doc_count: 5,
},
newAlerts: {
doc_count: 5,
},
recoveredAlerts: {
doc_count: 0,
},
doc_count: 875,
executionUuid: {
meta: {},
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{
key: '6705da7d-2635-499d-a6a8-1aee1ae1eac9',
doc_count: 27,
timeoutMessage: {
meta: {},
doc_count: 0,
},
},
ruleExecution: {
meta: {},
doc_count: 1,
numTriggeredActions: {
value: 5.0,
},
numScheduledActions: {
value: 5.0,
},
outcomeAndMessage: {
hits: {
total: {
value: 1,
relation: 'eq',
alertCounts: {
meta: {},
buckets: {
activeAlerts: {
doc_count: 5,
},
max_score: 1.0,
hits: [
{
_index: '.kibana-event-log-8.2.0-000001',
_id: 'S4wIZX8B8TGQpG7XQZns',
_score: 1.0,
_source: {
event: {
outcome: 'success',
newAlerts: {
doc_count: 5,
},
recoveredAlerts: {
doc_count: 0,
},
},
},
ruleExecution: {
meta: {},
doc_count: 1,
numTriggeredActions: {
value: 5.0,
},
numScheduledActions: {
value: 5.0,
},
outcomeAndMessage: {
hits: {
total: {
value: 1,
relation: 'eq',
},
max_score: 1.0,
hits: [
{
_index: '.kibana-event-log-8.2.0-000001',
_id: 'S4wIZX8B8TGQpG7XQZns',
_score: 1.0,
_source: {
event: {
outcome: 'success',
},
message:
"rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'",
},
message:
"rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'",
},
],
},
},
scheduleDelay: {
value: 3.126e9,
},
totalSearchDuration: {
value: 0.0,
},
esSearchDuration: {
value: 0.0,
},
executionDuration: {
value: 1.056e9,
},
executeStartTime: {
value: 1.646667512617e12,
value_as_string: '2022-03-07T15:38:32.617Z',
},
},
actionExecution: {
meta: {},
doc_count: 5,
actionOutcomes: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{
key: 'success',
doc_count: 5,
},
],
},
},
scheduleDelay: {
value: 3.126e9,
},
totalSearchDuration: {
value: 0.0,
},
esSearchDuration: {
value: 0.0,
},
executionDuration: {
value: 1.056e9,
},
executeStartTime: {
value: 1.646667512617e12,
value_as_string: '2022-03-07T15:38:32.617Z',
},
},
actionExecution: {
meta: {},
doc_count: 5,
actionOutcomes: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{
key: 'success',
{
key: '41b2755e-765a-4044-9745-b03875d5e79a',
doc_count: 32,
timeoutMessage: {
meta: {},
doc_count: 0,
},
alertCounts: {
meta: {},
buckets: {
activeAlerts: {
doc_count: 5,
},
],
},
},
},
{
key: '41b2755e-765a-4044-9745-b03875d5e79a',
doc_count: 32,
timeoutMessage: {
meta: {},
doc_count: 0,
},
alertCounts: {
meta: {},
buckets: {
activeAlerts: {
doc_count: 5,
},
newAlerts: {
doc_count: 5,
},
recoveredAlerts: {
doc_count: 5,
},
},
},
ruleExecution: {
meta: {},
doc_count: 1,
numTriggeredActions: {
value: 5.0,
},
numScheduledActions: {
value: 5.0,
},
outcomeAndMessage: {
hits: {
total: {
value: 1,
relation: 'eq',
newAlerts: {
doc_count: 5,
},
max_score: 1.0,
hits: [
{
_index: '.kibana-event-log-8.2.0-000001',
_id: 'a4wIZX8B8TGQpG7Xwpnz',
_score: 1.0,
_source: {
event: {
outcome: 'success',
recoveredAlerts: {
doc_count: 5,
},
},
},
ruleExecution: {
meta: {},
doc_count: 1,
numTriggeredActions: {
value: 5.0,
},
numScheduledActions: {
value: 5.0,
},
outcomeAndMessage: {
hits: {
total: {
value: 1,
relation: 'eq',
},
max_score: 1.0,
hits: [
{
_index: '.kibana-event-log-8.2.0-000001',
_id: 'a4wIZX8B8TGQpG7Xwpnz',
_score: 1.0,
_source: {
event: {
outcome: 'success',
},
message:
"rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'",
},
message:
"rule executed: example.always-firing:a348a740-9e2c-11ec-bd64-774ed95c43ef: 'test rule'",
},
],
},
},
scheduleDelay: {
value: 3.345e9,
},
totalSearchDuration: {
value: 0.0,
},
esSearchDuration: {
value: 0.0,
},
executionDuration: {
value: 1.165e9,
},
executeStartTime: {
value: 1.646667545604e12,
value_as_string: '2022-03-07T15:39:05.604Z',
},
},
actionExecution: {
meta: {},
doc_count: 5,
actionOutcomes: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{
key: 'success',
doc_count: 5,
},
],
},
},
scheduleDelay: {
value: 3.345e9,
},
totalSearchDuration: {
value: 0.0,
},
esSearchDuration: {
value: 0.0,
},
executionDuration: {
value: 1.165e9,
},
executeStartTime: {
value: 1.646667545604e12,
value_as_string: '2022-03-07T15:39:05.604Z',
},
},
actionExecution: {
meta: {},
doc_count: 5,
actionOutcomes: {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{
key: 'success',
doc_count: 5,
},
],
},
},
},
],
},
executionUuidCardinality: {
value: 374,
],
},
executionUuidCardinality: {
value: 374,
},
},
},
};

View file

@ -675,23 +675,32 @@ export function defineAlertTypes(
throw new Error('this alert is intended to fail');
},
};
const longRunningAlertType: RuleType<{}, {}, {}, {}, {}, 'default'> = {
id: 'test.longRunning',
name: 'Test: Long Running',
actionGroups: [
{
id: 'default',
name: 'Default',
function getLongRunningRuleType() {
const paramsSchema = schema.object({
delay: schema.maybe(schema.number({ defaultValue: 5000 })),
});
type ParamsType = TypeOf<typeof paramsSchema>;
const result: RuleType<ParamsType, {}, {}, {}, {}, 'default'> = {
id: 'test.longRunning',
name: 'Test: Long Running',
actionGroups: [
{
id: 'default',
name: 'Default',
},
],
producer: 'alertsFixture',
defaultActionGroupId: 'default',
minimumLicenseRequired: 'basic',
isExportable: true,
async executor(ruleExecutorOptions) {
const { params } = ruleExecutorOptions;
await new Promise((resolve) => setTimeout(resolve, params.delay ?? 5000));
},
],
producer: 'alertsFixture',
defaultActionGroupId: 'default',
minimumLicenseRequired: 'basic',
isExportable: true,
async executor() {
await new Promise((resolve) => setTimeout(resolve, 5000));
},
};
};
return result;
}
const exampleAlwaysFiringAlertType: RuleType<{}, {}, {}, {}, {}, 'small' | 'medium' | 'large'> = {
id: 'example.always-firing',
name: 'Always firing',
@ -769,7 +778,7 @@ export function defineAlertTypes(
alerting.registerType(onlyStateVariablesAlertType);
alerting.registerType(getPatternFiringAlertType());
alerting.registerType(throwAlertType);
alerting.registerType(longRunningAlertType);
alerting.registerType(getLongRunningRuleType());
alerting.registerType(goldNoopAlertType);
alerting.registerType(exampleAlwaysFiringAlertType);
alerting.registerType(multipleSearchesRuleType);

View file

@ -126,6 +126,32 @@ export default function createGetExecutionLogTests({ getService }: FtrProviderCo
expect(response.body.errors).to.eql([]);
});
it('gets execution log for rule that is currently running', async () => {
const { body: createdRule } = await supertest
.post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`)
.set('kbn-xsrf', 'foo')
.send(getTestRuleData({ rule_type_id: 'test.longRunning', params: { delay: 120000 } }))
.expect(200);
objectRemover.add(Spaces.space1.id, createdRule.id, 'rule', 'alerting');
// wait for execute-start event that signals rule has started running
await waitForEvents(createdRule.id, 'alerting', new Map([['execute-start', { gte: 1 }]]));
const response = await supertest.get(
`${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${
createdRule.id
}/_execution_log?date_start=${dateStart}`
);
expect(response.status).to.eql(200);
// since these events should have been excluded from the agg, should return empty
expect(response.body.total).to.eql(0);
expect(response.body.data).to.eql([]);
expect(response.body.totalErrors).to.eql(0);
expect(response.body.errors).to.eql([]);
});
it('gets execution log for rule that performs ES searches', async () => {
const { body: createdRule } = await supertest
.post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`)