mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[Security Solution] Improve find rule and find rule status route performance (#99678)
* Fetch rule statuses using single aggregation instead of N separate requests * Optimize _find API and _find_statuses * Merge alerting framework errors into rule statuses * Add sortSchema for top hits agg, update terms.order schema Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Marshall Main
GitHub
parent
b2e6028327
commit
7f6d7b3642
19 changed files with 321 additions and 154 deletions
|
|
@ -7,6 +7,7 @@
|
|||
*/
|
||||
|
||||
import { schema as s, ObjectType } from '@kbn/config-schema';
|
||||
import { sortOrderSchema } from './common_schemas';
|
||||
|
||||
/**
|
||||
* Schemas for the Bucket aggregations.
|
||||
|
|
@ -85,6 +86,12 @@ export const bucketAggsSchemas: Record<string, ObjectType> = {
|
|||
min_doc_count: s.maybe(s.number({ min: 1 })),
|
||||
size: s.maybe(s.number()),
|
||||
show_term_doc_count_error: s.maybe(s.boolean()),
|
||||
order: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])),
|
||||
order: s.maybe(
|
||||
s.oneOf([
|
||||
sortOrderSchema,
|
||||
s.recordOf(s.string(), sortOrderSchema),
|
||||
s.arrayOf(s.recordOf(s.string(), sortOrderSchema)),
|
||||
])
|
||||
),
|
||||
}),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { schema as s } from '@kbn/config-schema';
|
||||
|
||||
// note: these schemas are not exhaustive. See the `Sort` type of `@elastic/elasticsearch` if you need to enhance it.
|
||||
const fieldSchema = s.string();
|
||||
export const sortOrderSchema = s.oneOf([s.literal('asc'), s.literal('desc'), s.literal('_doc')]);
|
||||
const sortModeSchema = s.oneOf([
|
||||
s.literal('min'),
|
||||
s.literal('max'),
|
||||
s.literal('sum'),
|
||||
s.literal('avg'),
|
||||
s.literal('median'),
|
||||
]);
|
||||
const fieldSortSchema = s.object({
|
||||
missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])),
|
||||
mode: s.maybe(sortModeSchema),
|
||||
order: s.maybe(sortOrderSchema),
|
||||
// nested and unmapped_type not implemented yet
|
||||
});
|
||||
const sortContainerSchema = s.recordOf(s.string(), s.oneOf([sortOrderSchema, fieldSortSchema]));
|
||||
const sortCombinationsSchema = s.oneOf([fieldSchema, sortContainerSchema]);
|
||||
export const sortSchema = s.oneOf([sortCombinationsSchema, s.arrayOf(sortCombinationsSchema)]);
|
||||
|
|
@ -7,6 +7,7 @@
|
|||
*/
|
||||
|
||||
import { schema as s, ObjectType } from '@kbn/config-schema';
|
||||
import { sortSchema } from './common_schemas';
|
||||
|
||||
/**
|
||||
* Schemas for the metrics Aggregations
|
||||
|
|
@ -68,7 +69,7 @@ export const metricsAggsSchemas: Record<string, ObjectType> = {
|
|||
stored_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])),
|
||||
from: s.maybe(s.number()),
|
||||
size: s.maybe(s.number()),
|
||||
sort: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])),
|
||||
sort: s.maybe(sortSchema),
|
||||
seq_no_primary_term: s.maybe(s.boolean()),
|
||||
version: s.maybe(s.boolean()),
|
||||
track_scores: s.maybe(s.boolean()),
|
||||
|
|
|
|||
|
|
@ -491,6 +491,72 @@ export const getFindResultStatus = (): SavedObjectsFindResponse<IRuleSavedAttrib
|
|||
],
|
||||
});
|
||||
|
||||
export const getFindBulkResultStatus = (): SavedObjectsFindResponse<IRuleSavedAttributesSavedObjectAttributes> => ({
|
||||
page: 1,
|
||||
per_page: 6,
|
||||
total: 2,
|
||||
saved_objects: [],
|
||||
aggregations: {
|
||||
alertIds: {
|
||||
buckets: [
|
||||
{
|
||||
key: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
most_recent_statuses: {
|
||||
hits: {
|
||||
hits: [
|
||||
{
|
||||
_source: {
|
||||
'siem-detection-engine-rule-status': {
|
||||
alertId: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
|
||||
statusDate: '2020-02-18T15:26:49.783Z',
|
||||
status: 'succeeded',
|
||||
lastFailureAt: undefined,
|
||||
lastSuccessAt: '2020-02-18T15:26:49.783Z',
|
||||
lastFailureMessage: undefined,
|
||||
lastSuccessMessage: 'succeeded',
|
||||
lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(),
|
||||
gap: '500.32',
|
||||
searchAfterTimeDurations: ['200.00'],
|
||||
bulkCreateTimeDurations: ['800.43'],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
key: '1ea5a820-4da1-4e82-92a1-2b43a7bece08',
|
||||
most_recent_statuses: {
|
||||
hits: {
|
||||
hits: [
|
||||
{
|
||||
_source: {
|
||||
'siem-detection-engine-rule-status': {
|
||||
alertId: '1ea5a820-4da1-4e82-92a1-2b43a7bece08',
|
||||
statusDate: '2020-02-18T15:15:58.806Z',
|
||||
status: 'failed',
|
||||
lastFailureAt: '2020-02-18T15:15:58.806Z',
|
||||
lastSuccessAt: '2020-02-13T20:31:59.855Z',
|
||||
lastFailureMessage:
|
||||
'Signal rule name: "Query with a rule id Number 1", id: "1ea5a820-4da1-4e82-92a1-2b43a7bece08", rule_id: "query-rule-id-1" has a time gap of 5 days (412682928ms), and could be missing signals within that time. Consider increasing your look behind time or adding more Kibana instances.',
|
||||
lastSuccessMessage: 'succeeded',
|
||||
lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(),
|
||||
gap: '500.32',
|
||||
searchAfterTimeDurations: ['200.00'],
|
||||
bulkCreateTimeDurations: ['800.43'],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
export const getEmptySignalsResponse = (): SignalSearchResponse => ({
|
||||
took: 1,
|
||||
timed_out: false,
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import {
|
|||
getAlertMock,
|
||||
getFindRequest,
|
||||
getFindResultWithSingleHit,
|
||||
getFindResultStatus,
|
||||
getFindBulkResultStatus,
|
||||
} from '../__mocks__/request_responses';
|
||||
import { requestContextMock, serverMock, requestMock } from '../__mocks__';
|
||||
import { findRulesRoute } from './find_rules_route';
|
||||
|
|
@ -27,7 +27,7 @@ describe('find_rules', () => {
|
|||
|
||||
clients.alertsClient.find.mockResolvedValue(getFindResultWithSingleHit());
|
||||
clients.alertsClient.get.mockResolvedValue(getAlertMock(getQueryRuleParams()));
|
||||
clients.savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
|
||||
clients.savedObjectsClient.find.mockResolvedValue(getFindBulkResultStatus());
|
||||
|
||||
findRulesRoute(server.router);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -15,11 +15,10 @@ import type { SecuritySolutionPluginRouter } from '../../../../types';
|
|||
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
|
||||
import { findRules } from '../../rules/find_rules';
|
||||
import { buildSiemResponse } from '../utils';
|
||||
|
||||
import { getRuleActionsSavedObject } from '../../rule_actions/get_rule_actions_saved_object';
|
||||
import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_saved_objects_client';
|
||||
import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
|
||||
import { transformFindAlerts } from './utils';
|
||||
import { getBulkRuleActionsSavedObject } from '../../rule_actions/get_bulk_rule_actions_saved_object';
|
||||
|
||||
export const findRulesRoute = (router: SecuritySolutionPluginRouter) => {
|
||||
router.get(
|
||||
|
|
@ -60,44 +59,11 @@ export const findRulesRoute = (router: SecuritySolutionPluginRouter) => {
|
|||
filter: query.filter,
|
||||
fields: query.fields,
|
||||
});
|
||||
|
||||
// if any rules attempted to execute but failed before the rule executor is called,
|
||||
// an execution status will be written directly onto the rule via the kibana alerting framework,
|
||||
// which we are filtering on and will write a failure status
|
||||
// for any rules found to be in a failing state into our rule status saved objects
|
||||
const failingRules = rules.data.filter(
|
||||
(rule) => rule.executionStatus != null && rule.executionStatus.status === 'error'
|
||||
);
|
||||
|
||||
const ruleStatuses = await Promise.all(
|
||||
rules.data.map(async (rule) => {
|
||||
const results = await ruleStatusClient.find({
|
||||
perPage: 1,
|
||||
sortField: 'statusDate',
|
||||
sortOrder: 'desc',
|
||||
search: rule.id,
|
||||
searchFields: ['alertId'],
|
||||
});
|
||||
const failingRule = failingRules.find((badRule) => badRule.id === rule.id);
|
||||
if (failingRule != null) {
|
||||
if (results.saved_objects.length > 0) {
|
||||
results.saved_objects[0].attributes.status = 'failed';
|
||||
results.saved_objects[0].attributes.lastFailureAt = failingRule.executionStatus.lastExecutionDate.toISOString();
|
||||
}
|
||||
}
|
||||
return results;
|
||||
})
|
||||
);
|
||||
const ruleActions = await Promise.all(
|
||||
rules.data.map(async (rule) => {
|
||||
const results = await getRuleActionsSavedObject({
|
||||
savedObjectsClient,
|
||||
ruleAlertId: rule.id,
|
||||
});
|
||||
|
||||
return results;
|
||||
})
|
||||
);
|
||||
const alertIds = rules.data.map((rule) => rule.id);
|
||||
const [ruleStatuses, ruleActions] = await Promise.all([
|
||||
ruleStatusClient.findBulk(alertIds, 1),
|
||||
getBulkRuleActionsSavedObject({ alertIds, savedObjectsClient }),
|
||||
]);
|
||||
const transformed = transformFindAlerts(rules, ruleActions, ruleStatuses);
|
||||
if (transformed == null) {
|
||||
return siemResponse.error({ statusCode: 500, body: 'Internal error transforming' });
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@
|
|||
|
||||
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
|
||||
import {
|
||||
getFindResultStatus,
|
||||
ruleStatusRequest,
|
||||
getAlertMock,
|
||||
getFindBulkResultStatus,
|
||||
} from '../__mocks__/request_responses';
|
||||
import { serverMock, requestContextMock, requestMock } from '../__mocks__';
|
||||
import { findRulesStatusesRoute } from './find_rules_status_route';
|
||||
|
|
@ -26,7 +26,7 @@ describe('find_statuses', () => {
|
|||
beforeEach(async () => {
|
||||
server = serverMock.create();
|
||||
({ clients, context } = requestContextMock.createTools());
|
||||
clients.savedObjectsClient.find.mockResolvedValue(getFindResultStatus()); // successful status search
|
||||
clients.savedObjectsClient.find.mockResolvedValue(getFindBulkResultStatus()); // successful status search
|
||||
clients.alertsClient.get.mockResolvedValue(getAlertMock(getQueryRuleParams()));
|
||||
findRulesStatusesRoute(server.router);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,14 +9,13 @@ import { transformError } from '@kbn/securitysolution-es-utils';
|
|||
import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
|
||||
import type { SecuritySolutionPluginRouter } from '../../../../types';
|
||||
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
|
||||
import { RuleStatusResponse } from '../../rules/types';
|
||||
import { buildSiemResponse, mergeStatuses, getFailingRules } from '../utils';
|
||||
|
||||
import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_saved_objects_client';
|
||||
import {
|
||||
findRulesStatusesSchema,
|
||||
FindRulesStatusesSchemaDecoded,
|
||||
} from '../../../../../common/detection_engine/schemas/request/find_rule_statuses_schema';
|
||||
import { mergeAlertWithSidecarStatus } from '../../schemas/rule_converters';
|
||||
|
||||
/**
|
||||
* Given a list of rule ids, return the current status and
|
||||
|
|
@ -51,45 +50,27 @@ export const findRulesStatusesRoute = (router: SecuritySolutionPluginRouter) =>
|
|||
const ids = body.ids;
|
||||
try {
|
||||
const ruleStatusClient = ruleStatusSavedObjectsClientFactory(savedObjectsClient);
|
||||
const failingRules = await getFailingRules(ids, alertsClient);
|
||||
const [statusesById, failingRules] = await Promise.all([
|
||||
ruleStatusClient.findBulk(ids, 6),
|
||||
getFailingRules(ids, alertsClient),
|
||||
]);
|
||||
|
||||
const statuses = await ids.reduce(async (acc, id) => {
|
||||
const accumulated = await acc;
|
||||
const lastFiveErrorsForId = await ruleStatusClient.find({
|
||||
perPage: 6,
|
||||
sortField: 'statusDate',
|
||||
sortOrder: 'desc',
|
||||
search: id,
|
||||
searchFields: ['alertId'],
|
||||
});
|
||||
const statuses = ids.reduce((acc, id) => {
|
||||
const lastFiveErrorsForId = statusesById[id];
|
||||
|
||||
if (lastFiveErrorsForId.saved_objects.length === 0) {
|
||||
return accumulated;
|
||||
if (lastFiveErrorsForId == null || lastFiveErrorsForId.length === 0) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
const failingRule = failingRules[id];
|
||||
const lastFailureAt = lastFiveErrorsForId.saved_objects[0].attributes.lastFailureAt;
|
||||
|
||||
if (
|
||||
failingRule != null &&
|
||||
(lastFailureAt == null ||
|
||||
new Date(failingRule.executionStatus.lastExecutionDate) > new Date(lastFailureAt))
|
||||
) {
|
||||
const currentStatus = lastFiveErrorsForId.saved_objects[0];
|
||||
currentStatus.attributes.lastFailureMessage = `Reason: ${failingRule.executionStatus.error?.reason} Message: ${failingRule.executionStatus.error?.message}`;
|
||||
currentStatus.attributes.lastFailureAt = failingRule.executionStatus.lastExecutionDate.toISOString();
|
||||
currentStatus.attributes.statusDate = failingRule.executionStatus.lastExecutionDate.toISOString();
|
||||
currentStatus.attributes.status = 'failed';
|
||||
const updatedLastFiveErrorsSO = [
|
||||
currentStatus,
|
||||
...lastFiveErrorsForId.saved_objects.slice(1),
|
||||
];
|
||||
|
||||
return mergeStatuses(id, updatedLastFiveErrorsSO, accumulated);
|
||||
if (failingRule != null) {
|
||||
const currentStatus = mergeAlertWithSidecarStatus(failingRule, lastFiveErrorsForId[0]);
|
||||
const updatedLastFiveErrorsSO = [currentStatus, ...lastFiveErrorsForId.slice(1)];
|
||||
return mergeStatuses(id, updatedLastFiveErrorsSO, acc);
|
||||
}
|
||||
return mergeStatuses(id, [...lastFiveErrorsForId.saved_objects], accumulated);
|
||||
}, Promise.resolve<RuleStatusResponse>({}));
|
||||
|
||||
return mergeStatuses(id, [...lastFiveErrorsForId], acc);
|
||||
}, {});
|
||||
return response.ok({ body: statuses });
|
||||
} catch (err) {
|
||||
const error = transformError(err);
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ import { PartialFilter } from '../../types';
|
|||
import { BulkError, ImportSuccessError } from '../utils';
|
||||
import { getOutputRuleAlertForRest } from '../__mocks__/utils';
|
||||
import { PartialAlert } from '../../../../../../alerting/server';
|
||||
import { SanitizedAlert } from '../../../../../../alerting/server/types';
|
||||
import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
|
||||
import { RuleAlertType } from '../../rules/types';
|
||||
import { ImportRulesSchemaDecoded } from '../../../../../common/detection_engine/schemas/request/import_rules_schema';
|
||||
|
|
@ -256,7 +255,7 @@ describe('utils', () => {
|
|||
|
||||
describe('transformFindAlerts', () => {
|
||||
test('outputs empty data set when data set is empty correct', () => {
|
||||
const output = transformFindAlerts({ data: [], page: 1, perPage: 0, total: 0 }, []);
|
||||
const output = transformFindAlerts({ data: [], page: 1, perPage: 0, total: 0 }, {}, {});
|
||||
expect(output).toEqual({ data: [], page: 1, perPage: 0, total: 0 });
|
||||
});
|
||||
|
||||
|
|
@ -268,7 +267,8 @@ describe('utils', () => {
|
|||
total: 0,
|
||||
data: [getAlertMock(getQueryRuleParams())],
|
||||
},
|
||||
[]
|
||||
{},
|
||||
{}
|
||||
);
|
||||
const expected = getOutputRuleAlertForRest();
|
||||
expect(output).toEqual({
|
||||
|
|
@ -278,20 +278,6 @@ describe('utils', () => {
|
|||
data: [expected],
|
||||
});
|
||||
});
|
||||
|
||||
test('returns 500 if the data is not of type siem alert', () => {
|
||||
const unsafeCast = ([{ name: 'something else' }] as unknown) as SanitizedAlert[];
|
||||
const output = transformFindAlerts(
|
||||
{
|
||||
data: unsafeCast,
|
||||
page: 1,
|
||||
perPage: 1,
|
||||
total: 1,
|
||||
},
|
||||
[]
|
||||
);
|
||||
expect(output).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('transform', () => {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { countBy } from 'lodash/fp';
|
||||
import { SavedObject, SavedObjectsFindResponse } from 'kibana/server';
|
||||
import { SavedObject } from 'kibana/server';
|
||||
import uuid from 'uuid';
|
||||
|
||||
import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema';
|
||||
|
|
@ -17,11 +17,10 @@ import { INTERNAL_IDENTIFIER } from '../../../../../common/constants';
|
|||
import {
|
||||
RuleAlertType,
|
||||
isAlertType,
|
||||
isAlertTypes,
|
||||
IRuleSavedAttributesSavedObjectAttributes,
|
||||
isRuleStatusFindType,
|
||||
isRuleStatusFindTypes,
|
||||
isRuleStatusSavedObjectType,
|
||||
IRuleStatusSOAttributes,
|
||||
} from '../../rules/types';
|
||||
import {
|
||||
createBulkErrorObject,
|
||||
|
|
@ -34,6 +33,7 @@ import {
|
|||
import { RuleActions } from '../../rule_actions/types';
|
||||
import { internalRuleToAPIResponse } from '../../schemas/rule_converters';
|
||||
import { RuleParams } from '../../schemas/rule_schemas';
|
||||
import { SanitizedAlert } from '../../../../../../alerting/common';
|
||||
|
||||
type PromiseFromStreams = ImportRulesSchemaDecoded | Error;
|
||||
|
||||
|
|
@ -103,11 +103,11 @@ export const transformTags = (tags: string[]): string[] => {
|
|||
// Transforms the data but will remove any null or undefined it encounters and not include
|
||||
// those on the export
|
||||
export const transformAlertToRule = (
|
||||
alert: RuleAlertType,
|
||||
alert: SanitizedAlert<RuleParams>,
|
||||
ruleActions?: RuleActions | null,
|
||||
ruleStatus?: SavedObject<IRuleSavedAttributesSavedObjectAttributes>
|
||||
): Partial<RulesSchema> => {
|
||||
return internalRuleToAPIResponse(alert, ruleActions, ruleStatus);
|
||||
return internalRuleToAPIResponse(alert, ruleActions, ruleStatus?.attributes);
|
||||
};
|
||||
|
||||
export const transformAlertsToRules = (alerts: RuleAlertType[]): Array<Partial<RulesSchema>> => {
|
||||
|
|
@ -116,33 +116,24 @@ export const transformAlertsToRules = (alerts: RuleAlertType[]): Array<Partial<R
|
|||
|
||||
export const transformFindAlerts = (
|
||||
findResults: FindResult<RuleParams>,
|
||||
ruleActions: Array<RuleActions | null>,
|
||||
ruleStatuses?: Array<SavedObjectsFindResponse<IRuleSavedAttributesSavedObjectAttributes>>
|
||||
ruleActions: { [key: string]: RuleActions | undefined },
|
||||
ruleStatuses: { [key: string]: IRuleStatusSOAttributes[] | undefined }
|
||||
): {
|
||||
page: number;
|
||||
perPage: number;
|
||||
total: number;
|
||||
data: Array<Partial<RulesSchema>>;
|
||||
} | null => {
|
||||
if (!ruleStatuses && isAlertTypes(findResults.data)) {
|
||||
return {
|
||||
page: findResults.page,
|
||||
perPage: findResults.perPage,
|
||||
total: findResults.total,
|
||||
data: findResults.data.map((alert, idx) => transformAlertToRule(alert, ruleActions[idx])),
|
||||
};
|
||||
} else if (isAlertTypes(findResults.data) && isRuleStatusFindTypes(ruleStatuses)) {
|
||||
return {
|
||||
page: findResults.page,
|
||||
perPage: findResults.perPage,
|
||||
total: findResults.total,
|
||||
data: findResults.data.map((alert, idx) =>
|
||||
transformAlertToRule(alert, ruleActions[idx], ruleStatuses[idx].saved_objects[0])
|
||||
),
|
||||
};
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
page: findResults.page,
|
||||
perPage: findResults.perPage,
|
||||
total: findResults.total,
|
||||
data: findResults.data.map((alert) => {
|
||||
const statuses = ruleStatuses[alert.id];
|
||||
const status = statuses ? statuses[0] : undefined;
|
||||
return internalRuleToAPIResponse(alert, ruleActions[alert.id], status);
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
export const transform = (
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import {
|
|||
getFailingRules,
|
||||
} from './utils';
|
||||
import { responseMock } from './__mocks__';
|
||||
import { exampleRuleStatus, exampleFindRuleStatusResponse } from '../signals/__mocks__/es_results';
|
||||
import { exampleRuleStatus } from '../signals/__mocks__/es_results';
|
||||
import { getAlertMock } from './__mocks__/request_responses';
|
||||
import { AlertExecutionStatusErrorReasons } from '../../../../../alerting/common';
|
||||
import { getQueryRuleParams } from '../schemas/rule_schemas.mock';
|
||||
|
|
@ -301,8 +301,8 @@ describe('utils', () => {
|
|||
const statusTwo = exampleRuleStatus();
|
||||
statusTwo.attributes.status = 'failed';
|
||||
const currentStatus = exampleRuleStatus();
|
||||
const foundRules = exampleFindRuleStatusResponse([currentStatus, statusOne, statusTwo]);
|
||||
const res = mergeStatuses(currentStatus.attributes.alertId, foundRules.saved_objects, {
|
||||
const foundRules = [currentStatus.attributes, statusOne.attributes, statusTwo.attributes];
|
||||
const res = mergeStatuses(currentStatus.attributes.alertId, foundRules, {
|
||||
'myfakealertid-8cfac': {
|
||||
current_status: {
|
||||
alert_id: 'myfakealertid-8cfac',
|
||||
|
|
|
|||
|
|
@ -14,11 +14,12 @@ import {
|
|||
RouteValidationFunction,
|
||||
KibanaResponseFactory,
|
||||
CustomHttpResponseOptions,
|
||||
SavedObjectsFindResult,
|
||||
} from '../../../../../../../src/core/server';
|
||||
import { AlertsClient } from '../../../../../alerting/server';
|
||||
import { RuleStatusResponse, IRuleStatusSOAttributes } from '../rules/types';
|
||||
|
||||
import { RuleParams } from '../schemas/rule_schemas';
|
||||
|
||||
export interface OutputError {
|
||||
message: string;
|
||||
statusCode: number;
|
||||
|
|
@ -277,7 +278,7 @@ export const convertToSnakeCase = <T extends Record<string, unknown>>(
|
|||
*/
|
||||
export const mergeStatuses = (
|
||||
id: string,
|
||||
currentStatusAndFailures: Array<SavedObjectsFindResult<IRuleStatusSOAttributes>>,
|
||||
currentStatusAndFailures: IRuleStatusSOAttributes[],
|
||||
acc: RuleStatusResponse
|
||||
): RuleStatusResponse => {
|
||||
if (currentStatusAndFailures.length === 0) {
|
||||
|
|
@ -286,7 +287,7 @@ export const mergeStatuses = (
|
|||
};
|
||||
}
|
||||
const convertedCurrentStatus = convertToSnakeCase<IRuleStatusSOAttributes>(
|
||||
currentStatusAndFailures[0].attributes
|
||||
currentStatusAndFailures[0]
|
||||
);
|
||||
return {
|
||||
...acc,
|
||||
|
|
@ -294,12 +295,12 @@ export const mergeStatuses = (
|
|||
current_status: convertedCurrentStatus,
|
||||
failures: currentStatusAndFailures
|
||||
.slice(1)
|
||||
.map((errorItem) => convertToSnakeCase<IRuleStatusSOAttributes>(errorItem.attributes)),
|
||||
.map((errorItem) => convertToSnakeCase<IRuleStatusSOAttributes>(errorItem)),
|
||||
},
|
||||
} as RuleStatusResponse;
|
||||
};
|
||||
|
||||
export type GetFailingRulesResult = Record<string, SanitizedAlert>;
|
||||
export type GetFailingRulesResult = Record<string, SanitizedAlert<RuleParams>>;
|
||||
|
||||
export const getFailingRules = async (
|
||||
ids: string[],
|
||||
|
|
@ -316,13 +317,11 @@ export const getFailingRules = async (
|
|||
return errorRules
|
||||
.filter((rule) => rule.executionStatus.status === 'error')
|
||||
.reduce<GetFailingRulesResult>((acc, failingRule) => {
|
||||
const accum = acc;
|
||||
const theRule = failingRule;
|
||||
return {
|
||||
[theRule.id]: {
|
||||
...theRule,
|
||||
[failingRule.id]: {
|
||||
...failingRule,
|
||||
},
|
||||
...accum,
|
||||
...acc,
|
||||
};
|
||||
}, {});
|
||||
} catch (exc) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { AlertServices } from '../../../../../alerting/server';
|
||||
import { ruleActionsSavedObjectType } from './saved_object_mappings';
|
||||
import { IRuleActionsAttributesSavedObjectAttributes } from './types';
|
||||
import { getRuleActionsFromSavedObject } from './utils';
|
||||
import { RulesActionsSavedObject } from './get_rule_actions_saved_object';
|
||||
import { buildChunkedOrFilter } from '../signals/utils';
|
||||
|
||||
interface GetBulkRuleActionsSavedObject {
|
||||
alertIds: string[];
|
||||
savedObjectsClient: AlertServices['savedObjectsClient'];
|
||||
}
|
||||
|
||||
export const getBulkRuleActionsSavedObject = async ({
|
||||
alertIds,
|
||||
savedObjectsClient,
|
||||
}: GetBulkRuleActionsSavedObject): Promise<Record<string, RulesActionsSavedObject>> => {
|
||||
const filter = buildChunkedOrFilter(
|
||||
`${ruleActionsSavedObjectType}.attributes.ruleAlertId`,
|
||||
alertIds
|
||||
);
|
||||
const {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
saved_objects,
|
||||
} = await savedObjectsClient.find<IRuleActionsAttributesSavedObjectAttributes>({
|
||||
type: ruleActionsSavedObjectType,
|
||||
perPage: 10000,
|
||||
filter,
|
||||
});
|
||||
return saved_objects.reduce((acc: { [key: string]: RulesActionsSavedObject }, savedObject) => {
|
||||
acc[savedObject.attributes.ruleAlertId] = getRuleActionsFromSavedObject(savedObject);
|
||||
return acc;
|
||||
}, {});
|
||||
};
|
||||
|
|
@ -211,12 +211,6 @@ export const isRuleStatusFindType = (
|
|||
return get('saved_objects', obj) != null;
|
||||
};
|
||||
|
||||
export const isRuleStatusFindTypes = (
|
||||
obj: unknown[] | undefined
|
||||
): obj is Array<SavedObjectsFindResponse<IRuleSavedAttributesSavedObjectAttributes>> => {
|
||||
return obj ? obj.every((ruleStatus) => isRuleStatusFindType(ruleStatus)) : false;
|
||||
};
|
||||
|
||||
export interface CreateRulesOptions {
|
||||
alertsClient: AlertsClient;
|
||||
anomalyThreshold: AnomalyThresholdOrUndefined;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@
|
|||
*/
|
||||
|
||||
import uuid from 'uuid';
|
||||
import { SavedObject } from 'kibana/server';
|
||||
import {
|
||||
normalizeMachineLearningJobIds,
|
||||
normalizeThresholdObject,
|
||||
|
|
@ -29,8 +28,8 @@ import { AppClient } from '../../../types';
|
|||
import { addTags } from '../rules/add_tags';
|
||||
import { DEFAULT_MAX_SIGNALS, SERVER_APP_ID, SIGNALS_ID } from '../../../../common/constants';
|
||||
import { transformRuleToAlertAction } from '../../../../common/detection_engine/transform_actions';
|
||||
import { Alert } from '../../../../../alerting/common';
|
||||
import { IRuleSavedAttributesSavedObjectAttributes } from '../rules/types';
|
||||
import { SanitizedAlert } from '../../../../../alerting/common';
|
||||
import { IRuleStatusSOAttributes } from '../rules/types';
|
||||
import { transformTags } from '../routes/rules/utils';
|
||||
|
||||
// These functions provide conversions from the request API schema to the internal rule schema and from the internal rule schema
|
||||
|
|
@ -270,10 +269,11 @@ export const commonParamsCamelToSnake = (params: BaseRuleParams) => {
|
|||
};
|
||||
|
||||
export const internalRuleToAPIResponse = (
|
||||
rule: Alert<RuleParams>,
|
||||
rule: SanitizedAlert<RuleParams>,
|
||||
ruleActions?: RuleActions | null,
|
||||
ruleStatus?: SavedObject<IRuleSavedAttributesSavedObjectAttributes>
|
||||
ruleStatus?: IRuleStatusSOAttributes
|
||||
): FullResponseSchema => {
|
||||
const mergedStatus = ruleStatus ? mergeAlertWithSidecarStatus(rule, ruleStatus) : undefined;
|
||||
return {
|
||||
// Alerting framework params
|
||||
id: rule.id,
|
||||
|
|
@ -293,11 +293,30 @@ export const internalRuleToAPIResponse = (
|
|||
throttle: ruleActions?.ruleThrottle || 'no_actions',
|
||||
actions: ruleActions?.actions ?? [],
|
||||
// Rule status
|
||||
status: ruleStatus?.attributes.status ?? undefined,
|
||||
status_date: ruleStatus?.attributes.statusDate ?? undefined,
|
||||
last_failure_at: ruleStatus?.attributes.lastFailureAt ?? undefined,
|
||||
last_success_at: ruleStatus?.attributes.lastSuccessAt ?? undefined,
|
||||
last_failure_message: ruleStatus?.attributes.lastFailureMessage ?? undefined,
|
||||
last_success_message: ruleStatus?.attributes.lastSuccessMessage ?? undefined,
|
||||
status: mergedStatus?.status ?? undefined,
|
||||
status_date: mergedStatus?.statusDate ?? undefined,
|
||||
last_failure_at: mergedStatus?.lastFailureAt ?? undefined,
|
||||
last_success_at: mergedStatus?.lastSuccessAt ?? undefined,
|
||||
last_failure_message: mergedStatus?.lastFailureMessage ?? undefined,
|
||||
last_success_message: mergedStatus?.lastSuccessMessage ?? undefined,
|
||||
};
|
||||
};
|
||||
|
||||
export const mergeAlertWithSidecarStatus = (
|
||||
alert: SanitizedAlert<RuleParams>,
|
||||
status: IRuleStatusSOAttributes
|
||||
): IRuleStatusSOAttributes => {
|
||||
if (
|
||||
new Date(alert.executionStatus.lastExecutionDate) > new Date(status.statusDate) &&
|
||||
alert.executionStatus.status === 'error'
|
||||
) {
|
||||
return {
|
||||
...status,
|
||||
lastFailureMessage: `Reason: ${alert.executionStatus.error?.reason} Message: ${alert.executionStatus.error?.message}`,
|
||||
lastFailureAt: alert.executionStatus.lastExecutionDate.toISOString(),
|
||||
statusDate: alert.executionStatus.lastExecutionDate.toISOString(),
|
||||
status: 'failed',
|
||||
};
|
||||
}
|
||||
return status;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { RuleStatusSavedObjectsClient } from '../rule_status_saved_objects_clien
|
|||
|
||||
const createMockRuleStatusSavedObjectsClient = (): jest.Mocked<RuleStatusSavedObjectsClient> => ({
|
||||
find: jest.fn(),
|
||||
findBulk: jest.fn(),
|
||||
create: jest.fn(),
|
||||
update: jest.fn(),
|
||||
delete: jest.fn(),
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { get } from 'lodash';
|
||||
import {
|
||||
SavedObjectsClientContract,
|
||||
SavedObject,
|
||||
|
|
@ -14,11 +15,13 @@ import {
|
|||
} from '../../../../../../../src/core/server';
|
||||
import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings';
|
||||
import { IRuleStatusSOAttributes } from '../rules/types';
|
||||
import { buildChunkedOrFilter } from './utils';
|
||||
|
||||
export interface RuleStatusSavedObjectsClient {
|
||||
find: (
|
||||
options?: Omit<SavedObjectsFindOptions, 'type'>
|
||||
) => Promise<SavedObjectsFindResponse<IRuleStatusSOAttributes>>;
|
||||
findBulk: (ids: string[], statusesPerId: number) => Promise<FindBulkResponse>;
|
||||
create: (attributes: IRuleStatusSOAttributes) => Promise<SavedObject<IRuleStatusSOAttributes>>;
|
||||
update: (
|
||||
id: string,
|
||||
|
|
@ -27,6 +30,10 @@ export interface RuleStatusSavedObjectsClient {
|
|||
delete: (id: string) => Promise<{}>;
|
||||
}
|
||||
|
||||
interface FindBulkResponse {
|
||||
[key: string]: IRuleStatusSOAttributes[] | undefined;
|
||||
}
|
||||
|
||||
export const ruleStatusSavedObjectsClientFactory = (
|
||||
savedObjectsClient: SavedObjectsClientContract
|
||||
): RuleStatusSavedObjectsClient => ({
|
||||
|
|
@ -35,6 +42,50 @@ export const ruleStatusSavedObjectsClientFactory = (
|
|||
...options,
|
||||
type: ruleStatusSavedObjectType,
|
||||
}),
|
||||
findBulk: async (ids, statusesPerId) => {
|
||||
if (ids.length === 0) {
|
||||
return {};
|
||||
}
|
||||
const filter = buildChunkedOrFilter(`${ruleStatusSavedObjectType}.attributes.alertId`, ids);
|
||||
const order: 'desc' = 'desc';
|
||||
const aggs = {
|
||||
alertIds: {
|
||||
terms: {
|
||||
field: `${ruleStatusSavedObjectType}.attributes.alertId`,
|
||||
size: ids.length,
|
||||
},
|
||||
aggs: {
|
||||
most_recent_statuses: {
|
||||
top_hits: {
|
||||
sort: [
|
||||
{
|
||||
[`${ruleStatusSavedObjectType}.statusDate`]: {
|
||||
order,
|
||||
},
|
||||
},
|
||||
],
|
||||
size: statusesPerId,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const results = await savedObjectsClient.find({
|
||||
filter,
|
||||
aggs,
|
||||
type: ruleStatusSavedObjectType,
|
||||
perPage: 0,
|
||||
});
|
||||
const buckets = get(results, 'aggregations.alertIds.buckets');
|
||||
return buckets.reduce((acc: Record<string, unknown>, bucket: unknown) => {
|
||||
const key = get(bucket, 'key');
|
||||
const hits = get(bucket, 'most_recent_statuses.hits.hits');
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const statuses = hits.map((hit: any) => hit._source['siem-detection-engine-rule-status']);
|
||||
acc[key] = statuses;
|
||||
return acc;
|
||||
}, {});
|
||||
},
|
||||
create: (attributes) => savedObjectsClient.create(ruleStatusSavedObjectType, attributes),
|
||||
update: (id, attributes) => savedObjectsClient.update(ruleStatusSavedObjectType, id, attributes),
|
||||
delete: (id) => savedObjectsClient.delete(ruleStatusSavedObjectType, id),
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ import {
|
|||
createTotalHitsFromSearchResult,
|
||||
lastValidDate,
|
||||
calculateThresholdSignalUuid,
|
||||
buildChunkedOrFilter,
|
||||
} from './utils';
|
||||
import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types';
|
||||
import {
|
||||
|
|
@ -1473,4 +1474,26 @@ describe('utils', () => {
|
|||
expect(signalUuid).toEqual('ee8870dc-45ff-5e6c-a2f9-80886651ce03');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildChunkedOrFilter', () => {
|
||||
test('should return undefined if no values are provided', () => {
|
||||
const filter = buildChunkedOrFilter('field.name', []);
|
||||
expect(filter).toEqual(undefined);
|
||||
});
|
||||
|
||||
test('should return a filter with a single value', () => {
|
||||
const filter = buildChunkedOrFilter('field.name', ['id-1']);
|
||||
expect(filter).toEqual('field.name: ("id-1")');
|
||||
});
|
||||
|
||||
test('should return a filter with a multiple values', () => {
|
||||
const filter = buildChunkedOrFilter('field.name', ['id-1', 'id-2']);
|
||||
expect(filter).toEqual('field.name: ("id-1" OR "id-2")');
|
||||
});
|
||||
|
||||
test('should return a filter with a multiple values chunked', () => {
|
||||
const filter = buildChunkedOrFilter('field.name', ['id-1', 'id-2', 'id-3'], 2);
|
||||
expect(filter).toEqual('field.name: ("id-1" OR "id-2") OR field.name: ("id-3")');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import moment from 'moment';
|
|||
import uuidv5 from 'uuid/v5';
|
||||
import dateMath from '@elastic/datemath';
|
||||
import type { estypes } from '@elastic/elasticsearch';
|
||||
import { isEmpty, partition } from 'lodash';
|
||||
import { chunk, isEmpty, partition } from 'lodash';
|
||||
import { ApiResponse, Context } from '@elastic/elasticsearch/lib/Transport';
|
||||
|
||||
import { SortResults } from '@elastic/elasticsearch/api/types';
|
||||
|
|
@ -868,3 +868,16 @@ export const getSafeSortIds = (sortIds: SortResults | undefined) => {
|
|||
return sortId;
|
||||
});
|
||||
};
|
||||
|
||||
export const buildChunkedOrFilter = (field: string, values: string[], chunkSize: number = 1024) => {
|
||||
if (values.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const chunkedValues = chunk(values, chunkSize);
|
||||
return chunkedValues
|
||||
.map((subArray) => {
|
||||
const joinedValues = subArray.map((value) => `"${value}"`).join(' OR ');
|
||||
return `${field}: (${joinedValues})`;
|
||||
})
|
||||
.join(' OR ');
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue