mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 17:59:23 -04:00
[Security Solution][Detection Engine] Bubbles up more error messages from ES queries to the UI (#78004)
## Summary Fixes: https://github.com/elastic/kibana/issues/77254 Bubbles up error messages from ES queries that have _shards.failures in them. For example if you have errors in your exceptions list you will need to see them bubbled up. Steps to reproduce: Go to a detections rule and add an invalid value within the exceptions such as this one below: <img width="1523" alt="Screen Shot 2020-09-21 at 7 52 59 AM" src="https://user-images.githubusercontent.com/1151048/93817197-d1a53780-fc15-11ea-8cf2-4dd7fd5a3c13.png"> Notice that rsa.internal.level value is not a numeric but a text string. You should now see this error message where before you could not: <img width="1503" alt="Screen Shot 2020-09-21 at 7 52 44 AM" src="https://user-images.githubusercontent.com/1151048/93817231-e1bd1700-fc15-11ea-9038-99668233191a.png"> ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
This commit is contained in:
Frank Hassanabad
GitHub
parent
9450248ebe
commit
d79fbb3f5c
17 changed files with 647 additions and 155 deletions
|
|
@ -337,7 +337,7 @@ export const repeatedSearchResultsWithSortId = (
|
|||
guids: string[],
|
||||
ips?: string[],
|
||||
destIps?: string[]
|
||||
) => ({
|
||||
): SignalSearchResponse => ({
|
||||
took: 10,
|
||||
timed_out: false,
|
||||
_shards: {
|
||||
|
|
@ -364,7 +364,7 @@ export const repeatedSearchResultsWithNoSortId = (
|
|||
pageSize: number,
|
||||
guids: string[],
|
||||
ips?: string[]
|
||||
) => ({
|
||||
): SignalSearchResponse => ({
|
||||
took: 10,
|
||||
timed_out: false,
|
||||
_shards: {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import dateMath from '@elastic/datemath';
|
|||
|
||||
import { KibanaRequest } from '../../../../../../../src/core/server';
|
||||
import { MlPluginSetup } from '../../../../../ml/server';
|
||||
import { getAnomalies } from '../../machine_learning';
|
||||
import { AnomalyResults, getAnomalies } from '../../machine_learning';
|
||||
|
||||
export const findMlSignals = async ({
|
||||
ml,
|
||||
|
|
@ -24,7 +24,7 @@ export const findMlSignals = async ({
|
|||
anomalyThreshold: number;
|
||||
from: string;
|
||||
to: string;
|
||||
}) => {
|
||||
}): Promise<AnomalyResults> => {
|
||||
const { mlAnomalySearch } = ml.mlSystemProvider(request);
|
||||
const params = {
|
||||
jobIds: [jobId],
|
||||
|
|
@ -32,7 +32,5 @@ export const findMlSignals = async ({
|
|||
earliestMs: dateMath.parse(from)?.valueOf() ?? 0,
|
||||
latestMs: dateMath.parse(to)?.valueOf() ?? 0,
|
||||
};
|
||||
const relevantAnomalies = await getAnomalies(params, mlAnomalySearch);
|
||||
|
||||
return relevantAnomalies;
|
||||
return getAnomalies(params, mlAnomalySearch);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ export const findThresholdSignals = async ({
|
|||
}: FindThresholdSignalsParams): Promise<{
|
||||
searchResult: SignalSearchResponse;
|
||||
searchDuration: string;
|
||||
searchErrors: string[];
|
||||
}> => {
|
||||
const aggregations =
|
||||
threshold && !isEmpty(threshold.field)
|
||||
|
|
|
|||
|
|
@ -3,56 +3,18 @@
|
|||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import moment from 'moment';
|
||||
|
||||
import { AlertServices } from '../../../../../alerts/server';
|
||||
import { ListClient } from '../../../../../lists/server';
|
||||
import { RuleAlertAction } from '../../../../common/detection_engine/types';
|
||||
import { RuleTypeParams, RefreshTypes } from '../types';
|
||||
import { Logger } from '../../../../../../../src/core/server';
|
||||
import { singleSearchAfter } from './single_search_after';
|
||||
import { singleBulkCreate } from './single_bulk_create';
|
||||
import { BuildRuleMessage } from './rule_messages';
|
||||
import { SignalSearchResponse } from './types';
|
||||
import { filterEventsAgainstList } from './filter_events_with_list';
|
||||
import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
|
||||
import { getSignalTimeTuples } from './utils';
|
||||
|
||||
interface SearchAfterAndBulkCreateParams {
|
||||
gap: moment.Duration | null;
|
||||
previousStartedAt: Date | null | undefined;
|
||||
ruleParams: RuleTypeParams;
|
||||
services: AlertServices;
|
||||
listClient: ListClient;
|
||||
exceptionsList: ExceptionListItemSchema[];
|
||||
logger: Logger;
|
||||
id: string;
|
||||
inputIndexPattern: string[];
|
||||
signalsIndex: string;
|
||||
name: string;
|
||||
actions: RuleAlertAction[];
|
||||
createdAt: string;
|
||||
createdBy: string;
|
||||
updatedBy: string;
|
||||
updatedAt: string;
|
||||
interval: string;
|
||||
enabled: boolean;
|
||||
pageSize: number;
|
||||
filter: unknown;
|
||||
refresh: RefreshTypes;
|
||||
tags: string[];
|
||||
throttle: string;
|
||||
buildRuleMessage: BuildRuleMessage;
|
||||
}
|
||||
|
||||
export interface SearchAfterAndBulkCreateReturnType {
|
||||
success: boolean;
|
||||
searchAfterTimes: string[];
|
||||
bulkCreateTimes: string[];
|
||||
lastLookBackDate: Date | null | undefined;
|
||||
createdSignalsCount: number;
|
||||
errors: string[];
|
||||
}
|
||||
import {
|
||||
createSearchAfterReturnType,
|
||||
createSearchAfterReturnTypeFromResponse,
|
||||
createTotalHitsFromSearchResult,
|
||||
getSignalTimeTuples,
|
||||
mergeReturns,
|
||||
} from './utils';
|
||||
import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from './types';
|
||||
|
||||
// search_after through documents and re-index using bulk endpoint.
|
||||
export const searchAfterAndBulkCreate = async ({
|
||||
|
|
@ -81,14 +43,7 @@ export const searchAfterAndBulkCreate = async ({
|
|||
throttle,
|
||||
buildRuleMessage,
|
||||
}: SearchAfterAndBulkCreateParams): Promise<SearchAfterAndBulkCreateReturnType> => {
|
||||
const toReturn: SearchAfterAndBulkCreateReturnType = {
|
||||
success: true,
|
||||
searchAfterTimes: [],
|
||||
bulkCreateTimes: [],
|
||||
lastLookBackDate: null,
|
||||
createdSignalsCount: 0,
|
||||
errors: [],
|
||||
};
|
||||
let toReturn = createSearchAfterReturnType();
|
||||
|
||||
// sortId tells us where to start our next consecutive search_after query
|
||||
let sortId: string | undefined;
|
||||
|
|
@ -108,13 +63,15 @@ export const searchAfterAndBulkCreate = async ({
|
|||
buildRuleMessage,
|
||||
});
|
||||
logger.debug(buildRuleMessage(`totalToFromTuples: ${totalToFromTuples.length}`));
|
||||
|
||||
while (totalToFromTuples.length > 0) {
|
||||
const tuple = totalToFromTuples.pop();
|
||||
if (tuple == null || tuple.to == null || tuple.from == null) {
|
||||
logger.error(buildRuleMessage(`[-] malformed date tuple`));
|
||||
toReturn.success = false;
|
||||
toReturn.errors = [...new Set([...toReturn.errors, 'malformed date tuple'])];
|
||||
return toReturn;
|
||||
return createSearchAfterReturnType({
|
||||
success: false,
|
||||
errors: ['malformed date tuple'],
|
||||
});
|
||||
}
|
||||
signalsCreatedCount = 0;
|
||||
while (signalsCreatedCount < tuple.maxSignals) {
|
||||
|
|
@ -122,29 +79,27 @@ export const searchAfterAndBulkCreate = async ({
|
|||
logger.debug(buildRuleMessage(`sortIds: ${sortId}`));
|
||||
|
||||
// perform search_after with optionally undefined sortId
|
||||
const {
|
||||
searchResult,
|
||||
searchDuration,
|
||||
}: { searchResult: SignalSearchResponse; searchDuration: string } = await singleSearchAfter(
|
||||
{
|
||||
searchAfterSortId: sortId,
|
||||
index: inputIndexPattern,
|
||||
from: tuple.from.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
services,
|
||||
logger,
|
||||
filter,
|
||||
pageSize: tuple.maxSignals < pageSize ? Math.ceil(tuple.maxSignals) : pageSize, // maximum number of docs to receive per search result.
|
||||
timestampOverride: ruleParams.timestampOverride,
|
||||
}
|
||||
);
|
||||
toReturn.searchAfterTimes.push(searchDuration);
|
||||
|
||||
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
|
||||
searchAfterSortId: sortId,
|
||||
index: inputIndexPattern,
|
||||
from: tuple.from.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
services,
|
||||
logger,
|
||||
filter,
|
||||
pageSize: tuple.maxSignals < pageSize ? Math.ceil(tuple.maxSignals) : pageSize, // maximum number of docs to receive per search result.
|
||||
timestampOverride: ruleParams.timestampOverride,
|
||||
});
|
||||
toReturn = mergeReturns([
|
||||
toReturn,
|
||||
createSearchAfterReturnTypeFromResponse({ searchResult }),
|
||||
createSearchAfterReturnType({
|
||||
searchAfterTimes: [searchDuration],
|
||||
errors: searchErrors,
|
||||
}),
|
||||
]);
|
||||
// determine if there are any candidate signals to be processed
|
||||
const totalHits =
|
||||
typeof searchResult.hits.total === 'number'
|
||||
? searchResult.hits.total
|
||||
: searchResult.hits.total.value;
|
||||
const totalHits = createTotalHitsFromSearchResult({ searchResult });
|
||||
logger.debug(buildRuleMessage(`totalHits: ${totalHits}`));
|
||||
logger.debug(
|
||||
buildRuleMessage(`searchResult.hit.hits.length: ${searchResult.hits.hits.length}`)
|
||||
|
|
@ -168,17 +123,11 @@ export const searchAfterAndBulkCreate = async ({
|
|||
);
|
||||
break;
|
||||
}
|
||||
toReturn.lastLookBackDate =
|
||||
searchResult.hits.hits.length > 0
|
||||
? new Date(
|
||||
searchResult.hits.hits[searchResult.hits.hits.length - 1]?._source['@timestamp']
|
||||
)
|
||||
: null;
|
||||
|
||||
// filter out the search results that match with the values found in the list.
|
||||
// the resulting set are signals to be indexed, given they are not duplicates
|
||||
// of signals already present in the signals index.
|
||||
const filteredEvents: SignalSearchResponse = await filterEventsAgainstList({
|
||||
const filteredEvents = await filterEventsAgainstList({
|
||||
listClient,
|
||||
exceptionsList,
|
||||
logger,
|
||||
|
|
@ -222,19 +171,21 @@ export const searchAfterAndBulkCreate = async ({
|
|||
tags,
|
||||
throttle,
|
||||
});
|
||||
logger.debug(buildRuleMessage(`created ${createdCount} signals`));
|
||||
toReturn.createdSignalsCount += createdCount;
|
||||
toReturn = mergeReturns([
|
||||
toReturn,
|
||||
createSearchAfterReturnType({
|
||||
success: bulkSuccess,
|
||||
createdSignalsCount: createdCount,
|
||||
bulkCreateTimes: bulkDuration ? [bulkDuration] : undefined,
|
||||
errors: bulkErrors,
|
||||
}),
|
||||
]);
|
||||
signalsCreatedCount += createdCount;
|
||||
logger.debug(buildRuleMessage(`created ${createdCount} signals`));
|
||||
logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`));
|
||||
if (bulkDuration) {
|
||||
toReturn.bulkCreateTimes.push(bulkDuration);
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
buildRuleMessage(`filteredEvents.hits.hits: ${filteredEvents.hits.hits.length}`)
|
||||
);
|
||||
toReturn.success = toReturn.success && bulkSuccess;
|
||||
toReturn.errors = [...new Set([...toReturn.errors, ...bulkErrors])];
|
||||
}
|
||||
|
||||
// we are guaranteed to have searchResult hits at this point
|
||||
|
|
@ -249,9 +200,13 @@ export const searchAfterAndBulkCreate = async ({
|
|||
}
|
||||
} catch (exc: unknown) {
|
||||
logger.error(buildRuleMessage(`[-] search_after and bulk threw an error ${exc}`));
|
||||
toReturn.success = false;
|
||||
toReturn.errors = [...new Set([...toReturn.errors, `${exc}`])];
|
||||
return toReturn;
|
||||
return mergeReturns([
|
||||
toReturn,
|
||||
createSearchAfterReturnType({
|
||||
success: false,
|
||||
errors: [`${exc}`],
|
||||
}),
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,11 +18,8 @@ import {
|
|||
sortExceptionItems,
|
||||
} from './utils';
|
||||
import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates';
|
||||
import { RuleExecutorOptions } from './types';
|
||||
import {
|
||||
searchAfterAndBulkCreate,
|
||||
SearchAfterAndBulkCreateReturnType,
|
||||
} from './search_after_bulk_create';
|
||||
import { RuleExecutorOptions, SearchAfterAndBulkCreateReturnType } from './types';
|
||||
import { searchAfterAndBulkCreate } from './search_after_bulk_create';
|
||||
import { scheduleNotificationActions } from '../notifications/schedule_notification_actions';
|
||||
import { RuleAlertType } from '../rules/types';
|
||||
import { findMlSignals } from './find_ml_signals';
|
||||
|
|
@ -36,7 +33,17 @@ jest.mock('./rule_status_saved_objects_client');
|
|||
jest.mock('./rule_status_service');
|
||||
jest.mock('./search_after_bulk_create');
|
||||
jest.mock('./get_filter');
|
||||
jest.mock('./utils');
|
||||
jest.mock('./utils', () => {
|
||||
const original = jest.requireActual('./utils');
|
||||
return {
|
||||
...original,
|
||||
getGapBetweenRuns: jest.fn(),
|
||||
getGapMaxCatchupRatio: jest.fn(),
|
||||
getListsClient: jest.fn(),
|
||||
getExceptions: jest.fn(),
|
||||
sortExceptionItems: jest.fn(),
|
||||
};
|
||||
});
|
||||
jest.mock('../notifications/schedule_notification_actions');
|
||||
jest.mock('./find_ml_signals');
|
||||
jest.mock('./bulk_create_ml_signals');
|
||||
|
|
@ -383,6 +390,7 @@ describe('rules_notification_alert_type', () => {
|
|||
},
|
||||
]);
|
||||
(findMlSignals as jest.Mock).mockResolvedValue({
|
||||
_shards: {},
|
||||
hits: {
|
||||
hits: [],
|
||||
},
|
||||
|
|
@ -401,6 +409,7 @@ describe('rules_notification_alert_type', () => {
|
|||
payload = getPayload(ruleAlert, alertServices) as jest.Mocked<RuleExecutorOptions>;
|
||||
jobsSummaryMock.mockResolvedValue([]);
|
||||
(findMlSignals as jest.Mock).mockResolvedValue({
|
||||
_shards: {},
|
||||
hits: {
|
||||
hits: [],
|
||||
},
|
||||
|
|
@ -409,6 +418,7 @@ describe('rules_notification_alert_type', () => {
|
|||
success: true,
|
||||
bulkCreateDuration: 0,
|
||||
createdItemsCount: 0,
|
||||
errors: [],
|
||||
});
|
||||
await alert.executor(payload);
|
||||
expect(ruleStatusService.success).not.toHaveBeenCalled();
|
||||
|
|
@ -425,6 +435,7 @@ describe('rules_notification_alert_type', () => {
|
|||
},
|
||||
]);
|
||||
(findMlSignals as jest.Mock).mockResolvedValue({
|
||||
_shards: { failed: 0 },
|
||||
hits: {
|
||||
hits: [{}],
|
||||
},
|
||||
|
|
@ -433,6 +444,7 @@ describe('rules_notification_alert_type', () => {
|
|||
success: true,
|
||||
bulkCreateDuration: 1,
|
||||
createdItemsCount: 1,
|
||||
errors: [],
|
||||
});
|
||||
await alert.executor(payload);
|
||||
expect(ruleStatusService.success).toHaveBeenCalled();
|
||||
|
|
@ -460,6 +472,7 @@ describe('rules_notification_alert_type', () => {
|
|||
});
|
||||
jobsSummaryMock.mockResolvedValue([]);
|
||||
(findMlSignals as jest.Mock).mockResolvedValue({
|
||||
_shards: { failed: 0 },
|
||||
hits: {
|
||||
hits: [{}],
|
||||
},
|
||||
|
|
@ -468,6 +481,7 @@ describe('rules_notification_alert_type', () => {
|
|||
success: true,
|
||||
bulkCreateDuration: 1,
|
||||
createdItemsCount: 1,
|
||||
errors: [],
|
||||
});
|
||||
|
||||
await alert.executor(payload);
|
||||
|
|
|
|||
|
|
@ -22,10 +22,7 @@ import {
|
|||
import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates';
|
||||
import { SetupPlugins } from '../../../plugin';
|
||||
import { getInputIndex } from './get_input_output_index';
|
||||
import {
|
||||
searchAfterAndBulkCreate,
|
||||
SearchAfterAndBulkCreateReturnType,
|
||||
} from './search_after_bulk_create';
|
||||
import { searchAfterAndBulkCreate } from './search_after_bulk_create';
|
||||
import { getFilter } from './get_filter';
|
||||
import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types';
|
||||
import {
|
||||
|
|
@ -34,6 +31,10 @@ import {
|
|||
getExceptions,
|
||||
getGapMaxCatchupRatio,
|
||||
MAX_RULE_GAP_RATIO,
|
||||
createErrorsFromShard,
|
||||
createSearchAfterReturnType,
|
||||
mergeReturns,
|
||||
createSearchAfterReturnTypeFromResponse,
|
||||
} from './utils';
|
||||
import { signalParamsSchema } from './signal_params_schema';
|
||||
import { siemRuleActionGroups } from './siem_rule_action_groups';
|
||||
|
|
@ -104,14 +105,7 @@ export const signalRulesAlertType = ({
|
|||
} = params;
|
||||
const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE);
|
||||
let hasError: boolean = false;
|
||||
let result: SearchAfterAndBulkCreateReturnType = {
|
||||
success: false,
|
||||
bulkCreateTimes: [],
|
||||
searchAfterTimes: [],
|
||||
lastLookBackDate: null,
|
||||
createdSignalsCount: 0,
|
||||
errors: [],
|
||||
};
|
||||
let result = createSearchAfterReturnType();
|
||||
const ruleStatusClient = ruleStatusSavedObjectsClientFactory(services.savedObjectsClient);
|
||||
const ruleStatusService = await ruleStatusServiceFactory({
|
||||
alertId,
|
||||
|
|
@ -255,12 +249,22 @@ export const signalRulesAlertType = ({
|
|||
refresh,
|
||||
tags,
|
||||
});
|
||||
result.success = success;
|
||||
result.errors = errors;
|
||||
result.createdSignalsCount = createdItemsCount;
|
||||
if (bulkCreateDuration) {
|
||||
result.bulkCreateTimes.push(bulkCreateDuration);
|
||||
}
|
||||
// The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] }
|
||||
const shardFailures =
|
||||
(anomalyResults._shards as typeof anomalyResults._shards & { failures: [] }).failures ??
|
||||
[];
|
||||
const searchErrors = createErrorsFromShard({
|
||||
errors: shardFailures,
|
||||
});
|
||||
result = mergeReturns([
|
||||
result,
|
||||
createSearchAfterReturnType({
|
||||
success: success && anomalyResults._shards.failed === 0,
|
||||
errors: [...errors, ...searchErrors],
|
||||
createdSignalsCount: createdItemsCount,
|
||||
bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [],
|
||||
}),
|
||||
]);
|
||||
} else if (isEqlRule(type)) {
|
||||
throw new Error('EQL Rules are under development, execution is not yet implemented');
|
||||
} else if (isThresholdRule(type) && threshold) {
|
||||
|
|
@ -276,7 +280,7 @@ export const signalRulesAlertType = ({
|
|||
lists: exceptionItems ?? [],
|
||||
});
|
||||
|
||||
const { searchResult: thresholdResults } = await findThresholdSignals({
|
||||
const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({
|
||||
inputIndexPattern: inputIndex,
|
||||
from,
|
||||
to,
|
||||
|
|
@ -313,12 +317,16 @@ export const signalRulesAlertType = ({
|
|||
refresh,
|
||||
tags,
|
||||
});
|
||||
result.success = success;
|
||||
result.errors = errors;
|
||||
result.createdSignalsCount = createdItemsCount;
|
||||
if (bulkCreateDuration) {
|
||||
result.bulkCreateTimes.push(bulkCreateDuration);
|
||||
}
|
||||
result = mergeReturns([
|
||||
result,
|
||||
createSearchAfterReturnTypeFromResponse({ searchResult: thresholdResults }),
|
||||
createSearchAfterReturnType({
|
||||
success,
|
||||
errors: [...errors, ...searchErrors],
|
||||
createdSignalsCount: createdItemsCount,
|
||||
bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [],
|
||||
}),
|
||||
]);
|
||||
} else if (isThreatMatchRule(type)) {
|
||||
if (
|
||||
threatQuery == null ||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import {
|
|||
} from './__mocks__/es_results';
|
||||
import { singleSearchAfter } from './single_search_after';
|
||||
import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks';
|
||||
import { ShardError } from '../../types';
|
||||
|
||||
describe('singleSearchAfter', () => {
|
||||
const mockService: AlertServicesMock = alertsMock.createAlertServices();
|
||||
|
|
@ -20,10 +21,9 @@ describe('singleSearchAfter', () => {
|
|||
});
|
||||
|
||||
test('if singleSearchAfter works without a given sort id', async () => {
|
||||
let searchAfterSortId;
|
||||
mockService.callCluster.mockResolvedValue(sampleDocSearchResultsNoSortId);
|
||||
mockService.callCluster.mockResolvedValue(sampleDocSearchResultsNoSortId());
|
||||
const { searchResult } = await singleSearchAfter({
|
||||
searchAfterSortId,
|
||||
searchAfterSortId: undefined,
|
||||
index: [],
|
||||
from: 'now-360s',
|
||||
to: 'now',
|
||||
|
|
@ -33,11 +33,73 @@ describe('singleSearchAfter', () => {
|
|||
filter: undefined,
|
||||
timestampOverride: undefined,
|
||||
});
|
||||
expect(searchResult).toEqual(sampleDocSearchResultsNoSortId);
|
||||
expect(searchResult).toEqual(sampleDocSearchResultsNoSortId());
|
||||
});
|
||||
test('if singleSearchAfter returns an empty failure array', async () => {
|
||||
mockService.callCluster.mockResolvedValue(sampleDocSearchResultsNoSortId());
|
||||
const { searchErrors } = await singleSearchAfter({
|
||||
searchAfterSortId: undefined,
|
||||
index: [],
|
||||
from: 'now-360s',
|
||||
to: 'now',
|
||||
services: mockService,
|
||||
logger: mockLogger,
|
||||
pageSize: 1,
|
||||
filter: undefined,
|
||||
timestampOverride: undefined,
|
||||
});
|
||||
expect(searchErrors).toEqual([]);
|
||||
});
|
||||
test('if singleSearchAfter will return an error array', async () => {
|
||||
const errors: ShardError[] = [
|
||||
{
|
||||
shard: 1,
|
||||
index: 'index-123',
|
||||
node: 'node-123',
|
||||
reason: {
|
||||
type: 'some type',
|
||||
reason: 'some reason',
|
||||
index_uuid: 'uuid-123',
|
||||
index: 'index-123',
|
||||
caused_by: {
|
||||
type: 'some type',
|
||||
reason: 'some reason',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
mockService.callCluster.mockResolvedValue({
|
||||
took: 10,
|
||||
timed_out: false,
|
||||
_shards: {
|
||||
total: 10,
|
||||
successful: 10,
|
||||
failed: 1,
|
||||
skipped: 0,
|
||||
failures: errors,
|
||||
},
|
||||
hits: {
|
||||
total: 100,
|
||||
max_score: 100,
|
||||
hits: [],
|
||||
},
|
||||
});
|
||||
const { searchErrors } = await singleSearchAfter({
|
||||
searchAfterSortId: undefined,
|
||||
index: [],
|
||||
from: 'now-360s',
|
||||
to: 'now',
|
||||
services: mockService,
|
||||
logger: mockLogger,
|
||||
pageSize: 1,
|
||||
filter: undefined,
|
||||
timestampOverride: undefined,
|
||||
});
|
||||
expect(searchErrors).toEqual(['reason: some reason, type: some type, caused by: some reason']);
|
||||
});
|
||||
test('if singleSearchAfter works with a given sort id', async () => {
|
||||
const searchAfterSortId = '1234567891111';
|
||||
mockService.callCluster.mockResolvedValue(sampleDocSearchResultsWithSortId);
|
||||
mockService.callCluster.mockResolvedValue(sampleDocSearchResultsWithSortId());
|
||||
const { searchResult } = await singleSearchAfter({
|
||||
searchAfterSortId,
|
||||
index: [],
|
||||
|
|
@ -49,7 +111,7 @@ describe('singleSearchAfter', () => {
|
|||
filter: undefined,
|
||||
timestampOverride: undefined,
|
||||
});
|
||||
expect(searchResult).toEqual(sampleDocSearchResultsWithSortId);
|
||||
expect(searchResult).toEqual(sampleDocSearchResultsWithSortId());
|
||||
});
|
||||
test('if singleSearchAfter throws error', async () => {
|
||||
const searchAfterSortId = '1234567891111';
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { AlertServices } from '../../../../../alerts/server';
|
|||
import { Logger } from '../../../../../../../src/core/server';
|
||||
import { SignalSearchResponse } from './types';
|
||||
import { buildEventsSearchQuery } from './build_events_query';
|
||||
import { makeFloatString } from './utils';
|
||||
import { createErrorsFromShard, makeFloatString } from './utils';
|
||||
import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas';
|
||||
|
||||
interface SingleSearchAfterParams {
|
||||
|
|
@ -40,6 +40,7 @@ export const singleSearchAfter = async ({
|
|||
}: SingleSearchAfterParams): Promise<{
|
||||
searchResult: SignalSearchResponse;
|
||||
searchDuration: string;
|
||||
searchErrors: string[];
|
||||
}> => {
|
||||
try {
|
||||
const searchAfterQuery = buildEventsSearchQuery({
|
||||
|
|
@ -59,7 +60,14 @@ export const singleSearchAfter = async ({
|
|||
searchAfterQuery
|
||||
);
|
||||
const end = performance.now();
|
||||
return { searchResult: nextSearchAfterResult, searchDuration: makeFloatString(end - start) };
|
||||
const searchErrors = createErrorsFromShard({
|
||||
errors: nextSearchAfterResult._shards.failures ?? [],
|
||||
});
|
||||
return {
|
||||
searchResult: nextSearchAfterResult,
|
||||
searchDuration: makeFloatString(end - start),
|
||||
searchErrors,
|
||||
};
|
||||
} catch (exc) {
|
||||
logger.error(`[-] nextSearchAfter threw an error ${exc}`);
|
||||
throw exc;
|
||||
|
|
|
|||
|
|
@ -9,12 +9,10 @@ import { getThreatList } from './get_threat_list';
|
|||
import { buildThreatMappingFilter } from './build_threat_mapping_filter';
|
||||
|
||||
import { getFilter } from '../get_filter';
|
||||
import {
|
||||
searchAfterAndBulkCreate,
|
||||
SearchAfterAndBulkCreateReturnType,
|
||||
} from '../search_after_bulk_create';
|
||||
import { searchAfterAndBulkCreate } from '../search_after_bulk_create';
|
||||
import { CreateThreatSignalOptions, ThreatListItem } from './types';
|
||||
import { combineResults } from './utils';
|
||||
import { SearchAfterAndBulkCreateReturnType } from '../types';
|
||||
|
||||
export const createThreatSignal = async ({
|
||||
threatMapping,
|
||||
|
|
|
|||
|
|
@ -6,9 +6,9 @@
|
|||
|
||||
import { getThreatList } from './get_threat_list';
|
||||
|
||||
import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create';
|
||||
import { CreateThreatSignalsOptions } from './types';
|
||||
import { createThreatSignal } from './create_threat_signal';
|
||||
import { SearchAfterAndBulkCreateReturnType } from '../types';
|
||||
|
||||
export const createThreatSignals = async ({
|
||||
threatMapping,
|
||||
|
|
|
|||
|
|
@ -19,10 +19,10 @@ import {
|
|||
import { PartialFilter, RuleTypeParams } from '../../types';
|
||||
import { AlertServices } from '../../../../../../alerts/server';
|
||||
import { ExceptionListItemSchema } from '../../../../../../lists/common/schemas';
|
||||
import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create';
|
||||
import { ILegacyScopedClusterClient, Logger } from '../../../../../../../../src/core/server';
|
||||
import { RuleAlertAction } from '../../../../../common/detection_engine/types';
|
||||
import { BuildRuleMessage } from '../rule_messages';
|
||||
import { SearchAfterAndBulkCreateReturnType } from '../types';
|
||||
|
||||
export interface CreateThreatSignalsOptions {
|
||||
threatMapping: ThreatMapping;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create';
|
||||
import { SearchAfterAndBulkCreateReturnType } from '../types';
|
||||
|
||||
import { calculateAdditiveMax, combineResults } from './utils';
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create';
|
||||
import { SearchAfterAndBulkCreateReturnType } from '../types';
|
||||
|
||||
/**
|
||||
* Given two timers this will take the max of each and add them to each other and return that addition.
|
||||
|
|
|
|||
|
|
@ -5,12 +5,22 @@
|
|||
*/
|
||||
|
||||
import { DslQuery, Filter } from 'src/plugins/data/common';
|
||||
import moment from 'moment';
|
||||
import { Status } from '../../../../common/detection_engine/schemas/common/schemas';
|
||||
import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
|
||||
import { AlertType, AlertTypeState, AlertExecutorOptions } from '../../../../../alerts/server';
|
||||
import {
|
||||
AlertType,
|
||||
AlertTypeState,
|
||||
AlertExecutorOptions,
|
||||
AlertServices,
|
||||
} from '../../../../../alerts/server';
|
||||
import { RuleAlertAction } from '../../../../common/detection_engine/types';
|
||||
import { RuleTypeParams } from '../types';
|
||||
import { RuleTypeParams, RefreshTypes } from '../types';
|
||||
import { SearchResponse } from '../../types';
|
||||
import { ListClient } from '../../../../../lists/server';
|
||||
import { Logger } from '../../../../../../../src/core/server';
|
||||
import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
|
||||
import { BuildRuleMessage } from './rule_messages';
|
||||
|
||||
// used for gap detection code
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
|
|
@ -179,3 +189,39 @@ export interface QueryFilter {
|
|||
must_not: Filter[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface SearchAfterAndBulkCreateParams {
|
||||
gap: moment.Duration | null;
|
||||
previousStartedAt: Date | null | undefined;
|
||||
ruleParams: RuleTypeParams;
|
||||
services: AlertServices;
|
||||
listClient: ListClient;
|
||||
exceptionsList: ExceptionListItemSchema[];
|
||||
logger: Logger;
|
||||
id: string;
|
||||
inputIndexPattern: string[];
|
||||
signalsIndex: string;
|
||||
name: string;
|
||||
actions: RuleAlertAction[];
|
||||
createdAt: string;
|
||||
createdBy: string;
|
||||
updatedBy: string;
|
||||
updatedAt: string;
|
||||
interval: string;
|
||||
enabled: boolean;
|
||||
pageSize: number;
|
||||
filter: unknown;
|
||||
refresh: RefreshTypes;
|
||||
tags: string[];
|
||||
throttle: string;
|
||||
buildRuleMessage: BuildRuleMessage;
|
||||
}
|
||||
|
||||
export interface SearchAfterAndBulkCreateReturnType {
|
||||
success: boolean;
|
||||
searchAfterTimes: string[];
|
||||
bulkCreateTimes: string[];
|
||||
lastLookBackDate: Date | null | undefined;
|
||||
createdSignalsCount: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,15 +25,25 @@ import {
|
|||
getListsClient,
|
||||
getSignalTimeTuples,
|
||||
getExceptions,
|
||||
createErrorsFromShard,
|
||||
createSearchAfterReturnTypeFromResponse,
|
||||
createSearchAfterReturnType,
|
||||
mergeReturns,
|
||||
createTotalHitsFromSearchResult,
|
||||
} from './utils';
|
||||
import { BulkResponseErrorAggregation } from './types';
|
||||
import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types';
|
||||
import {
|
||||
sampleBulkResponse,
|
||||
sampleEmptyBulkResponse,
|
||||
sampleBulkError,
|
||||
sampleBulkErrorItem,
|
||||
mockLogger,
|
||||
sampleDocSearchResultsWithSortId,
|
||||
sampleEmptyDocSearchResults,
|
||||
sampleDocSearchResultsNoSortIdNoHits,
|
||||
repeatedSearchResultsWithSortId,
|
||||
} from './__mocks__/es_results';
|
||||
import { ShardError } from '../../types';
|
||||
|
||||
const buildRuleMessage = buildRuleMessageFactory({
|
||||
id: 'fake id',
|
||||
|
|
@ -783,4 +793,278 @@ describe('utils', () => {
|
|||
expect(exceptions).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createErrorsFromShard', () => {
|
||||
test('empty errors will return an empty array', () => {
|
||||
const createdErrors = createErrorsFromShard({ errors: [] });
|
||||
expect(createdErrors).toEqual([]);
|
||||
});
|
||||
|
||||
test('single error will return single converted array of a string of a reason', () => {
|
||||
const errors: ShardError[] = [
|
||||
{
|
||||
shard: 1,
|
||||
index: 'index-123',
|
||||
node: 'node-123',
|
||||
reason: {
|
||||
type: 'some type',
|
||||
reason: 'some reason',
|
||||
index_uuid: 'uuid-123',
|
||||
index: 'index-123',
|
||||
caused_by: {
|
||||
type: 'some type',
|
||||
reason: 'some reason',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
const createdErrors = createErrorsFromShard({ errors });
|
||||
expect(createdErrors).toEqual([
|
||||
'reason: some reason, type: some type, caused by: some reason',
|
||||
]);
|
||||
});
|
||||
|
||||
test('two errors will return two converted arrays to a string of a reason', () => {
|
||||
const errors: ShardError[] = [
|
||||
{
|
||||
shard: 1,
|
||||
index: 'index-123',
|
||||
node: 'node-123',
|
||||
reason: {
|
||||
type: 'some type',
|
||||
reason: 'some reason',
|
||||
index_uuid: 'uuid-123',
|
||||
index: 'index-123',
|
||||
caused_by: {
|
||||
type: 'some type',
|
||||
reason: 'some reason',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
shard: 2,
|
||||
index: 'index-345',
|
||||
node: 'node-345',
|
||||
reason: {
|
||||
type: 'some type 2',
|
||||
reason: 'some reason 2',
|
||||
index_uuid: 'uuid-345',
|
||||
index: 'index-345',
|
||||
caused_by: {
|
||||
type: 'some type 2',
|
||||
reason: 'some reason 2',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
const createdErrors = createErrorsFromShard({ errors });
|
||||
expect(createdErrors).toEqual([
|
||||
'reason: some reason, type: some type, caused by: some reason',
|
||||
'reason: some reason 2, type: some type 2, caused by: some reason 2',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createSearchAfterReturnTypeFromResponse', () => {
|
||||
test('empty results will return successful type', () => {
|
||||
const searchResult = sampleEmptyDocSearchResults();
|
||||
const newSearchResult = createSearchAfterReturnTypeFromResponse({ searchResult });
|
||||
const expected: SearchAfterAndBulkCreateReturnType = {
|
||||
bulkCreateTimes: [],
|
||||
createdSignalsCount: 0,
|
||||
errors: [],
|
||||
lastLookBackDate: null,
|
||||
searchAfterTimes: [],
|
||||
success: true,
|
||||
};
|
||||
expect(newSearchResult).toEqual(expected);
|
||||
});
|
||||
|
||||
test('multiple results will return successful type with expected success', () => {
|
||||
const searchResult = sampleDocSearchResultsWithSortId();
|
||||
const newSearchResult = createSearchAfterReturnTypeFromResponse({ searchResult });
|
||||
const expected: SearchAfterAndBulkCreateReturnType = {
|
||||
bulkCreateTimes: [],
|
||||
createdSignalsCount: 0,
|
||||
errors: [],
|
||||
lastLookBackDate: new Date('2020-04-20T21:27:45.000Z'),
|
||||
searchAfterTimes: [],
|
||||
success: true,
|
||||
};
|
||||
expect(newSearchResult).toEqual(expected);
|
||||
});
|
||||
|
||||
test('result with error will create success: false within the result set', () => {
|
||||
const searchResult = sampleDocSearchResultsNoSortIdNoHits();
|
||||
searchResult._shards.failed = 1;
|
||||
const { success } = createSearchAfterReturnTypeFromResponse({ searchResult });
|
||||
expect(success).toEqual(false);
|
||||
});
|
||||
|
||||
test('result with error will create success: false within the result set if failed is 2 or more', () => {
|
||||
const searchResult = sampleDocSearchResultsNoSortIdNoHits();
|
||||
searchResult._shards.failed = 2;
|
||||
const { success } = createSearchAfterReturnTypeFromResponse({ searchResult });
|
||||
expect(success).toEqual(false);
|
||||
});
|
||||
|
||||
test('result with error will create success: true within the result set if failed is 0', () => {
|
||||
const searchResult = sampleDocSearchResultsNoSortIdNoHits();
|
||||
searchResult._shards.failed = 0;
|
||||
const { success } = createSearchAfterReturnTypeFromResponse({ searchResult });
|
||||
expect(success).toEqual(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createSearchAfterReturnType', () => {
|
||||
test('createSearchAfterReturnType will return full object when nothing is passed', () => {
|
||||
const searchAfterReturnType = createSearchAfterReturnType();
|
||||
const expected: SearchAfterAndBulkCreateReturnType = {
|
||||
bulkCreateTimes: [],
|
||||
createdSignalsCount: 0,
|
||||
errors: [],
|
||||
lastLookBackDate: null,
|
||||
searchAfterTimes: [],
|
||||
success: true,
|
||||
};
|
||||
expect(searchAfterReturnType).toEqual(expected);
|
||||
});
|
||||
|
||||
test('createSearchAfterReturnType can override all values', () => {
|
||||
const searchAfterReturnType = createSearchAfterReturnType({
|
||||
bulkCreateTimes: ['123'],
|
||||
createdSignalsCount: 5,
|
||||
errors: ['error 1'],
|
||||
lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'),
|
||||
searchAfterTimes: ['123'],
|
||||
success: false,
|
||||
});
|
||||
const expected: SearchAfterAndBulkCreateReturnType = {
|
||||
bulkCreateTimes: ['123'],
|
||||
createdSignalsCount: 5,
|
||||
errors: ['error 1'],
|
||||
lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'),
|
||||
searchAfterTimes: ['123'],
|
||||
success: false,
|
||||
};
|
||||
expect(searchAfterReturnType).toEqual(expected);
|
||||
});
|
||||
|
||||
test('createSearchAfterReturnType can override select values', () => {
|
||||
const searchAfterReturnType = createSearchAfterReturnType({
|
||||
createdSignalsCount: 5,
|
||||
errors: ['error 1'],
|
||||
});
|
||||
const expected: SearchAfterAndBulkCreateReturnType = {
|
||||
bulkCreateTimes: [],
|
||||
createdSignalsCount: 5,
|
||||
errors: ['error 1'],
|
||||
lastLookBackDate: null,
|
||||
searchAfterTimes: [],
|
||||
success: true,
|
||||
};
|
||||
expect(searchAfterReturnType).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeReturns', () => {
|
||||
test('it merges a default "prev" and "next" correctly ', () => {
|
||||
const merged = mergeReturns([createSearchAfterReturnType(), createSearchAfterReturnType()]);
|
||||
const expected: SearchAfterAndBulkCreateReturnType = {
|
||||
bulkCreateTimes: [],
|
||||
createdSignalsCount: 0,
|
||||
errors: [],
|
||||
lastLookBackDate: null,
|
||||
searchAfterTimes: [],
|
||||
success: true,
|
||||
};
|
||||
expect(merged).toEqual(expected);
|
||||
});
|
||||
|
||||
test('it merges search in with two default search results where "prev" "success" is false correctly', () => {
|
||||
const { success } = mergeReturns([
|
||||
createSearchAfterReturnType({ success: false }),
|
||||
createSearchAfterReturnType(),
|
||||
]);
|
||||
expect(success).toEqual(false);
|
||||
});
|
||||
|
||||
test('it merges search in with two default search results where "next" "success" is false correctly', () => {
|
||||
const { success } = mergeReturns([
|
||||
createSearchAfterReturnType(),
|
||||
createSearchAfterReturnType({ success: false }),
|
||||
]);
|
||||
expect(success).toEqual(false);
|
||||
});
|
||||
|
||||
test('it merges search where the lastLookBackDate is the "next" date when given', () => {
|
||||
const { lastLookBackDate } = mergeReturns([
|
||||
createSearchAfterReturnType({
|
||||
lastLookBackDate: new Date('2020-08-21T19:21:46.194Z'),
|
||||
}),
|
||||
createSearchAfterReturnType({
|
||||
lastLookBackDate: new Date('2020-09-21T19:21:46.194Z'),
|
||||
}),
|
||||
]);
|
||||
expect(lastLookBackDate).toEqual(new Date('2020-09-21T19:21:46.194Z'));
|
||||
});
|
||||
|
||||
test('it merges search where the lastLookBackDate is the "prev" if given undefined for "next', () => {
|
||||
const { lastLookBackDate } = mergeReturns([
|
||||
createSearchAfterReturnType({
|
||||
lastLookBackDate: new Date('2020-08-21T19:21:46.194Z'),
|
||||
}),
|
||||
createSearchAfterReturnType({
|
||||
lastLookBackDate: undefined,
|
||||
}),
|
||||
]);
|
||||
expect(lastLookBackDate).toEqual(new Date('2020-08-21T19:21:46.194Z'));
|
||||
});
|
||||
|
||||
test('it merges search where values from "next" and "prev" are computed together', () => {
|
||||
const merged = mergeReturns([
|
||||
createSearchAfterReturnType({
|
||||
bulkCreateTimes: ['123'],
|
||||
createdSignalsCount: 3,
|
||||
errors: ['error 1', 'error 2'],
|
||||
lastLookBackDate: new Date('2020-08-21T18:51:25.193Z'),
|
||||
searchAfterTimes: ['123'],
|
||||
success: true,
|
||||
}),
|
||||
createSearchAfterReturnType({
|
||||
bulkCreateTimes: ['456'],
|
||||
createdSignalsCount: 2,
|
||||
errors: ['error 3'],
|
||||
lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'),
|
||||
searchAfterTimes: ['567'],
|
||||
success: true,
|
||||
}),
|
||||
]);
|
||||
const expected: SearchAfterAndBulkCreateReturnType = {
|
||||
bulkCreateTimes: ['123', '456'], // concatenates the prev and next together
|
||||
createdSignalsCount: 5, // Adds the 3 and 2 together
|
||||
errors: ['error 1', 'error 2', 'error 3'], // concatenates the prev and next together
|
||||
lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), // takes the next lastLookBackDate
|
||||
searchAfterTimes: ['123', '567'], // concatenates the searchAfterTimes together
|
||||
success: true, // Defaults to success true is all of it was successful
|
||||
};
|
||||
expect(merged).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createTotalHitsFromSearchResult', () => {
|
||||
test('it should return 0 for empty results', () => {
|
||||
const result = createTotalHitsFromSearchResult({
|
||||
searchResult: sampleEmptyDocSearchResults(),
|
||||
});
|
||||
expect(result).toEqual(0);
|
||||
});
|
||||
|
||||
test('it should return 4 for 4 result sets', () => {
|
||||
const result = createTotalHitsFromSearchResult({
|
||||
searchResult: repeatedSearchResultsWithSortId(4, 1, ['1', '2', '3', '4']),
|
||||
});
|
||||
expect(result).toEqual(4);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -12,11 +12,18 @@ import { AlertServices, parseDuration } from '../../../../../alerts/server';
|
|||
import { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server';
|
||||
import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
|
||||
import { ListArray } from '../../../../common/detection_engine/schemas/types/lists';
|
||||
import { BulkResponse, BulkResponseErrorAggregation, isValidUnit } from './types';
|
||||
import {
|
||||
BulkResponse,
|
||||
BulkResponseErrorAggregation,
|
||||
isValidUnit,
|
||||
SearchAfterAndBulkCreateReturnType,
|
||||
SignalSearchResponse,
|
||||
} from './types';
|
||||
import { BuildRuleMessage } from './rule_messages';
|
||||
import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates';
|
||||
import { hasLargeValueList } from '../../../../common/detection_engine/utils';
|
||||
import { MAX_EXCEPTION_LIST_SIZE } from '../../../../../lists/common/constants';
|
||||
import { ShardError } from '../../types';
|
||||
|
||||
interface SortExceptionsReturn {
|
||||
exceptionsWithValueLists: ExceptionListItemSchema[];
|
||||
|
|
@ -439,3 +446,97 @@ export const getSignalTimeTuples = ({
|
|||
);
|
||||
return totalToFromTuples;
|
||||
};
|
||||
|
||||
/**
|
||||
* Given errors from a search query this will return an array of strings derived from the errors.
|
||||
* @param errors The errors to derive the strings from
|
||||
*/
|
||||
export const createErrorsFromShard = ({ errors }: { errors: ShardError[] }): string[] => {
|
||||
return errors.map((error) => {
|
||||
return `reason: ${error.reason.reason}, type: ${error.reason.caused_by.type}, caused by: ${error.reason.caused_by.reason}`;
|
||||
});
|
||||
};
|
||||
|
||||
export const createSearchAfterReturnTypeFromResponse = ({
|
||||
searchResult,
|
||||
}: {
|
||||
searchResult: SignalSearchResponse;
|
||||
}): SearchAfterAndBulkCreateReturnType => {
|
||||
return createSearchAfterReturnType({
|
||||
success: searchResult._shards.failed === 0,
|
||||
lastLookBackDate:
|
||||
searchResult.hits.hits.length > 0
|
||||
? new Date(searchResult.hits.hits[searchResult.hits.hits.length - 1]?._source['@timestamp'])
|
||||
: undefined,
|
||||
});
|
||||
};
|
||||
|
||||
export const createSearchAfterReturnType = ({
|
||||
success,
|
||||
searchAfterTimes,
|
||||
bulkCreateTimes,
|
||||
lastLookBackDate,
|
||||
createdSignalsCount,
|
||||
errors,
|
||||
}: {
|
||||
success?: boolean | undefined;
|
||||
searchAfterTimes?: string[] | undefined;
|
||||
bulkCreateTimes?: string[] | undefined;
|
||||
lastLookBackDate?: Date | undefined;
|
||||
createdSignalsCount?: number | undefined;
|
||||
errors?: string[] | undefined;
|
||||
} = {}): SearchAfterAndBulkCreateReturnType => {
|
||||
return {
|
||||
success: success ?? true,
|
||||
searchAfterTimes: searchAfterTimes ?? [],
|
||||
bulkCreateTimes: bulkCreateTimes ?? [],
|
||||
lastLookBackDate: lastLookBackDate ?? null,
|
||||
createdSignalsCount: createdSignalsCount ?? 0,
|
||||
errors: errors ?? [],
|
||||
};
|
||||
};
|
||||
|
||||
export const mergeReturns = (
|
||||
searchAfters: SearchAfterAndBulkCreateReturnType[]
|
||||
): SearchAfterAndBulkCreateReturnType => {
|
||||
return searchAfters.reduce((prev, next) => {
|
||||
const {
|
||||
success: existingSuccess,
|
||||
searchAfterTimes: existingSearchAfterTimes,
|
||||
bulkCreateTimes: existingBulkCreateTimes,
|
||||
lastLookBackDate: existingLastLookBackDate,
|
||||
createdSignalsCount: existingCreatedSignalsCount,
|
||||
errors: existingErrors,
|
||||
} = prev;
|
||||
|
||||
const {
|
||||
success: newSuccess,
|
||||
searchAfterTimes: newSearchAfterTimes,
|
||||
bulkCreateTimes: newBulkCreateTimes,
|
||||
lastLookBackDate: newLastLookBackDate,
|
||||
createdSignalsCount: newCreatedSignalsCount,
|
||||
errors: newErrors,
|
||||
} = next;
|
||||
|
||||
return {
|
||||
success: existingSuccess && newSuccess,
|
||||
searchAfterTimes: [...existingSearchAfterTimes, ...newSearchAfterTimes],
|
||||
bulkCreateTimes: [...existingBulkCreateTimes, ...newBulkCreateTimes],
|
||||
lastLookBackDate: newLastLookBackDate ?? existingLastLookBackDate,
|
||||
createdSignalsCount: existingCreatedSignalsCount + newCreatedSignalsCount,
|
||||
errors: [...new Set([...existingErrors, ...newErrors])],
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
export const createTotalHitsFromSearchResult = ({
|
||||
searchResult,
|
||||
}: {
|
||||
searchResult: SignalSearchResponse;
|
||||
}): number => {
|
||||
const totalHits =
|
||||
typeof searchResult.hits.total === 'number'
|
||||
? searchResult.hits.total
|
||||
: searchResult.hits.total.value;
|
||||
return totalHits;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -98,6 +98,23 @@ export interface ShardsResponse {
|
|||
successful: number;
|
||||
failed: number;
|
||||
skipped: number;
|
||||
failures?: ShardError[];
|
||||
}
|
||||
|
||||
export interface ShardError {
|
||||
shard: number;
|
||||
index: string;
|
||||
node: string;
|
||||
reason: {
|
||||
type: string;
|
||||
reason: string;
|
||||
index_uuid: string;
|
||||
index: string;
|
||||
caused_by: {
|
||||
type: string;
|
||||
reason: string;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export interface Explanation {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue