mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 17:59:23 -04:00
[Security Solution][Exceptions] Implement exceptions for ML rules (#84006)
* Implement exceptions for ML rules * Remove unused import * Better implicit types * Retrieve ML rule index pattern for exception field suggestions and autocomplete * Add ML job logic to edit exception modal * Remove unnecessary logic change Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Marshall Main
GitHub
parent
4f3d72b413
commit
d47c70cd53
17 changed files with 552 additions and 179 deletions
|
|
@ -7,6 +7,7 @@
|
|||
import { getQueryFilter, buildExceptionFilter, buildEqlSearchRequest } from './get_query_filter';
|
||||
import { Filter, EsQueryConfig } from 'src/plugins/data/public';
|
||||
import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock';
|
||||
import { ExceptionListItemSchema } from '../shared_imports';
|
||||
|
||||
describe('get_filter', () => {
|
||||
describe('getQueryFilter', () => {
|
||||
|
|
@ -919,19 +920,27 @@ describe('get_filter', () => {
|
|||
dateFormatTZ: 'Zulu',
|
||||
};
|
||||
test('it should build a filter without chunking exception items', () => {
|
||||
const exceptionFilter = buildExceptionFilter(
|
||||
[
|
||||
{ language: 'kuery', query: 'host.name: linux and some.field: value' },
|
||||
{ language: 'kuery', query: 'user.name: name' },
|
||||
const exceptionItem1: ExceptionListItemSchema = {
|
||||
...getExceptionListItemSchemaMock(),
|
||||
entries: [
|
||||
{ field: 'host.name', operator: 'included', type: 'match', value: 'linux' },
|
||||
{ field: 'some.field', operator: 'included', type: 'match', value: 'value' },
|
||||
],
|
||||
{
|
||||
};
|
||||
const exceptionItem2: ExceptionListItemSchema = {
|
||||
...getExceptionListItemSchemaMock(),
|
||||
entries: [{ field: 'user.name', operator: 'included', type: 'match', value: 'name' }],
|
||||
};
|
||||
const exceptionFilter = buildExceptionFilter({
|
||||
lists: [exceptionItem1, exceptionItem2],
|
||||
config,
|
||||
excludeExceptions: true,
|
||||
chunkSize: 2,
|
||||
indexPattern: {
|
||||
fields: [],
|
||||
title: 'auditbeat-*',
|
||||
},
|
||||
config,
|
||||
true,
|
||||
2
|
||||
);
|
||||
});
|
||||
expect(exceptionFilter).toEqual({
|
||||
meta: {
|
||||
alias: null,
|
||||
|
|
@ -949,7 +958,7 @@ describe('get_filter', () => {
|
|||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
match: {
|
||||
match_phrase: {
|
||||
'host.name': 'linux',
|
||||
},
|
||||
},
|
||||
|
|
@ -961,7 +970,7 @@ describe('get_filter', () => {
|
|||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
match: {
|
||||
match_phrase: {
|
||||
'some.field': 'value',
|
||||
},
|
||||
},
|
||||
|
|
@ -976,7 +985,7 @@ describe('get_filter', () => {
|
|||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
match: {
|
||||
match_phrase: {
|
||||
'user.name': 'name',
|
||||
},
|
||||
},
|
||||
|
|
@ -990,20 +999,31 @@ describe('get_filter', () => {
|
|||
});
|
||||
|
||||
test('it should properly chunk exception items', () => {
|
||||
const exceptionFilter = buildExceptionFilter(
|
||||
[
|
||||
{ language: 'kuery', query: 'host.name: linux and some.field: value' },
|
||||
{ language: 'kuery', query: 'user.name: name' },
|
||||
{ language: 'kuery', query: 'file.path: /safe/path' },
|
||||
const exceptionItem1: ExceptionListItemSchema = {
|
||||
...getExceptionListItemSchemaMock(),
|
||||
entries: [
|
||||
{ field: 'host.name', operator: 'included', type: 'match', value: 'linux' },
|
||||
{ field: 'some.field', operator: 'included', type: 'match', value: 'value' },
|
||||
],
|
||||
{
|
||||
};
|
||||
const exceptionItem2: ExceptionListItemSchema = {
|
||||
...getExceptionListItemSchemaMock(),
|
||||
entries: [{ field: 'user.name', operator: 'included', type: 'match', value: 'name' }],
|
||||
};
|
||||
const exceptionItem3: ExceptionListItemSchema = {
|
||||
...getExceptionListItemSchemaMock(),
|
||||
entries: [{ field: 'file.path', operator: 'included', type: 'match', value: '/safe/path' }],
|
||||
};
|
||||
const exceptionFilter = buildExceptionFilter({
|
||||
lists: [exceptionItem1, exceptionItem2, exceptionItem3],
|
||||
config,
|
||||
excludeExceptions: true,
|
||||
chunkSize: 2,
|
||||
indexPattern: {
|
||||
fields: [],
|
||||
title: 'auditbeat-*',
|
||||
},
|
||||
config,
|
||||
true,
|
||||
2
|
||||
);
|
||||
});
|
||||
expect(exceptionFilter).toEqual({
|
||||
meta: {
|
||||
alias: null,
|
||||
|
|
@ -1024,7 +1044,7 @@ describe('get_filter', () => {
|
|||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
match: {
|
||||
match_phrase: {
|
||||
'host.name': 'linux',
|
||||
},
|
||||
},
|
||||
|
|
@ -1036,7 +1056,7 @@ describe('get_filter', () => {
|
|||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
match: {
|
||||
match_phrase: {
|
||||
'some.field': 'value',
|
||||
},
|
||||
},
|
||||
|
|
@ -1051,7 +1071,7 @@ describe('get_filter', () => {
|
|||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
match: {
|
||||
match_phrase: {
|
||||
'user.name': 'name',
|
||||
},
|
||||
},
|
||||
|
|
@ -1069,7 +1089,7 @@ describe('get_filter', () => {
|
|||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
match: {
|
||||
match_phrase: {
|
||||
'file.path': '/safe/path',
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@
|
|||
|
||||
import {
|
||||
Filter,
|
||||
Query,
|
||||
IIndexPattern,
|
||||
isFilterDisabled,
|
||||
buildEsQuery,
|
||||
|
|
@ -18,15 +17,10 @@ import {
|
|||
} from '../../../lists/common/schemas';
|
||||
import { ESBoolQuery } from '../typed_json';
|
||||
import { buildExceptionListQueries } from './build_exceptions_query';
|
||||
import {
|
||||
Query as QueryString,
|
||||
Language,
|
||||
Index,
|
||||
TimestampOverrideOrUndefined,
|
||||
} from './schemas/common/schemas';
|
||||
import { Query, Language, Index, TimestampOverrideOrUndefined } from './schemas/common/schemas';
|
||||
|
||||
export const getQueryFilter = (
|
||||
query: QueryString,
|
||||
query: Query,
|
||||
language: Language,
|
||||
filters: Array<Partial<Filter>>,
|
||||
index: Index,
|
||||
|
|
@ -53,19 +47,18 @@ export const getQueryFilter = (
|
|||
* buildEsQuery, this allows us to offer nested queries
|
||||
* regardless
|
||||
*/
|
||||
const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists });
|
||||
if (exceptionQueries.length > 0) {
|
||||
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
|
||||
// allowing us to make 1024-item chunks of exception list items.
|
||||
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
|
||||
// very conservative value.
|
||||
const exceptionFilter = buildExceptionFilter(
|
||||
exceptionQueries,
|
||||
indexPattern,
|
||||
config,
|
||||
excludeExceptions,
|
||||
1024
|
||||
);
|
||||
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
|
||||
// allowing us to make 1024-item chunks of exception list items.
|
||||
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
|
||||
// very conservative value.
|
||||
const exceptionFilter = buildExceptionFilter({
|
||||
lists,
|
||||
config,
|
||||
excludeExceptions,
|
||||
chunkSize: 1024,
|
||||
indexPattern,
|
||||
});
|
||||
if (exceptionFilter !== undefined) {
|
||||
enabledFilters.push(exceptionFilter);
|
||||
}
|
||||
const initialQuery = { query, language };
|
||||
|
|
@ -101,15 +94,17 @@ export const buildEqlSearchRequest = (
|
|||
ignoreFilterIfFieldNotInIndex: false,
|
||||
dateFormatTZ: 'Zulu',
|
||||
};
|
||||
const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists });
|
||||
let exceptionFilter: Filter | undefined;
|
||||
if (exceptionQueries.length > 0) {
|
||||
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
|
||||
// allowing us to make 1024-item chunks of exception list items.
|
||||
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
|
||||
// very conservative value.
|
||||
exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024);
|
||||
}
|
||||
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
|
||||
// allowing us to make 1024-item chunks of exception list items.
|
||||
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
|
||||
// very conservative value.
|
||||
const exceptionFilter = buildExceptionFilter({
|
||||
lists: exceptionLists,
|
||||
config,
|
||||
excludeExceptions: true,
|
||||
chunkSize: 1024,
|
||||
indexPattern,
|
||||
});
|
||||
const indexString = index.join();
|
||||
const requestFilter: unknown[] = [
|
||||
{
|
||||
|
|
@ -154,13 +149,23 @@ export const buildEqlSearchRequest = (
|
|||
}
|
||||
};
|
||||
|
||||
export const buildExceptionFilter = (
|
||||
exceptionQueries: Query[],
|
||||
indexPattern: IIndexPattern,
|
||||
config: EsQueryConfig,
|
||||
excludeExceptions: boolean,
|
||||
chunkSize: number
|
||||
) => {
|
||||
export const buildExceptionFilter = ({
|
||||
lists,
|
||||
config,
|
||||
excludeExceptions,
|
||||
chunkSize,
|
||||
indexPattern,
|
||||
}: {
|
||||
lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>;
|
||||
config: EsQueryConfig;
|
||||
excludeExceptions: boolean;
|
||||
chunkSize: number;
|
||||
indexPattern?: IIndexPattern;
|
||||
}) => {
|
||||
const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists });
|
||||
if (exceptionQueries.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const exceptionFilter: Filter = {
|
||||
meta: {
|
||||
alias: null,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
/* eslint complexity: ["error", 30]*/
|
||||
|
||||
import React, { memo, useEffect, useState, useCallback, useMemo } from 'react';
|
||||
import styled, { css } from 'styled-components';
|
||||
import {
|
||||
|
|
@ -53,6 +55,7 @@ import {
|
|||
import { ErrorInfo, ErrorCallout } from '../error_callout';
|
||||
import { ExceptionsBuilderExceptionItem } from '../types';
|
||||
import { useFetchIndex } from '../../../containers/source';
|
||||
import { useGetInstalledJob } from '../../ml/hooks/use_get_jobs';
|
||||
|
||||
export interface AddExceptionModalProps {
|
||||
ruleName: string;
|
||||
|
|
@ -108,7 +111,7 @@ export const AddExceptionModal = memo(function AddExceptionModal({
|
|||
const { http } = useKibana().services;
|
||||
const [errorsExist, setErrorExists] = useState(false);
|
||||
const [comment, setComment] = useState('');
|
||||
const { rule: maybeRule } = useRuleAsync(ruleId);
|
||||
const { rule: maybeRule, loading: isRuleLoading } = useRuleAsync(ruleId);
|
||||
const [shouldCloseAlert, setShouldCloseAlert] = useState(false);
|
||||
const [shouldBulkCloseAlert, setShouldBulkCloseAlert] = useState(false);
|
||||
const [shouldDisableBulkClose, setShouldDisableBulkClose] = useState(false);
|
||||
|
|
@ -124,8 +127,22 @@ export const AddExceptionModal = memo(function AddExceptionModal({
|
|||
const [isSignalIndexPatternLoading, { indexPatterns: signalIndexPatterns }] = useFetchIndex(
|
||||
memoSignalIndexName
|
||||
);
|
||||
const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(ruleIndices);
|
||||
|
||||
const memoMlJobIds = useMemo(
|
||||
() => (maybeRule?.machine_learning_job_id != null ? [maybeRule.machine_learning_job_id] : []),
|
||||
[maybeRule]
|
||||
);
|
||||
const { loading: mlJobLoading, jobs } = useGetInstalledJob(memoMlJobIds);
|
||||
|
||||
const memoRuleIndices = useMemo(() => {
|
||||
if (jobs.length > 0) {
|
||||
return jobs[0].results_index_name ? [`.ml-anomalies-${jobs[0].results_index_name}`] : [];
|
||||
} else {
|
||||
return ruleIndices;
|
||||
}
|
||||
}, [jobs, ruleIndices]);
|
||||
|
||||
const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(memoRuleIndices);
|
||||
const onError = useCallback(
|
||||
(error: Error): void => {
|
||||
addError(error, { title: i18n.ADD_EXCEPTION_ERROR });
|
||||
|
|
@ -364,6 +381,8 @@ export const AddExceptionModal = memo(function AddExceptionModal({
|
|||
!isSignalIndexPatternLoading &&
|
||||
!isLoadingExceptionList &&
|
||||
!isIndexPatternLoading &&
|
||||
!isRuleLoading &&
|
||||
!mlJobLoading &&
|
||||
ruleExceptionList && (
|
||||
<>
|
||||
<ModalBodySection className="builder-section">
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ import {
|
|||
} from '../helpers';
|
||||
import { Loader } from '../../loader';
|
||||
import { ErrorInfo, ErrorCallout } from '../error_callout';
|
||||
import { useGetInstalledJob } from '../../ml/hooks/use_get_jobs';
|
||||
|
||||
interface EditExceptionModalProps {
|
||||
ruleName: string;
|
||||
|
|
@ -100,7 +101,7 @@ export const EditExceptionModal = memo(function EditExceptionModal({
|
|||
const { http } = useKibana().services;
|
||||
const [comment, setComment] = useState('');
|
||||
const [errorsExist, setErrorExists] = useState(false);
|
||||
const { rule: maybeRule } = useRuleAsync(ruleId);
|
||||
const { rule: maybeRule, loading: isRuleLoading } = useRuleAsync(ruleId);
|
||||
const [updateError, setUpdateError] = useState<ErrorInfo | null>(null);
|
||||
const [hasVersionConflict, setHasVersionConflict] = useState(false);
|
||||
const [shouldBulkCloseAlert, setShouldBulkCloseAlert] = useState(false);
|
||||
|
|
@ -117,7 +118,21 @@ export const EditExceptionModal = memo(function EditExceptionModal({
|
|||
memoSignalIndexName
|
||||
);
|
||||
|
||||
const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(ruleIndices);
|
||||
const memoMlJobIds = useMemo(
|
||||
() => (maybeRule?.machine_learning_job_id != null ? [maybeRule.machine_learning_job_id] : []),
|
||||
[maybeRule]
|
||||
);
|
||||
const { loading: mlJobLoading, jobs } = useGetInstalledJob(memoMlJobIds);
|
||||
|
||||
const memoRuleIndices = useMemo(() => {
|
||||
if (jobs.length > 0) {
|
||||
return jobs[0].results_index_name ? [`.ml-anomalies-${jobs[0].results_index_name}`] : [];
|
||||
} else {
|
||||
return ruleIndices;
|
||||
}
|
||||
}, [jobs, ruleIndices]);
|
||||
|
||||
const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(memoRuleIndices);
|
||||
|
||||
const handleExceptionUpdateError = useCallback(
|
||||
(error: Error, statusCode: number | null, message: string | null) => {
|
||||
|
|
@ -280,69 +295,75 @@ export const EditExceptionModal = memo(function EditExceptionModal({
|
|||
{(addExceptionIsLoading || isIndexPatternLoading || isSignalIndexLoading) && (
|
||||
<Loader data-test-subj="loadingEditExceptionModal" size="xl" />
|
||||
)}
|
||||
{!isSignalIndexLoading && !addExceptionIsLoading && !isIndexPatternLoading && (
|
||||
<>
|
||||
<ModalBodySection className="builder-section">
|
||||
{isRuleEQLSequenceStatement && (
|
||||
<>
|
||||
<EuiCallOut
|
||||
data-test-subj="eql-sequence-callout"
|
||||
title={i18n.EDIT_EXCEPTION_SEQUENCE_WARNING}
|
||||
/>
|
||||
<EuiSpacer />
|
||||
</>
|
||||
)}
|
||||
<EuiText>{i18n.EXCEPTION_BUILDER_INFO}</EuiText>
|
||||
<EuiSpacer />
|
||||
<ExceptionBuilderComponent
|
||||
exceptionListItems={[exceptionItem]}
|
||||
listType={exceptionListType}
|
||||
listId={exceptionItem.list_id}
|
||||
listNamespaceType={exceptionItem.namespace_type}
|
||||
ruleName={ruleName}
|
||||
isOrDisabled
|
||||
isAndDisabled={false}
|
||||
isNestedDisabled={false}
|
||||
data-test-subj="edit-exception-modal-builder"
|
||||
id-aria="edit-exception-modal-builder"
|
||||
onChange={handleBuilderOnChange}
|
||||
indexPatterns={indexPatterns}
|
||||
ruleType={maybeRule?.type}
|
||||
/>
|
||||
|
||||
<EuiSpacer />
|
||||
|
||||
<AddExceptionComments
|
||||
exceptionItemComments={exceptionItem.comments}
|
||||
newCommentValue={comment}
|
||||
newCommentOnChange={onCommentChange}
|
||||
/>
|
||||
</ModalBodySection>
|
||||
<EuiHorizontalRule />
|
||||
<ModalBodySection>
|
||||
<EuiFormRow fullWidth>
|
||||
<EuiCheckbox
|
||||
data-test-subj="close-alert-on-add-edit-exception-checkbox"
|
||||
id="close-alert-on-add-edit-exception-checkbox"
|
||||
label={
|
||||
shouldDisableBulkClose ? i18n.BULK_CLOSE_LABEL_DISABLED : i18n.BULK_CLOSE_LABEL
|
||||
}
|
||||
checked={shouldBulkCloseAlert}
|
||||
onChange={onBulkCloseAlertCheckboxChange}
|
||||
disabled={shouldDisableBulkClose}
|
||||
{!isSignalIndexLoading &&
|
||||
!addExceptionIsLoading &&
|
||||
!isIndexPatternLoading &&
|
||||
!isRuleLoading &&
|
||||
!mlJobLoading && (
|
||||
<>
|
||||
<ModalBodySection className="builder-section">
|
||||
{isRuleEQLSequenceStatement && (
|
||||
<>
|
||||
<EuiCallOut
|
||||
data-test-subj="eql-sequence-callout"
|
||||
title={i18n.EDIT_EXCEPTION_SEQUENCE_WARNING}
|
||||
/>
|
||||
<EuiSpacer />
|
||||
</>
|
||||
)}
|
||||
<EuiText>{i18n.EXCEPTION_BUILDER_INFO}</EuiText>
|
||||
<EuiSpacer />
|
||||
<ExceptionBuilderComponent
|
||||
exceptionListItems={[exceptionItem]}
|
||||
listType={exceptionListType}
|
||||
listId={exceptionItem.list_id}
|
||||
listNamespaceType={exceptionItem.namespace_type}
|
||||
ruleName={ruleName}
|
||||
isOrDisabled
|
||||
isAndDisabled={false}
|
||||
isNestedDisabled={false}
|
||||
data-test-subj="edit-exception-modal-builder"
|
||||
id-aria="edit-exception-modal-builder"
|
||||
onChange={handleBuilderOnChange}
|
||||
indexPatterns={indexPatterns}
|
||||
ruleType={maybeRule?.type}
|
||||
/>
|
||||
</EuiFormRow>
|
||||
{exceptionListType === 'endpoint' && (
|
||||
<>
|
||||
<EuiSpacer />
|
||||
<EuiText data-test-subj="edit-exception-endpoint-text" color="subdued" size="s">
|
||||
{i18n.ENDPOINT_QUARANTINE_TEXT}
|
||||
</EuiText>
|
||||
</>
|
||||
)}
|
||||
</ModalBodySection>
|
||||
</>
|
||||
)}
|
||||
|
||||
<EuiSpacer />
|
||||
|
||||
<AddExceptionComments
|
||||
exceptionItemComments={exceptionItem.comments}
|
||||
newCommentValue={comment}
|
||||
newCommentOnChange={onCommentChange}
|
||||
/>
|
||||
</ModalBodySection>
|
||||
<EuiHorizontalRule />
|
||||
<ModalBodySection>
|
||||
<EuiFormRow fullWidth>
|
||||
<EuiCheckbox
|
||||
data-test-subj="close-alert-on-add-edit-exception-checkbox"
|
||||
id="close-alert-on-add-edit-exception-checkbox"
|
||||
label={
|
||||
shouldDisableBulkClose
|
||||
? i18n.BULK_CLOSE_LABEL_DISABLED
|
||||
: i18n.BULK_CLOSE_LABEL
|
||||
}
|
||||
checked={shouldBulkCloseAlert}
|
||||
onChange={onBulkCloseAlertCheckboxChange}
|
||||
disabled={shouldDisableBulkClose}
|
||||
/>
|
||||
</EuiFormRow>
|
||||
{exceptionListType === 'endpoint' && (
|
||||
<>
|
||||
<EuiSpacer />
|
||||
<EuiText data-test-subj="edit-exception-endpoint-text" color="subdued" size="s">
|
||||
{i18n.ENDPOINT_QUARANTINE_TEXT}
|
||||
</EuiText>
|
||||
</>
|
||||
)}
|
||||
</ModalBodySection>
|
||||
</>
|
||||
)}
|
||||
{updateError != null && (
|
||||
<ModalBodySection>
|
||||
<ErrorCallout
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { CombinedJobWithStats } from '../../../../../../ml/common/types/anomaly_detection_jobs';
|
||||
import { HttpSetup } from '../../../../../../../../src/core/public';
|
||||
|
||||
export interface GetJobsArgs {
|
||||
http: HttpSetup;
|
||||
jobIds: string[];
|
||||
signal: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches details for a set of ML jobs
|
||||
*
|
||||
* @param http HTTP Service
|
||||
* @param jobIds Array of job IDs to filter against
|
||||
* @param signal to cancel request
|
||||
*
|
||||
* @throws An error if response is not OK
|
||||
*/
|
||||
export const getJobs = async ({
|
||||
http,
|
||||
jobIds,
|
||||
signal,
|
||||
}: GetJobsArgs): Promise<CombinedJobWithStats[]> =>
|
||||
http.fetch<CombinedJobWithStats[]>('/api/ml/jobs/jobs', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ jobIds }),
|
||||
asSystemRequest: true,
|
||||
signal,
|
||||
});
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useAsync, withOptionalSignal } from '../../../../shared_imports';
|
||||
import { getJobs } from '../api/get_jobs';
|
||||
import { CombinedJobWithStats } from '../../../../../../ml/common/types/anomaly_detection_jobs';
|
||||
|
||||
import { hasMlUserPermissions } from '../../../../../common/machine_learning/has_ml_user_permissions';
|
||||
import { hasMlLicense } from '../../../../../common/machine_learning/has_ml_license';
|
||||
import { useAppToasts } from '../../../hooks/use_app_toasts';
|
||||
import { useHttp } from '../../../lib/kibana';
|
||||
import { useMlCapabilities } from './use_ml_capabilities';
|
||||
import * as i18n from '../translations';
|
||||
|
||||
const _getJobs = withOptionalSignal(getJobs);
|
||||
|
||||
export const useGetJobs = () => useAsync(_getJobs);
|
||||
|
||||
export interface UseGetInstalledJobReturn {
|
||||
loading: boolean;
|
||||
jobs: CombinedJobWithStats[];
|
||||
isMlUser: boolean;
|
||||
isLicensed: boolean;
|
||||
}
|
||||
|
||||
export const useGetInstalledJob = (jobIds: string[]): UseGetInstalledJobReturn => {
|
||||
const [jobs, setJobs] = useState<CombinedJobWithStats[]>([]);
|
||||
const { addError } = useAppToasts();
|
||||
const mlCapabilities = useMlCapabilities();
|
||||
const http = useHttp();
|
||||
const { error, loading, result, start } = useGetJobs();
|
||||
|
||||
const isMlUser = hasMlUserPermissions(mlCapabilities);
|
||||
const isLicensed = hasMlLicense(mlCapabilities);
|
||||
|
||||
useEffect(() => {
|
||||
if (isMlUser && isLicensed && jobIds.length > 0) {
|
||||
start({ http, jobIds });
|
||||
}
|
||||
}, [http, isMlUser, isLicensed, start, jobIds]);
|
||||
|
||||
useEffect(() => {
|
||||
if (result) {
|
||||
setJobs(result);
|
||||
}
|
||||
}, [result]);
|
||||
|
||||
useEffect(() => {
|
||||
if (error) {
|
||||
addError(error, { title: i18n.SIEM_JOB_FETCH_FAILURE });
|
||||
}
|
||||
}, [addError, error]);
|
||||
|
||||
return { isLicensed, isMlUser, jobs, loading };
|
||||
};
|
||||
|
|
@ -21,7 +21,6 @@ import { TimelineId } from '../../../../../common/types/timeline';
|
|||
import { DEFAULT_INDEX_PATTERN } from '../../../../../common/constants';
|
||||
import { Status, Type } from '../../../../../common/detection_engine/schemas/common/schemas';
|
||||
import { isThresholdRule } from '../../../../../common/detection_engine/utils';
|
||||
import { isMlRule } from '../../../../../common/machine_learning/helpers';
|
||||
import { timelineActions } from '../../../../timelines/store/timeline';
|
||||
import { EventsTd, EventsTdContent } from '../../../../timelines/components/timeline/styles';
|
||||
import { DEFAULT_ICON_BUTTON_WIDTH } from '../../../../timelines/components/timeline/helpers';
|
||||
|
|
@ -75,11 +74,17 @@ const AlertContextMenuComponent: React.FC<AlertContextMenuProps> = ({
|
|||
'',
|
||||
[ecsRowData]
|
||||
);
|
||||
const ruleIndices = useMemo(
|
||||
(): string[] =>
|
||||
(ecsRowData.signal?.rule && ecsRowData.signal.rule.index) ?? DEFAULT_INDEX_PATTERN,
|
||||
[ecsRowData]
|
||||
);
|
||||
const ruleIndices = useMemo((): string[] => {
|
||||
if (
|
||||
ecsRowData.signal?.rule &&
|
||||
ecsRowData.signal.rule.index &&
|
||||
ecsRowData.signal.rule.index.length > 0
|
||||
) {
|
||||
return ecsRowData.signal.rule.index;
|
||||
} else {
|
||||
return DEFAULT_INDEX_PATTERN;
|
||||
}
|
||||
}, [ecsRowData]);
|
||||
|
||||
const { addWarning } = useAppToasts();
|
||||
|
||||
|
|
@ -321,7 +326,7 @@ const AlertContextMenuComponent: React.FC<AlertContextMenuProps> = ({
|
|||
const areExceptionsAllowed = useMemo((): boolean => {
|
||||
const ruleTypes = getOr([], 'signal.rule.type', ecsRowData);
|
||||
const [ruleType] = ruleTypes as Type[];
|
||||
return !isMlRule(ruleType) && !isThresholdRule(ruleType);
|
||||
return !isThresholdRule(ruleType);
|
||||
}, [ecsRowData]);
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import { EuiAccordion, EuiFlexItem, EuiSpacer, EuiFormRow } from '@elastic/eui';
|
|||
import React, { FC, memo, useCallback, useEffect, useState } from 'react';
|
||||
import styled from 'styled-components';
|
||||
|
||||
import { isMlRule } from '../../../../../common/machine_learning/helpers';
|
||||
import { isThresholdRule } from '../../../../../common/detection_engine/utils';
|
||||
import {
|
||||
RuleStepProps,
|
||||
|
|
@ -76,10 +75,7 @@ const StepAboutRuleComponent: FC<StepAboutRuleProps> = ({
|
|||
const [severityValue, setSeverityValue] = useState<string>(initialState.severity.value);
|
||||
const [indexPatternLoading, { indexPatterns }] = useFetchIndex(defineRuleData?.index ?? []);
|
||||
|
||||
const canUseExceptions =
|
||||
defineRuleData?.ruleType &&
|
||||
!isMlRule(defineRuleData.ruleType) &&
|
||||
!isThresholdRule(defineRuleData.ruleType);
|
||||
const canUseExceptions = defineRuleData?.ruleType && !isThresholdRule(defineRuleData.ruleType);
|
||||
|
||||
const { form } = useForm<AboutStepRule>({
|
||||
defaultValue: initialState,
|
||||
|
|
|
|||
|
|
@ -80,7 +80,6 @@ import { DEFAULT_INDEX_PATTERN } from '../../../../../../common/constants';
|
|||
import { useFullScreen } from '../../../../../common/containers/use_full_screen';
|
||||
import { Display } from '../../../../../hosts/pages/display';
|
||||
import { ExceptionListTypeEnum, ExceptionListIdentifiers } from '../../../../../shared_imports';
|
||||
import { isMlRule } from '../../../../../../common/machine_learning/helpers';
|
||||
import { isThresholdRule } from '../../../../../../common/detection_engine/utils';
|
||||
import { useRuleAsync } from '../../../../containers/detection_engine/rules/use_rule_async';
|
||||
import { showGlobalFilters } from '../../../../../timelines/components/timeline/helpers';
|
||||
|
|
@ -104,7 +103,7 @@ enum RuleDetailTabs {
|
|||
}
|
||||
|
||||
const getRuleDetailsTabs = (rule: Rule | null) => {
|
||||
const canUseExceptions = rule && !isMlRule(rule.type) && !isThresholdRule(rule.type);
|
||||
const canUseExceptions = rule && !isThresholdRule(rule.type);
|
||||
return [
|
||||
{
|
||||
id: RuleDetailTabs.alerts,
|
||||
|
|
|
|||
|
|
@ -150,8 +150,8 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig
|
|||
|
||||
export const sampleDocWithSortId = (
|
||||
someUuid: string = sampleIdGuid,
|
||||
ip?: string,
|
||||
destIp?: string
|
||||
ip?: string | string[],
|
||||
destIp?: string | string[]
|
||||
): SignalSourceHit => ({
|
||||
_index: 'myFakeSignalIndex',
|
||||
_type: 'doc',
|
||||
|
|
@ -502,8 +502,8 @@ export const repeatedSearchResultsWithSortId = (
|
|||
total: number,
|
||||
pageSize: number,
|
||||
guids: string[],
|
||||
ips?: string[],
|
||||
destIps?: string[]
|
||||
ips?: Array<string | string[]>,
|
||||
destIps?: Array<string | string[]>
|
||||
): SignalSearchResponse => ({
|
||||
took: 10,
|
||||
timed_out: false,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@
|
|||
|
||||
import { flow, omit } from 'lodash/fp';
|
||||
import set from 'set-value';
|
||||
import { SearchResponse } from 'elasticsearch';
|
||||
|
||||
import { Logger } from '../../../../../../../src/core/server';
|
||||
import { AlertServices } from '../../../../../alerts/server';
|
||||
|
|
@ -15,6 +14,7 @@ import { RuleTypeParams, RefreshTypes } from '../types';
|
|||
import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create';
|
||||
import { AnomalyResults, Anomaly } from '../../machine_learning';
|
||||
import { BuildRuleMessage } from './rule_messages';
|
||||
import { SearchResponse } from '../../types';
|
||||
|
||||
interface BulkCreateMlSignalsParams {
|
||||
actions: RuleAlertAction[];
|
||||
|
|
|
|||
|
|
@ -400,6 +400,87 @@ describe('filterEventsAgainstList', () => {
|
|||
'9.9.9.9',
|
||||
]).toEqual(ipVals);
|
||||
});
|
||||
|
||||
it('should respond with less items in the list given one exception item with two entries of type list and array of values in document', async () => {
|
||||
const exceptionItem = getExceptionListItemSchemaMock();
|
||||
exceptionItem.entries = [
|
||||
{
|
||||
field: 'source.ip',
|
||||
operator: 'included',
|
||||
type: 'list',
|
||||
list: {
|
||||
id: 'ci-badguys.txt',
|
||||
type: 'ip',
|
||||
},
|
||||
},
|
||||
{
|
||||
field: 'destination.ip',
|
||||
operator: 'included',
|
||||
type: 'list',
|
||||
list: {
|
||||
id: 'ci-badguys-again.txt',
|
||||
type: 'ip',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
// this call represents an exception list with a value list containing ['2.2.2.2']
|
||||
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
|
||||
{ ...getListItemResponseMock(), value: '2.2.2.2' },
|
||||
]);
|
||||
// this call represents an exception list with a value list containing ['4.4.4.4']
|
||||
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
|
||||
{ ...getListItemResponseMock(), value: '4.4.4.4' },
|
||||
]);
|
||||
|
||||
const res = await filterEventsAgainstList({
|
||||
logger: mockLogger,
|
||||
listClient,
|
||||
exceptionsList: [exceptionItem],
|
||||
eventSearchResult: repeatedSearchResultsWithSortId(
|
||||
3,
|
||||
3,
|
||||
someGuids.slice(0, 3),
|
||||
[
|
||||
['1.1.1.1', '1.1.1.1'],
|
||||
['1.1.1.1', '2.2.2.2'],
|
||||
['2.2.2.2', '3.3.3.3'],
|
||||
],
|
||||
[
|
||||
['1.1.1.1', '2.2.2.2'],
|
||||
['2.2.2.2', '3.3.3.3'],
|
||||
['3.3.3.3', '4.4.4.4'],
|
||||
]
|
||||
),
|
||||
buildRuleMessage,
|
||||
});
|
||||
expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
|
||||
expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([
|
||||
'1.1.1.1',
|
||||
'2.2.2.2',
|
||||
'3.3.3.3',
|
||||
]);
|
||||
expect((listClient.getListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([
|
||||
'1.1.1.1',
|
||||
'2.2.2.2',
|
||||
'3.3.3.3',
|
||||
'4.4.4.4',
|
||||
]);
|
||||
expect(res.hits.hits.length).toEqual(2);
|
||||
|
||||
// @ts-expect-error
|
||||
const sourceIpVals = res.hits.hits.map((item) => item._source.source.ip);
|
||||
expect([
|
||||
['1.1.1.1', '1.1.1.1'],
|
||||
['1.1.1.1', '2.2.2.2'],
|
||||
]).toEqual(sourceIpVals);
|
||||
// @ts-expect-error
|
||||
const destIpVals = res.hits.hits.map((item) => item._source.destination.ip);
|
||||
expect([
|
||||
['1.1.1.1', '2.2.2.2'],
|
||||
['2.2.2.2', '3.3.3.3'],
|
||||
]).toEqual(destIpVals);
|
||||
});
|
||||
});
|
||||
describe('operator type is excluded', () => {
|
||||
it('should respond with empty list if no items match value list', async () => {
|
||||
|
|
@ -463,5 +544,86 @@ describe('filterEventsAgainstList', () => {
|
|||
);
|
||||
expect(res.hits.hits.length).toEqual(2);
|
||||
});
|
||||
|
||||
it('should respond with less items in the list given one exception item with two entries of type list and array of values in document', async () => {
|
||||
const exceptionItem = getExceptionListItemSchemaMock();
|
||||
exceptionItem.entries = [
|
||||
{
|
||||
field: 'source.ip',
|
||||
operator: 'excluded',
|
||||
type: 'list',
|
||||
list: {
|
||||
id: 'ci-badguys.txt',
|
||||
type: 'ip',
|
||||
},
|
||||
},
|
||||
{
|
||||
field: 'destination.ip',
|
||||
operator: 'excluded',
|
||||
type: 'list',
|
||||
list: {
|
||||
id: 'ci-badguys-again.txt',
|
||||
type: 'ip',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
// this call represents an exception list with a value list containing ['2.2.2.2']
|
||||
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
|
||||
{ ...getListItemResponseMock(), value: '2.2.2.2' },
|
||||
]);
|
||||
// this call represents an exception list with a value list containing ['4.4.4.4']
|
||||
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
|
||||
{ ...getListItemResponseMock(), value: '4.4.4.4' },
|
||||
]);
|
||||
|
||||
const res = await filterEventsAgainstList({
|
||||
logger: mockLogger,
|
||||
listClient,
|
||||
exceptionsList: [exceptionItem],
|
||||
eventSearchResult: repeatedSearchResultsWithSortId(
|
||||
3,
|
||||
3,
|
||||
someGuids.slice(0, 3),
|
||||
[
|
||||
['1.1.1.1', '1.1.1.1'],
|
||||
['1.1.1.1', '2.2.2.2'],
|
||||
['2.2.2.2', '3.3.3.3'],
|
||||
],
|
||||
[
|
||||
['1.1.1.1', '2.2.2.2'],
|
||||
['2.2.2.2', '3.3.3.3'],
|
||||
['3.3.3.3', '4.4.4.4'],
|
||||
]
|
||||
),
|
||||
buildRuleMessage,
|
||||
});
|
||||
expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
|
||||
expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([
|
||||
'1.1.1.1',
|
||||
'2.2.2.2',
|
||||
'3.3.3.3',
|
||||
]);
|
||||
expect((listClient.getListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([
|
||||
'1.1.1.1',
|
||||
'2.2.2.2',
|
||||
'3.3.3.3',
|
||||
'4.4.4.4',
|
||||
]);
|
||||
expect(res.hits.hits.length).toEqual(2);
|
||||
|
||||
// @ts-expect-error
|
||||
const sourceIpVals = res.hits.hits.map((item) => item._source.source.ip);
|
||||
expect([
|
||||
['1.1.1.1', '2.2.2.2'],
|
||||
['2.2.2.2', '3.3.3.3'],
|
||||
]).toEqual(sourceIpVals);
|
||||
// @ts-expect-error
|
||||
const destIpVals = res.hits.hits.map((item) => item._source.destination.ip);
|
||||
expect([
|
||||
['2.2.2.2', '3.3.3.3'],
|
||||
['3.3.3.3', '4.4.4.4'],
|
||||
]).toEqual(destIpVals);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import { get } from 'lodash/fp';
|
|||
import { Logger } from 'src/core/server';
|
||||
|
||||
import { ListClient } from '../../../../../lists/server';
|
||||
import { SignalSearchResponse } from './types';
|
||||
import { BuildRuleMessage } from './rule_messages';
|
||||
import {
|
||||
EntryList,
|
||||
|
|
@ -17,16 +16,23 @@ import {
|
|||
} from '../../../../../lists/common/schemas';
|
||||
import { hasLargeValueList } from '../../../../common/detection_engine/utils';
|
||||
import { SearchTypes } from '../../../../common/detection_engine/types';
|
||||
import { SearchResponse } from '../../types';
|
||||
|
||||
interface FilterEventsAgainstList {
|
||||
listClient: ListClient;
|
||||
exceptionsList: ExceptionListItemSchema[];
|
||||
logger: Logger;
|
||||
eventSearchResult: SignalSearchResponse;
|
||||
buildRuleMessage: BuildRuleMessage;
|
||||
}
|
||||
// narrow unioned type to be single
|
||||
const isStringableType = (val: SearchTypes): val is string | number | boolean =>
|
||||
['string', 'number', 'boolean'].includes(typeof val);
|
||||
|
||||
export const createSetToFilterAgainst = async ({
|
||||
const isStringableArray = (val: SearchTypes): val is Array<string | number | boolean> => {
|
||||
if (!Array.isArray(val)) {
|
||||
return false;
|
||||
}
|
||||
// TS does not allow .every to be called on val as-is, even though every type in the union
|
||||
// is an array. https://github.com/microsoft/TypeScript/issues/36390
|
||||
// @ts-expect-error
|
||||
return val.every((subVal) => isStringableType(subVal));
|
||||
};
|
||||
|
||||
export const createSetToFilterAgainst = async <T>({
|
||||
events,
|
||||
field,
|
||||
listId,
|
||||
|
|
@ -35,7 +41,7 @@ export const createSetToFilterAgainst = async ({
|
|||
logger,
|
||||
buildRuleMessage,
|
||||
}: {
|
||||
events: SignalSearchResponse['hits']['hits'];
|
||||
events: SearchResponse<T>['hits']['hits'];
|
||||
field: string;
|
||||
listId: string;
|
||||
listType: Type;
|
||||
|
|
@ -43,13 +49,14 @@ export const createSetToFilterAgainst = async ({
|
|||
logger: Logger;
|
||||
buildRuleMessage: BuildRuleMessage;
|
||||
}): Promise<Set<SearchTypes>> => {
|
||||
// narrow unioned type to be single
|
||||
const isStringableType = (val: SearchTypes) =>
|
||||
['string', 'number', 'boolean'].includes(typeof val);
|
||||
const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => {
|
||||
const valueField = get(field, searchResultItem._source);
|
||||
if (valueField != null && isStringableType(valueField)) {
|
||||
acc.add(valueField.toString());
|
||||
if (valueField != null) {
|
||||
if (isStringableType(valueField)) {
|
||||
acc.add(valueField.toString());
|
||||
} else if (isStringableArray(valueField)) {
|
||||
valueField.forEach((subVal) => acc.add(subVal.toString()));
|
||||
}
|
||||
}
|
||||
return acc;
|
||||
}, new Set<string>());
|
||||
|
|
@ -71,13 +78,19 @@ export const createSetToFilterAgainst = async ({
|
|||
return matchedListItemsSet;
|
||||
};
|
||||
|
||||
export const filterEventsAgainstList = async ({
|
||||
export const filterEventsAgainstList = async <T>({
|
||||
listClient,
|
||||
exceptionsList,
|
||||
logger,
|
||||
eventSearchResult,
|
||||
buildRuleMessage,
|
||||
}: FilterEventsAgainstList): Promise<SignalSearchResponse> => {
|
||||
}: {
|
||||
listClient: ListClient;
|
||||
exceptionsList: ExceptionListItemSchema[];
|
||||
logger: Logger;
|
||||
eventSearchResult: SearchResponse<T>;
|
||||
buildRuleMessage: BuildRuleMessage;
|
||||
}): Promise<SearchResponse<T>> => {
|
||||
try {
|
||||
if (exceptionsList == null || exceptionsList.length === 0) {
|
||||
logger.debug(buildRuleMessage('about to return original search result'));
|
||||
|
|
@ -108,9 +121,9 @@ export const filterEventsAgainstList = async ({
|
|||
});
|
||||
|
||||
// now that we have all the exception items which are value lists (whether single entry or have multiple entries)
|
||||
const res = await valueListExceptionItems.reduce<Promise<SignalSearchResponse['hits']['hits']>>(
|
||||
const res = await valueListExceptionItems.reduce<Promise<SearchResponse<T>['hits']['hits']>>(
|
||||
async (
|
||||
filteredAccum: Promise<SignalSearchResponse['hits']['hits']>,
|
||||
filteredAccum: Promise<SearchResponse<T>['hits']['hits']>,
|
||||
exceptionItem: ExceptionListItemSchema
|
||||
) => {
|
||||
// 1. acquire the values from the specified fields to check
|
||||
|
|
@ -152,15 +165,23 @@ export const filterEventsAgainstList = async ({
|
|||
const vals = fieldAndSetTuples.map((tuple) => {
|
||||
const eventItem = get(tuple.field, item._source);
|
||||
if (tuple.operator === 'included') {
|
||||
// only create a signal if the event is not in the value list
|
||||
// only create a signal if the field value is not in the value list
|
||||
if (eventItem != null) {
|
||||
return !tuple.matchedSet.has(eventItem);
|
||||
if (isStringableType(eventItem)) {
|
||||
return !tuple.matchedSet.has(eventItem);
|
||||
} else if (isStringableArray(eventItem)) {
|
||||
return !eventItem.some((val) => tuple.matchedSet.has(val));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} else if (tuple.operator === 'excluded') {
|
||||
// only create a signal if the event is in the value list
|
||||
// only create a signal if the field value is in the value list
|
||||
if (eventItem != null) {
|
||||
return tuple.matchedSet.has(eventItem);
|
||||
if (isStringableType(eventItem)) {
|
||||
return tuple.matchedSet.has(eventItem);
|
||||
} else if (isStringableArray(eventItem)) {
|
||||
return eventItem.some((val) => tuple.matchedSet.has(val));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
@ -175,10 +196,10 @@ export const filterEventsAgainstList = async ({
|
|||
const toReturn = filteredEvents;
|
||||
return toReturn;
|
||||
},
|
||||
Promise.resolve<SignalSearchResponse['hits']['hits']>(eventSearchResult.hits.hits)
|
||||
Promise.resolve<SearchResponse<T>['hits']['hits']>(eventSearchResult.hits.hits)
|
||||
);
|
||||
|
||||
const toReturn: SignalSearchResponse = {
|
||||
const toReturn: SearchResponse<T> = {
|
||||
took: eventSearchResult.took,
|
||||
timed_out: eventSearchResult.timed_out,
|
||||
_shards: eventSearchResult._shards,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
|
||||
import dateMath from '@elastic/datemath';
|
||||
import { ExceptionListItemSchema } from '../../../../../lists/common';
|
||||
|
||||
import { KibanaRequest, SavedObjectsClientContract } from '../../../../../../../src/core/server';
|
||||
import { MlPluginSetup } from '../../../../../ml/server';
|
||||
|
|
@ -18,6 +19,7 @@ export const findMlSignals = async ({
|
|||
anomalyThreshold,
|
||||
from,
|
||||
to,
|
||||
exceptionItems,
|
||||
}: {
|
||||
ml: MlPluginSetup;
|
||||
request: KibanaRequest;
|
||||
|
|
@ -26,6 +28,7 @@ export const findMlSignals = async ({
|
|||
anomalyThreshold: number;
|
||||
from: string;
|
||||
to: string;
|
||||
exceptionItems: ExceptionListItemSchema[];
|
||||
}): Promise<AnomalyResults> => {
|
||||
const { mlAnomalySearch } = ml.mlSystemProvider(request, savedObjectsClient);
|
||||
const params = {
|
||||
|
|
@ -33,6 +36,7 @@ export const findMlSignals = async ({
|
|||
threshold: anomalyThreshold,
|
||||
earliestMs: dateMath.parse(from)?.valueOf() ?? 0,
|
||||
latestMs: dateMath.parse(to)?.valueOf() ?? 0,
|
||||
exceptionItems,
|
||||
};
|
||||
return getAnomalies(params, mlAnomalySearch);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk
|
|||
import { createThreatSignals } from './threat_mapping/create_threat_signals';
|
||||
import { getIndexVersion } from '../routes/index/get_index_version';
|
||||
import { MIN_EQL_RULE_INDEX_VERSION } from '../routes/index/get_signals_template';
|
||||
import { filterEventsAgainstList } from './filter_events_with_list';
|
||||
|
||||
export const signalRulesAlertType = ({
|
||||
logger,
|
||||
|
|
@ -242,9 +243,18 @@ export const signalRulesAlertType = ({
|
|||
anomalyThreshold,
|
||||
from,
|
||||
to,
|
||||
exceptionItems: exceptionItems ?? [],
|
||||
});
|
||||
|
||||
const anomalyCount = anomalyResults.hits.hits.length;
|
||||
const filteredAnomalyResults = await filterEventsAgainstList({
|
||||
listClient,
|
||||
exceptionsList: exceptionItems ?? [],
|
||||
logger,
|
||||
eventSearchResult: anomalyResults,
|
||||
buildRuleMessage,
|
||||
});
|
||||
|
||||
const anomalyCount = filteredAnomalyResults.hits.hits.length;
|
||||
if (anomalyCount) {
|
||||
logger.info(buildRuleMessage(`Found ${anomalyCount} signals from ML anomalies.`));
|
||||
}
|
||||
|
|
@ -257,7 +267,7 @@ export const signalRulesAlertType = ({
|
|||
} = await bulkCreateMlSignals({
|
||||
actions,
|
||||
throttle,
|
||||
someResult: anomalyResults,
|
||||
someResult: filteredAnomalyResults,
|
||||
ruleParams: params,
|
||||
services,
|
||||
logger,
|
||||
|
|
@ -276,15 +286,16 @@ export const signalRulesAlertType = ({
|
|||
});
|
||||
// The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] }
|
||||
const shardFailures =
|
||||
(anomalyResults._shards as typeof anomalyResults._shards & { failures: [] }).failures ??
|
||||
[];
|
||||
(filteredAnomalyResults._shards as typeof filteredAnomalyResults._shards & {
|
||||
failures: [];
|
||||
}).failures ?? [];
|
||||
const searchErrors = createErrorsFromShard({
|
||||
errors: shardFailures,
|
||||
});
|
||||
result = mergeReturns([
|
||||
result,
|
||||
createSearchAfterReturnType({
|
||||
success: success && anomalyResults._shards.failed === 0,
|
||||
success: success && filteredAnomalyResults._shards.failed === 0,
|
||||
errors: [...errors, ...searchErrors],
|
||||
createdSignalsCount: createdItemsCount,
|
||||
bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [],
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { getExceptionListItemSchemaMock } from '../../../../lists/common/schemas/response/exception_list_item_schema.mock';
|
||||
import { getAnomalies, AnomaliesSearchParams } from '.';
|
||||
|
||||
const getFiltersFromMock = (mock: jest.Mock) => {
|
||||
|
|
@ -23,6 +24,7 @@ describe('getAnomalies', () => {
|
|||
threshold: 5,
|
||||
earliestMs: 1588517231429,
|
||||
latestMs: 1588617231429,
|
||||
exceptionItems: [getExceptionListItemSchemaMock(), getExceptionListItemSchemaMock()],
|
||||
};
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { SearchResponse } from 'elasticsearch';
|
||||
import { RequestParams } from '@elastic/elasticsearch';
|
||||
|
||||
import { ExceptionListItemSchema } from '../../../../lists/common';
|
||||
import { buildExceptionFilter } from '../../../common/detection_engine/get_query_filter';
|
||||
import { AnomalyRecordDoc as Anomaly } from '../../../../ml/server';
|
||||
import { SearchResponse } from '../types';
|
||||
|
||||
export { Anomaly };
|
||||
export type AnomalyResults = SearchResponse<Anomaly>;
|
||||
|
|
@ -21,6 +23,7 @@ export interface AnomaliesSearchParams {
|
|||
threshold: number;
|
||||
earliestMs: number;
|
||||
latestMs: number;
|
||||
exceptionItems: ExceptionListItemSchema[];
|
||||
maxRecords?: number;
|
||||
}
|
||||
|
||||
|
|
@ -49,6 +52,17 @@ export const getAnomalies = async (
|
|||
},
|
||||
},
|
||||
],
|
||||
must_not: buildExceptionFilter({
|
||||
lists: params.exceptionItems,
|
||||
config: {
|
||||
allowLeadingWildcards: true,
|
||||
queryStringOptions: { analyze_wildcard: true },
|
||||
ignoreFilterIfFieldNotInIndex: false,
|
||||
dateFormatTZ: 'Zulu',
|
||||
},
|
||||
excludeExceptions: true,
|
||||
chunkSize: 1024,
|
||||
})?.query,
|
||||
},
|
||||
},
|
||||
sort: [{ record_score: { order: 'desc' } }],
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue