mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
[8.x] [Security Solution][Detection Engine] adds preview logged requests for new terms, threshold, query, ML rule types (#203320) (#208581)
# Backport This will backport the following commits from `main` to `8.x`: - [[Security Solution][Detection Engine] adds preview logged requests for new terms, threshold, query, ML rule types (#203320)](https://github.com/elastic/kibana/pull/203320) <!--- Backport version: 9.4.3 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Vitalii Dmyterko","email":"92328789+vitaliidm@users.noreply.github.com"},"sourceCommit":{"committedDate":"2025-01-28T16:50:00Z","message":"[Security Solution][Detection Engine] adds preview logged requests for new terms, threshold, query, ML rule types (#203320)\n\n## Summary\r\n\r\n- partially addresses https://github.com/elastic/kibana/issues/202545\r\n(except of IM rule type)\r\n- extends logged requests preview for:\r\n - [x] New terms\r\n - [x] Query\r\n - [x] ML\r\n - [x] Threshold\r\n- For Threshold, Query, New terms rule type introduced Page view, where\r\neach loop of rule execution is presented as a separate page\r\n- Only first 2 search queries requests of each type are logged for\r\nperformance reasons(rule can have very a large and multiple requests).\r\nThat's why property **request** was made not mandatory in\r\n`rule_preview.schema.yaml`\r\n\r\n\r\n### DEMO\r\n\r\n\r\n\r\nhttps://github.com/user-attachments/assets/abfbd3ff-d06c-4892-b805-0f05084042ed\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"0f996c36151d2128f56246cb69d9315c8ea6d085","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["v9.0.0","Team:Detections and Resp","Team: SecuritySolution","release_note:feature","Team:Detection Engine","backport:version","v8.18.0"],"title":"[Security Solution][Detection Engine] adds preview logged requests for new terms, threshold, query, ML rule types","number":203320,"url":"https://github.com/elastic/kibana/pull/203320","mergeCommit":{"message":"[Security Solution][Detection Engine] adds preview logged requests for new terms, threshold, query, ML rule types (#203320)\n\n## Summary\r\n\r\n- partially addresses https://github.com/elastic/kibana/issues/202545\r\n(except of IM rule type)\r\n- extends logged requests preview for:\r\n - [x] New terms\r\n - [x] Query\r\n - [x] ML\r\n - [x] Threshold\r\n- For Threshold, Query, New terms rule type introduced Page view, where\r\neach loop of rule execution is presented as a separate page\r\n- Only first 2 search queries requests of each type are logged for\r\nperformance reasons(rule can have very a large and multiple requests).\r\nThat's why property **request** was made not mandatory in\r\n`rule_preview.schema.yaml`\r\n\r\n\r\n### DEMO\r\n\r\n\r\n\r\nhttps://github.com/user-attachments/assets/abfbd3ff-d06c-4892-b805-0f05084042ed\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"0f996c36151d2128f56246cb69d9315c8ea6d085"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/203320","number":203320,"mergeCommit":{"message":"[Security Solution][Detection Engine] adds preview logged requests for new terms, threshold, query, ML rule types (#203320)\n\n## Summary\r\n\r\n- partially addresses https://github.com/elastic/kibana/issues/202545\r\n(except of IM rule type)\r\n- extends logged requests preview for:\r\n - [x] New terms\r\n - [x] Query\r\n - [x] ML\r\n - [x] Threshold\r\n- For Threshold, Query, New terms rule type introduced Page view, where\r\neach loop of rule execution is presented as a separate page\r\n- Only first 2 search queries requests of each type are logged for\r\nperformance reasons(rule can have very a large and multiple requests).\r\nThat's why property **request** was made not mandatory in\r\n`rule_preview.schema.yaml`\r\n\r\n\r\n### DEMO\r\n\r\n\r\n\r\nhttps://github.com/user-attachments/assets/abfbd3ff-d06c-4892-b805-0f05084042ed\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"0f996c36151d2128f56246cb69d9315c8ea6d085"}},{"branch":"8.x","label":"v8.18.0","branchLabelMappingKey":"^v8.18.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT--> Co-authored-by: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com>
This commit is contained in:
Kibana Machine
GitHub
parent
5779a170d5
commit
8d79524ba6
47 changed files with 1423 additions and 177 deletions
|
|
@ -43693,8 +43693,8 @@ components:
|
|||
type: integer
|
||||
request:
|
||||
$ref: '#/components/schemas/Security_Detections_API_NonEmptyString'
|
||||
required:
|
||||
- request
|
||||
request_type:
|
||||
$ref: '#/components/schemas/Security_Detections_API_NonEmptyString'
|
||||
Security_Detections_API_RulePreviewLogs:
|
||||
type: object
|
||||
properties:
|
||||
|
|
|
|||
|
|
@ -32071,8 +32071,8 @@ components:
|
|||
type: integer
|
||||
request:
|
||||
$ref: '#/components/schemas/Security_Detections_API_NonEmptyString'
|
||||
required:
|
||||
- request
|
||||
request_type:
|
||||
$ref: '#/components/schemas/Security_Detections_API_NonEmptyString'
|
||||
Security_Detections_API_RulePreviewLogs:
|
||||
type: object
|
||||
properties:
|
||||
|
|
|
|||
|
|
@ -37,9 +37,10 @@ export const RulePreviewParams = z.object({
|
|||
|
||||
export type RulePreviewLoggedRequest = z.infer<typeof RulePreviewLoggedRequest>;
|
||||
export const RulePreviewLoggedRequest = z.object({
|
||||
request: NonEmptyString,
|
||||
request: NonEmptyString.optional(),
|
||||
description: NonEmptyString.optional(),
|
||||
duration: z.number().int().optional(),
|
||||
request_type: NonEmptyString.optional(),
|
||||
});
|
||||
|
||||
export type RulePreviewLogs = z.infer<typeof RulePreviewLogs>;
|
||||
|
|
|
|||
|
|
@ -110,8 +110,8 @@ components:
|
|||
$ref: '../../model/primitives.schema.yaml#/components/schemas/NonEmptyString'
|
||||
duration:
|
||||
type: integer
|
||||
required:
|
||||
- request
|
||||
request_type:
|
||||
$ref: '../../model/primitives.schema.yaml#/components/schemas/NonEmptyString'
|
||||
|
||||
RulePreviewLogs:
|
||||
type: object
|
||||
|
|
|
|||
|
|
@ -5269,8 +5269,8 @@ components:
|
|||
type: integer
|
||||
request:
|
||||
$ref: '#/components/schemas/NonEmptyString'
|
||||
required:
|
||||
- request
|
||||
request_type:
|
||||
$ref: '#/components/schemas/NonEmptyString'
|
||||
RulePreviewLogs:
|
||||
type: object
|
||||
properties:
|
||||
|
|
|
|||
|
|
@ -4418,8 +4418,8 @@ components:
|
|||
type: integer
|
||||
request:
|
||||
$ref: '#/components/schemas/NonEmptyString'
|
||||
required:
|
||||
- request
|
||||
request_type:
|
||||
$ref: '#/components/schemas/NonEmptyString'
|
||||
RulePreviewLogs:
|
||||
type: object
|
||||
properties:
|
||||
|
|
|
|||
|
|
@ -91,3 +91,30 @@ export const previewLogs: RulePreviewLogs[] = [
|
|||
],
|
||||
},
|
||||
];
|
||||
|
||||
export const queryRuleTypePreviewLogs: RulePreviewLogs[] = [
|
||||
{
|
||||
errors: [],
|
||||
warnings: [
|
||||
'This rule reached the maximum alert limit for the rule execution. Some alerts were not created.',
|
||||
],
|
||||
startedAt: '2025-01-21T16:48:50.891Z',
|
||||
duration: 1103,
|
||||
requests: [
|
||||
{
|
||||
request:
|
||||
'POST /apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*,very-unique/_search?allow_no_indices=true&ignore_unavailable=true\n{\n "size": 100,\n "query": {\n "bool": {\n "filter": [\n {\n "bool": {\n "must": [],\n "filter": [\n {\n "query_string": {\n "query": "*"\n }\n }\n ],\n "should": [],\n "must_not": []\n }\n },\n {\n "range": {\n "@timestamp": {\n "lte": "2025-01-21T16:48:50.891Z",\n "gte": "2025-01-21T15:17:50.891Z",\n "format": "strict_date_optional_time"\n }\n }\n }\n ]\n }\n },\n "fields": [\n {\n "field": "*",\n "include_unmapped": true\n },\n {\n "field": "@timestamp",\n "format": "strict_date_optional_time"\n }\n ],\n "runtime_mappings": {},\n "sort": [\n {\n "@timestamp": {\n "order": "asc",\n "unmapped_type": "date"\n }\n }\n ]\n}',
|
||||
description: 'Find documents',
|
||||
request_type: 'findDocuments',
|
||||
duration: 137,
|
||||
},
|
||||
{
|
||||
request:
|
||||
'POST /apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*,very-unique/_search?allow_no_indices=true&ignore_unavailable=true\n{\n "size": 100,\n "query": {\n "bool": {\n "filter": [\n {\n "bool": {\n "must": [],\n "filter": [\n {\n "query_string": {\n "query": "*"\n }\n }\n ],\n "should": [],\n "must_not": []\n }\n },\n {\n "range": {\n "@timestamp": {\n "lte": "2025-01-21T16:48:50.891Z",\n "gte": "2025-01-21T15:17:50.891Z",\n "format": "strict_date_optional_time"\n }\n }\n }\n ]\n }\n },\n "fields": [\n {\n "field": "*",\n "include_unmapped": true\n },\n {\n "field": "@timestamp",\n "format": "strict_date_optional_time"\n }\n ],\n "runtime_mappings": {},\n "sort": [\n {\n "@timestamp": {\n "order": "asc",\n "unmapped_type": "date"\n }\n }\n ],\n "search_after": [\n 1737472675562\n ]\n}',
|
||||
description: 'Find documents after cursor [1737472675562]',
|
||||
request_type: 'findDocuments',
|
||||
duration: 192,
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
|
|
|||
|
|
@ -40,16 +40,18 @@ jest.mock('../../../../common/hooks/use_experimental_features', () => ({
|
|||
}));
|
||||
|
||||
// rule types that do not support logged requests
|
||||
const doNotSupportLoggedRequests: Type[] = [
|
||||
const doNotSupportLoggedRequests: Type[] = ['threat_match'];
|
||||
|
||||
const supportLoggedRequests: Type[] = [
|
||||
'esql',
|
||||
'eql',
|
||||
'threshold',
|
||||
'threat_match',
|
||||
'machine_learning',
|
||||
'query',
|
||||
'saved_query',
|
||||
'new_terms',
|
||||
];
|
||||
|
||||
const supportLoggedRequests: Type[] = ['esql', 'eql'];
|
||||
|
||||
const getMockIndexPattern = (): DataViewBase => ({
|
||||
fields,
|
||||
id: '1234',
|
||||
|
|
|
|||
|
|
@ -43,7 +43,15 @@ import { usePreviewInvocationCount } from './use_preview_invocation_count';
|
|||
|
||||
export const REASONABLE_INVOCATION_COUNT = 200;
|
||||
|
||||
const RULE_TYPES_SUPPORTING_LOGGED_REQUESTS: Type[] = ['esql', 'eql'];
|
||||
const RULE_TYPES_SUPPORTING_LOGGED_REQUESTS: Type[] = [
|
||||
'esql',
|
||||
'eql',
|
||||
'threshold',
|
||||
'machine_learning',
|
||||
'query',
|
||||
'saved_query',
|
||||
'new_terms',
|
||||
];
|
||||
|
||||
const timeRanges = [
|
||||
{ start: 'now/d', end: 'now', label: 'Today' },
|
||||
|
|
@ -316,6 +324,7 @@ const RulePreviewComponent: React.FC<RulePreviewProps> = ({
|
|||
hasNoiseWarning={hasNoiseWarning}
|
||||
isAborted={isAborted}
|
||||
showElasticsearchRequests={showElasticsearchRequests && isLoggedRequestsSupported}
|
||||
ruleType={ruleType}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -12,17 +12,17 @@ import userEvent from '@testing-library/user-event';
|
|||
import { TestProviders } from '../../../../common/mock/test_providers';
|
||||
import { LoggedRequests } from './logged_requests';
|
||||
|
||||
import { previewLogs } from './__mocks__/preview_logs';
|
||||
import { previewLogs, queryRuleTypePreviewLogs } from './__mocks__/preview_logs';
|
||||
|
||||
describe('LoggedRequests', () => {
|
||||
it('should not render component if logs are empty', () => {
|
||||
render(<LoggedRequests logs={[]} />, { wrapper: TestProviders });
|
||||
render(<LoggedRequests logs={[]} ruleType="esql" />, { wrapper: TestProviders });
|
||||
|
||||
expect(screen.queryByTestId('preview-logged-requests-accordion')).toBeNull();
|
||||
});
|
||||
|
||||
it('should open accordion on click and render list of request items', async () => {
|
||||
render(<LoggedRequests logs={previewLogs} />, { wrapper: TestProviders });
|
||||
render(<LoggedRequests logs={previewLogs} ruleType="esql" />, { wrapper: TestProviders });
|
||||
|
||||
expect(screen.queryByTestId('preview-logged-requests-accordion')).toBeInTheDocument();
|
||||
|
||||
|
|
@ -32,7 +32,7 @@ describe('LoggedRequests', () => {
|
|||
});
|
||||
|
||||
it('should render code content on logged request item accordion click', async () => {
|
||||
render(<LoggedRequests logs={previewLogs} />, { wrapper: TestProviders });
|
||||
render(<LoggedRequests logs={previewLogs} ruleType="esql" />, { wrapper: TestProviders });
|
||||
|
||||
expect(screen.queryByTestId('preview-logged-requests-accordion')).toBeInTheDocument();
|
||||
|
||||
|
|
@ -65,4 +65,32 @@ describe('LoggedRequests', () => {
|
|||
/POST \/packetbeat-8\.14\.2\/_search\?ignore_unavailable=true/
|
||||
);
|
||||
});
|
||||
|
||||
it('should render code content when rule supports page view', async () => {
|
||||
render(<LoggedRequests logs={queryRuleTypePreviewLogs} ruleType="query" />, {
|
||||
wrapper: TestProviders,
|
||||
});
|
||||
|
||||
expect(screen.queryByTestId('preview-logged-requests-accordion')).toBeInTheDocument();
|
||||
|
||||
await userEvent.click(screen.getByText('Preview logged requests'));
|
||||
|
||||
const loggedRequestsItem = screen.getAllByTestId('preview-logged-requests-item-accordion')[0];
|
||||
|
||||
expect(loggedRequestsItem).toHaveTextContent('Rule execution started at');
|
||||
expect(loggedRequestsItem).toHaveTextContent('[1103ms]');
|
||||
|
||||
await userEvent.click(loggedRequestsItem.querySelector('button') as HTMLElement);
|
||||
|
||||
expect(screen.getAllByTestId('preview-logged-requests-page-accordion')).toHaveLength(2);
|
||||
|
||||
await userEvent.click(screen.getByText('Page 1 of search queries'));
|
||||
|
||||
expect(screen.getAllByTestId('preview-logged-request-description')[0]).toHaveTextContent(
|
||||
'Find documents [137ms]'
|
||||
);
|
||||
expect(screen.getAllByTestId('preview-logged-request-code-block')[0]).toHaveTextContent(
|
||||
'POST /apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*,very-unique/_search?allow_no_indices=true&ignore_unavailable=true'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@
|
|||
import type { FC } from 'react';
|
||||
import React, { useMemo } from 'react';
|
||||
import { EuiSpacer } from '@elastic/eui';
|
||||
import { css } from '@emotion/css';
|
||||
import type { Type } from '@kbn/securitysolution-io-ts-alerting-types';
|
||||
import { css } from '@emotion/react';
|
||||
|
||||
import type { RulePreviewLogs } from '../../../../../common/api/detection_engine';
|
||||
import * as i18n from './translations';
|
||||
|
|
@ -16,7 +17,10 @@ import { OptimizedAccordion } from './optimized_accordion';
|
|||
import { LoggedRequestsItem } from './logged_requests_item';
|
||||
import { useAccordionStyling } from './use_accordion_styling';
|
||||
|
||||
const LoggedRequestsComponent: FC<{ logs: RulePreviewLogs[] }> = ({ logs }) => {
|
||||
const LoggedRequestsComponent: FC<{ logs: RulePreviewLogs[]; ruleType: Type }> = ({
|
||||
logs,
|
||||
ruleType,
|
||||
}) => {
|
||||
const cssStyles = useAccordionStyling();
|
||||
|
||||
const AccordionContent = useMemo(
|
||||
|
|
@ -25,12 +29,12 @@ const LoggedRequestsComponent: FC<{ logs: RulePreviewLogs[] }> = ({ logs }) => {
|
|||
<EuiSpacer size="m" />
|
||||
{logs.map((log) => (
|
||||
<React.Fragment key={log.startedAt}>
|
||||
<LoggedRequestsItem {...log} />
|
||||
<LoggedRequestsItem {...log} ruleType={ruleType} />
|
||||
</React.Fragment>
|
||||
))}
|
||||
</>
|
||||
),
|
||||
[logs]
|
||||
[logs, ruleType]
|
||||
);
|
||||
|
||||
if (logs.length === 0) {
|
||||
|
|
|
|||
|
|
@ -7,24 +7,27 @@
|
|||
|
||||
import type { FC, PropsWithChildren } from 'react';
|
||||
import React from 'react';
|
||||
import { css } from '@emotion/css';
|
||||
import { css } from '@emotion/react';
|
||||
|
||||
import { EuiSpacer, EuiCodeBlock, useEuiPaddingSize, EuiFlexItem } from '@elastic/eui';
|
||||
import { useEuiPaddingSize, EuiText, EuiSpacer } from '@elastic/eui';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import type { Type } from '@kbn/securitysolution-io-ts-alerting-types';
|
||||
import type { RulePreviewLogs } from '../../../../../common/api/detection_engine';
|
||||
import * as i18n from './translations';
|
||||
import { PreferenceFormattedDate } from '../../../../common/components/formatted_date';
|
||||
import { OptimizedAccordion } from './optimized_accordion';
|
||||
import { LoggedRequestsQuery } from './logged_requests_query';
|
||||
import { useAccordionStyling } from './use_accordion_styling';
|
||||
import { LoggedRequestsPages, isPageViewSupported } from './logged_requests_pages';
|
||||
|
||||
const LoggedRequestsItemComponent: FC<PropsWithChildren<RulePreviewLogs>> = ({
|
||||
const LoggedRequestsItemComponent: FC<PropsWithChildren<RulePreviewLogs & { ruleType: Type }>> = ({
|
||||
startedAt,
|
||||
duration,
|
||||
requests,
|
||||
requests = [],
|
||||
ruleType,
|
||||
}) => {
|
||||
const paddingLarge = useEuiPaddingSize('l');
|
||||
const cssStyles = useAccordionStyling();
|
||||
|
||||
return (
|
||||
<OptimizedAccordion
|
||||
data-test-subj="preview-logged-requests-item-accordion"
|
||||
|
|
@ -48,29 +51,26 @@ const LoggedRequestsItemComponent: FC<PropsWithChildren<RulePreviewLogs>> = ({
|
|||
${cssStyles}
|
||||
`}
|
||||
>
|
||||
{(requests ?? []).map((request, key) => (
|
||||
<EuiFlexItem
|
||||
key={key}
|
||||
css={css`
|
||||
padding-left: ${paddingLarge};
|
||||
`}
|
||||
>
|
||||
<EuiSpacer size="l" />
|
||||
<span data-test-subj="preview-logged-request-description">
|
||||
{request?.description ?? null} {request?.duration ? `[${request.duration}ms]` : null}
|
||||
</span>
|
||||
{requests.length > 2 ? (
|
||||
<>
|
||||
<EuiSpacer size="s" />
|
||||
<EuiCodeBlock
|
||||
language="json"
|
||||
isCopyable
|
||||
overflowHeight={300}
|
||||
isVirtualized
|
||||
data-test-subj="preview-logged-request-code-block"
|
||||
<EuiText
|
||||
color="warning"
|
||||
size="s"
|
||||
css={css`
|
||||
margin-left: ${paddingLarge};
|
||||
`}
|
||||
>
|
||||
{request.request}
|
||||
</EuiCodeBlock>
|
||||
</EuiFlexItem>
|
||||
))}
|
||||
{i18n.REQUESTS_SAMPLE_WARNING}
|
||||
</EuiText>
|
||||
<EuiSpacer size="s" />
|
||||
</>
|
||||
) : null}
|
||||
{isPageViewSupported(ruleType) ? (
|
||||
<LoggedRequestsPages requests={requests} ruleType={ruleType} />
|
||||
) : (
|
||||
requests.map((request, key) => <LoggedRequestsQuery key={key} {...request} />)
|
||||
)}
|
||||
</OptimizedAccordion>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,128 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { render, screen } from '@testing-library/react';
|
||||
|
||||
import { TestProviders } from '../../../../common/mock/test_providers';
|
||||
import { LoggedRequestsPages } from './logged_requests_pages';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
|
||||
const customQueryRuleTypeRequests = [
|
||||
{
|
||||
request_type: 'findDocuments',
|
||||
description: 'request #1',
|
||||
request: 'POST test/_search',
|
||||
duration: 10,
|
||||
},
|
||||
{
|
||||
request_type: 'findDocuments',
|
||||
description: 'request #2',
|
||||
request: 'POST test/_search',
|
||||
duration: 10,
|
||||
},
|
||||
{
|
||||
request_type: 'findDocuments',
|
||||
description: 'request #3',
|
||||
},
|
||||
];
|
||||
|
||||
describe('LoggedRequestsPages', () => {
|
||||
it('should render 3 pages for query rule', () => {
|
||||
render(<LoggedRequestsPages requests={customQueryRuleTypeRequests} ruleType="query" />, {
|
||||
wrapper: TestProviders,
|
||||
});
|
||||
|
||||
const pages = screen.getAllByTestId('preview-logged-requests-page-accordion');
|
||||
expect(pages).toHaveLength(3);
|
||||
expect(pages[0]).toHaveTextContent('Page 1 of search queries');
|
||||
expect(pages[2]).toHaveTextContent('Page 3 of search queries');
|
||||
});
|
||||
|
||||
it('should render 3 pages for saved_query rule', () => {
|
||||
render(<LoggedRequestsPages requests={customQueryRuleTypeRequests} ruleType="saved_query" />, {
|
||||
wrapper: TestProviders,
|
||||
});
|
||||
|
||||
const pages = screen.getAllByTestId('preview-logged-requests-page-accordion');
|
||||
expect(pages).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('should render 2 pages for threshold rule', () => {
|
||||
const requests = [
|
||||
{
|
||||
request_type: 'findThresholdBuckets',
|
||||
description: 'request #1',
|
||||
request: 'POST test/_search',
|
||||
duration: 10,
|
||||
},
|
||||
{
|
||||
request_type: 'findThresholdBuckets',
|
||||
description: 'request #2',
|
||||
request: 'POST test/_search',
|
||||
duration: 10,
|
||||
},
|
||||
];
|
||||
render(<LoggedRequestsPages requests={requests} ruleType="threshold" />, {
|
||||
wrapper: TestProviders,
|
||||
});
|
||||
|
||||
const pages = screen.getAllByTestId('preview-logged-requests-page-accordion');
|
||||
expect(pages).toHaveLength(2);
|
||||
expect(pages[0]).toHaveTextContent('Page 1 of search queries');
|
||||
expect(pages[1]).toHaveTextContent('Page 2 of search queries');
|
||||
});
|
||||
|
||||
it('should render 2 pages for new_terms rule', async () => {
|
||||
const requests = [
|
||||
{
|
||||
request_type: 'findAllTerms',
|
||||
description: 'request #1',
|
||||
request: 'POST test/_search',
|
||||
duration: 10,
|
||||
},
|
||||
{
|
||||
request_type: 'findNewTerms',
|
||||
description: 'request #2',
|
||||
request: 'POST test/_search',
|
||||
duration: 10,
|
||||
},
|
||||
{
|
||||
request_type: 'findDocuments',
|
||||
description: 'request #3',
|
||||
request: 'POST test/_search',
|
||||
duration: 10,
|
||||
},
|
||||
{
|
||||
request_type: 'findAllTerms',
|
||||
description: 'request #4',
|
||||
request: 'POST test/_search',
|
||||
duration: 10,
|
||||
},
|
||||
{
|
||||
request_type: 'findNewTerms',
|
||||
description: 'request #5',
|
||||
request: 'POST test/_search',
|
||||
duration: 10,
|
||||
},
|
||||
];
|
||||
render(<LoggedRequestsPages requests={requests} ruleType="new_terms" />, {
|
||||
wrapper: TestProviders,
|
||||
});
|
||||
|
||||
const pages = screen.getAllByTestId('preview-logged-requests-page-accordion');
|
||||
expect(pages).toHaveLength(2);
|
||||
|
||||
// renders 3 requests on page 1
|
||||
await userEvent.click(screen.getByText('Page 1 of search queries'));
|
||||
expect(screen.getAllByTestId('preview-logged-request-code-block')).toHaveLength(3);
|
||||
|
||||
// renders 2 additional requests on page 2
|
||||
await userEvent.click(screen.getByText('Page 2 of search queries'));
|
||||
expect(screen.getAllByTestId('preview-logged-request-code-block')).toHaveLength(5);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { FC } from 'react';
|
||||
import React from 'react';
|
||||
import { css } from '@emotion/react';
|
||||
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import { useEuiPaddingSize } from '@elastic/eui';
|
||||
import type { Type } from '@kbn/securitysolution-io-ts-alerting-types';
|
||||
|
||||
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine';
|
||||
import { OptimizedAccordion } from './optimized_accordion';
|
||||
import { useAccordionStyling } from './use_accordion_styling';
|
||||
import { LoggedRequestsQuery } from './logged_requests_query';
|
||||
|
||||
const ruleRequestsTypesMap = {
|
||||
query: {
|
||||
findDocuments: 'pageDelimiter',
|
||||
},
|
||||
saved_query: {
|
||||
findDocuments: 'pageDelimiter',
|
||||
},
|
||||
threshold: {
|
||||
findThresholdBuckets: 'pageDelimiter',
|
||||
},
|
||||
new_terms: {
|
||||
findAllTerms: 'pageDelimiter',
|
||||
},
|
||||
};
|
||||
|
||||
type RuleTypesWithPages = keyof typeof ruleRequestsTypesMap;
|
||||
|
||||
export const isPageViewSupported = (ruleType: Type): ruleType is RuleTypesWithPages =>
|
||||
ruleType in ruleRequestsTypesMap;
|
||||
|
||||
const hasRequestType = (
|
||||
ruleType: RuleTypesWithPages,
|
||||
requestType: string
|
||||
): requestType is keyof (typeof ruleRequestsTypesMap)[typeof ruleType] =>
|
||||
requestType in ruleRequestsTypesMap[ruleType];
|
||||
|
||||
const transformRequestsToPages = (
|
||||
requests: RulePreviewLoggedRequest[],
|
||||
ruleType: RuleTypesWithPages
|
||||
): RulePreviewLoggedRequest[][] => {
|
||||
const pages: RulePreviewLoggedRequest[][] = [];
|
||||
requests.forEach((request) => {
|
||||
if (pages.length === 0) {
|
||||
pages.push([request]);
|
||||
} else if (
|
||||
request.request_type &&
|
||||
hasRequestType(ruleType, request.request_type) &&
|
||||
ruleRequestsTypesMap[ruleType][request.request_type] === 'pageDelimiter'
|
||||
) {
|
||||
pages.push([request]);
|
||||
} else {
|
||||
pages.at(-1)?.push(request);
|
||||
}
|
||||
});
|
||||
|
||||
return pages;
|
||||
};
|
||||
|
||||
const LoggedRequestsPagesComponent: FC<{
|
||||
requests: RulePreviewLoggedRequest[];
|
||||
ruleType: RuleTypesWithPages;
|
||||
}> = ({ requests, ruleType }) => {
|
||||
const cssStyles = useAccordionStyling();
|
||||
const paddingLarge = useEuiPaddingSize('l');
|
||||
const pages = transformRequestsToPages(requests, ruleType);
|
||||
|
||||
return (
|
||||
<>
|
||||
{pages.map((pageRequests, key) => (
|
||||
<OptimizedAccordion
|
||||
key={key}
|
||||
id={`preview-logged-requests-page-accordion-${key}`}
|
||||
data-test-subj="preview-logged-requests-page-accordion"
|
||||
buttonContent={
|
||||
<FormattedMessage
|
||||
id="xpack.securitySolution.detectionEngine.queryPreview.loggedRequestPageLabel"
|
||||
defaultMessage="Page {pageNumber} of search queries"
|
||||
values={{ pageNumber: key + 1 }}
|
||||
/>
|
||||
}
|
||||
borders="horizontal"
|
||||
css={css`
|
||||
margin-left: ${paddingLarge};
|
||||
${cssStyles}
|
||||
`}
|
||||
>
|
||||
{pageRequests.map((request, requestKey) => (
|
||||
<LoggedRequestsQuery key={requestKey} {...request} />
|
||||
))}
|
||||
</OptimizedAccordion>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export const LoggedRequestsPages = React.memo(LoggedRequestsPagesComponent);
|
||||
LoggedRequestsPages.displayName = 'LoggedRequestsPages';
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { render, screen } from '@testing-library/react';
|
||||
|
||||
import { LoggedRequestsQuery } from './logged_requests_query';
|
||||
|
||||
const description = 'Retrieve source documents when ES|QL query is not aggregable';
|
||||
const duration = 8;
|
||||
const request =
|
||||
'POST /packetbeat-8.14.2/_search?ignore_unavailable=true\n{\n "query": {\n "bool": {\n "filter": {\n "ids": {\n "values": [\n "yB7awpEBluhaSO8ejVKZ",\n "yR7awpEBluhaSO8ejVKZ",\n "yh7awpEBluhaSO8ejVKZ",\n "yx7awpEBluhaSO8ejVKZ",\n "zB7awpEBluhaSO8ejVKZ",\n "zR7awpEBluhaSO8ejVKZ",\n "zh7awpEBluhaSO8ejVKZ",\n "zx7awpEBluhaSO8ejVKZ",\n "0B7awpEBluhaSO8ejVKZ",\n "0R7awpEBluhaSO8ejVKZ",\n "0h7awpEBluhaSO8ejVKZ",\n "0x7awpEBluhaSO8ejVKZ",\n "1B7awpEBluhaSO8ejVKZ",\n "1R7awpEBluhaSO8ejVKZ",\n "1h7awpEBluhaSO8ejVKZ",\n "1x7awpEBluhaSO8ejVKZ",\n "2B7awpEBluhaSO8ejVKZ",\n "2R7awpEBluhaSO8ejVKZ",\n "2h7awpEBluhaSO8ejVKZ",\n "2x7awpEBluhaSO8ejVKZ",\n "3B7awpEBluhaSO8ejVKZ",\n "3R7awpEBluhaSO8ejVKZ",\n "3h7awpEBluhaSO8ejVKZ",\n "3x7awpEBluhaSO8ejVKZ",\n "4B7awpEBluhaSO8ejVKZ",\n "4R7awpEBluhaSO8ejVKZ",\n "4h7awpEBluhaSO8ejVKZ",\n "4x7awpEBluhaSO8ejVKZ",\n "5B7awpEBluhaSO8ejVKZ",\n "5R7awpEBluhaSO8ejVKZ",\n "5h7awpEBluhaSO8ejVKZ",\n "5x7awpEBluhaSO8ejVKZ",\n "6B7awpEBluhaSO8ejVKZ",\n "6R7awpEBluhaSO8ejVKZ",\n "6h7awpEBluhaSO8ejVKZ",\n "6x7awpEBluhaSO8ejVKZ",\n "7B7awpEBluhaSO8ejVKZ",\n "7R7awpEBluhaSO8ejVKZ",\n "7h7awpEBluhaSO8ejVKZ",\n "7x7awpEBluhaSO8ejVKZ",\n "8B7awpEBluhaSO8ejVKZ",\n "8R7awpEBluhaSO8ejVKZ",\n "8h7awpEBluhaSO8ejVKZ",\n "8x7awpEBluhaSO8ejVKZ",\n "9B7awpEBluhaSO8ejVKZ",\n "9R7awpEBluhaSO8ejVKZ",\n "9h7awpEBluhaSO8ejVKZ",\n "9x7awpEBluhaSO8ejVKZ",\n "-B7awpEBluhaSO8ejVKZ",\n "-R7awpEBluhaSO8ejVKZ",\n "-h7awpEBluhaSO8ejVKZ",\n "-x7awpEBluhaSO8ejVKZ",\n "_B7awpEBluhaSO8ejVKZ",\n "_R7awpEBluhaSO8ejVKZ",\n "_h7awpEBluhaSO8ejVKZ",\n "_x7awpEBluhaSO8ejVKZ",\n "AB7awpEBluhaSO8ejVOZ",\n "AR7awpEBluhaSO8ejVOZ",\n "Ah7awpEBluhaSO8ejVOZ",\n "Ax7awpEBluhaSO8ejVOZ",\n "BB7awpEBluhaSO8ejVOZ",\n "BR7awpEBluhaSO8ejVOZ",\n "Bh7awpEBluhaSO8ejVOZ",\n "Bx7awpEBluhaSO8ejVOZ",\n "CB7awpEBluhaSO8ejVOZ",\n "CR7awpEBluhaSO8ejVOZ",\n "Ch7awpEBluhaSO8ejVOZ",\n "Cx7awpEBluhaSO8ejVOZ",\n "DB7awpEBluhaSO8ejVOZ",\n "DR7awpEBluhaSO8ejVOZ",\n "Dh7awpEBluhaSO8ejVOZ",\n "Dx7awpEBluhaSO8ejVOZ",\n "EB7awpEBluhaSO8ejVOZ",\n "ER7awpEBluhaSO8ejVOZ",\n "Eh7awpEBluhaSO8ejVOZ",\n "Ex7awpEBluhaSO8ejVOZ",\n "FB7awpEBluhaSO8ejVOZ",\n "FR7awpEBluhaSO8ejVOZ",\n "Fh7awpEBluhaSO8ejVOZ",\n "Fx7awpEBluhaSO8ejVOZ",\n "GB7awpEBluhaSO8ejVOZ",\n "GR7awpEBluhaSO8ejVOZ",\n "Gh7awpEBluhaSO8ejVOZ",\n "Gx7awpEBluhaSO8ejVOZ",\n "HB7awpEBluhaSO8ejVOZ",\n "HR7awpEBluhaSO8ejVOZ",\n "Hh7awpEBluhaSO8ejVOZ",\n "Hx7awpEBluhaSO8ejVOZ",\n "IB7awpEBluhaSO8ejVOZ",\n "IR7awpEBluhaSO8ejVOZ",\n "Ih7awpEBluhaSO8ejVOZ",\n "Ix7awpEBluhaSO8ejVOZ",\n "JB7awpEBluhaSO8ejVOZ",\n "JR7awpEBluhaSO8ejVOZ",\n "Jh7awpEBluhaSO8ejVOZ",\n "Jx7awpEBluhaSO8ejVOZ",\n "KB7awpEBluhaSO8ejVOZ",\n "KR7awpEBluhaSO8ejVOZ",\n "Kh7awpEBluhaSO8ejVOZ",\n "Kx7awpEBluhaSO8ejVOZ",\n "LB7awpEBluhaSO8ejVOZ"\n ]\n }\n }\n }\n },\n "_source": false,\n "fields": [\n "*"\n ]\n}';
|
||||
|
||||
describe('LoggedRequestsQuery', () => {
|
||||
it('should not render code block when request field is empty', () => {
|
||||
render(<LoggedRequestsQuery description={description} duration={duration} />);
|
||||
|
||||
expect(screen.queryByTestId('preview-logged-request-code-block')).toBeNull();
|
||||
});
|
||||
|
||||
it('should render code block', () => {
|
||||
render(<LoggedRequestsQuery description={description} duration={duration} request={request} />);
|
||||
|
||||
expect(screen.queryByTestId('preview-logged-request-code-block')).toHaveTextContent(
|
||||
'POST /packetbeat-8.14.2/_search?ignore_unavailable=true'
|
||||
);
|
||||
});
|
||||
|
||||
it('should render duration', () => {
|
||||
render(<LoggedRequestsQuery description={description} duration={duration} request={request} />);
|
||||
|
||||
expect(screen.queryByTestId('preview-logged-request-description')).toHaveTextContent('8ms');
|
||||
});
|
||||
|
||||
it('should not render duration when it absent', () => {
|
||||
render(<LoggedRequestsQuery description={description} request={request} />);
|
||||
|
||||
expect(screen.queryByTestId('preview-logged-request-description')).not.toHaveTextContent('ms');
|
||||
});
|
||||
|
||||
it('should render description', () => {
|
||||
render(<LoggedRequestsQuery description={description} request={request} />);
|
||||
|
||||
expect(screen.queryByTestId('preview-logged-request-description')).toHaveTextContent(
|
||||
description
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { FC } from 'react';
|
||||
import React from 'react';
|
||||
|
||||
import { EuiSpacer, EuiCodeBlock, useEuiPaddingSize, EuiFlexItem } from '@elastic/eui';
|
||||
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine';
|
||||
|
||||
export const LoggedRequestsQueryComponent: FC<RulePreviewLoggedRequest> = ({
|
||||
description,
|
||||
duration,
|
||||
request,
|
||||
}) => {
|
||||
const paddingLarge = useEuiPaddingSize('l');
|
||||
|
||||
return (
|
||||
<EuiFlexItem css={{ 'padding-left': paddingLarge }}>
|
||||
<EuiSpacer size="l" />
|
||||
<span data-test-subj="preview-logged-request-description">
|
||||
{description ?? null} {duration ? `[${duration}ms]` : null}
|
||||
</span>
|
||||
<EuiSpacer size="s" />
|
||||
{request ? (
|
||||
<EuiCodeBlock
|
||||
language="json"
|
||||
isCopyable
|
||||
overflowHeight={300}
|
||||
isVirtualized
|
||||
data-test-subj="preview-logged-request-code-block"
|
||||
>
|
||||
{request}
|
||||
</EuiCodeBlock>
|
||||
) : null}
|
||||
</EuiFlexItem>
|
||||
);
|
||||
};
|
||||
|
||||
export const LoggedRequestsQuery = React.memo(LoggedRequestsQueryComponent);
|
||||
LoggedRequestsQuery.displayName = 'LoggedRequestsQuery';
|
||||
|
|
@ -9,6 +9,7 @@ import type { FC, PropsWithChildren } from 'react';
|
|||
import React, { Fragment, useMemo } from 'react';
|
||||
import { css } from '@emotion/css';
|
||||
import { EuiCallOut, EuiText, EuiSpacer, EuiAccordion } from '@elastic/eui';
|
||||
import type { Type } from '@kbn/securitysolution-io-ts-alerting-types';
|
||||
|
||||
import type { RulePreviewLogs } from '../../../../../common/api/detection_engine';
|
||||
import * as i18n from './translations';
|
||||
|
|
@ -20,6 +21,7 @@ interface PreviewLogsProps {
|
|||
hasNoiseWarning: boolean;
|
||||
isAborted: boolean;
|
||||
showElasticsearchRequests: boolean;
|
||||
ruleType: Type;
|
||||
}
|
||||
|
||||
interface SortedLogs {
|
||||
|
|
@ -53,6 +55,7 @@ const PreviewLogsComponent: React.FC<PreviewLogsProps> = ({
|
|||
hasNoiseWarning,
|
||||
isAborted,
|
||||
showElasticsearchRequests,
|
||||
ruleType,
|
||||
}) => {
|
||||
const sortedLogs = useMemo(
|
||||
() =>
|
||||
|
|
@ -76,7 +79,7 @@ const PreviewLogsComponent: React.FC<PreviewLogsProps> = ({
|
|||
<LogAccordion logs={sortedLogs.warnings}>
|
||||
{isAborted ? <CustomWarning message={i18n.PREVIEW_TIMEOUT_WARNING} /> : null}
|
||||
</LogAccordion>
|
||||
{showElasticsearchRequests ? <LoggedRequests logs={logs} /> : null}
|
||||
{showElasticsearchRequests ? <LoggedRequests logs={logs} ruleType={ruleType} /> : null}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -206,3 +206,10 @@ export const RULE_PREVIEW_DESCRIPTION = i18n.translate(
|
|||
'Rule preview reflects the current configuration of your rule settings and exceptions, click refresh icon to see the updated preview.',
|
||||
}
|
||||
);
|
||||
|
||||
export const REQUESTS_SAMPLE_WARNING = i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.createRule.rulePreviewRequestSampleWarningText',
|
||||
{
|
||||
defaultMessage: 'Sample search queries logged only for first 2 requests of each type.',
|
||||
}
|
||||
);
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ export const getEventCount = async ({
|
|||
secondaryTimestamp,
|
||||
searchAfterSortIds: undefined,
|
||||
runtimeMappings: undefined,
|
||||
}).body.query;
|
||||
}).body?.query;
|
||||
const response = await esClient.count({
|
||||
body: { query: eventSearchQueryBodyQuery },
|
||||
ignore_unavailable: true,
|
||||
|
|
|
|||
|
|
@ -18,7 +18,12 @@ import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts';
|
|||
|
||||
export const createMlAlertType = (
|
||||
createOptions: CreateRuleOptions
|
||||
): SecurityAlertType<MachineLearningRuleParams, {}, {}, 'default'> => {
|
||||
): SecurityAlertType<
|
||||
MachineLearningRuleParams,
|
||||
{ isLoggedRequestsEnabled?: boolean },
|
||||
{},
|
||||
'default'
|
||||
> => {
|
||||
const { experimentalFeatures, ml, licensing, scheduleNotificationResponseActionsService } =
|
||||
createOptions;
|
||||
return {
|
||||
|
|
@ -76,6 +81,7 @@ export const createMlAlertType = (
|
|||
alertSuppression: completeRule.ruleParams.alertSuppression,
|
||||
licensing,
|
||||
});
|
||||
const isLoggedRequestsEnabled = Boolean(state?.isLoggedRequestsEnabled);
|
||||
|
||||
const wrapSuppressedHits: WrapSuppressedHits = (events, buildReasonMessage) =>
|
||||
wrapSuppressedAlerts({
|
||||
|
|
@ -93,7 +99,7 @@ export const createMlAlertType = (
|
|||
intendedTimestamp,
|
||||
});
|
||||
|
||||
const result = await mlExecutor({
|
||||
const { result, loggedRequests } = await mlExecutor({
|
||||
completeRule,
|
||||
tuple,
|
||||
ml,
|
||||
|
|
@ -110,8 +116,9 @@ export const createMlAlertType = (
|
|||
isAlertSuppressionActive,
|
||||
experimentalFeatures,
|
||||
scheduleNotificationResponseActionsService,
|
||||
isLoggedRequestsEnabled,
|
||||
});
|
||||
return { ...result, state };
|
||||
return { ...result, state, ...(isLoggedRequestsEnabled ? { loggedRequests } : {}) };
|
||||
},
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -10,7 +10,10 @@ import type { KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server
|
|||
import type { MlPluginSetup } from '@kbn/ml-plugin/server';
|
||||
import type { Filter } from '@kbn/es-query';
|
||||
import type { AnomalyResults } from '../../../machine_learning';
|
||||
import { getAnomalies } from '../../../machine_learning';
|
||||
import { getAnomalies, buildAnomalyQuery } from '../../../machine_learning';
|
||||
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
|
||||
import { logSearchRequest } from '../utils/logged_requests';
|
||||
import * as i18n from '../translations';
|
||||
|
||||
export const findMlSignals = async ({
|
||||
ml,
|
||||
|
|
@ -22,6 +25,7 @@ export const findMlSignals = async ({
|
|||
to,
|
||||
maxSignals,
|
||||
exceptionFilter,
|
||||
isLoggedRequestsEnabled,
|
||||
}: {
|
||||
ml: MlPluginSetup;
|
||||
request: KibanaRequest;
|
||||
|
|
@ -32,7 +36,10 @@ export const findMlSignals = async ({
|
|||
to: string;
|
||||
maxSignals: number;
|
||||
exceptionFilter: Filter | undefined;
|
||||
}): Promise<AnomalyResults> => {
|
||||
isLoggedRequestsEnabled: boolean;
|
||||
}): Promise<{ anomalyResults: AnomalyResults; loggedRequests?: RulePreviewLoggedRequest[] }> => {
|
||||
const loggedRequests: RulePreviewLoggedRequest[] = [];
|
||||
|
||||
const { mlAnomalySearch } = ml.mlSystemProvider(request, savedObjectsClient);
|
||||
const params = {
|
||||
jobIds,
|
||||
|
|
@ -42,5 +49,18 @@ export const findMlSignals = async ({
|
|||
maxRecords: maxSignals,
|
||||
exceptionFilter,
|
||||
};
|
||||
return getAnomalies(params, mlAnomalySearch);
|
||||
|
||||
const anomalyResults = await getAnomalies(params, mlAnomalySearch);
|
||||
|
||||
if (isLoggedRequestsEnabled) {
|
||||
const searchQuery = buildAnomalyQuery(params);
|
||||
searchQuery.index = '.ml-anomalies-*';
|
||||
loggedRequests.push({
|
||||
request: logSearchRequest(searchQuery),
|
||||
description: i18n.ML_SEARCH_ANOMALIES_DESCRIPTION,
|
||||
duration: anomalyResults.took,
|
||||
request_type: 'findAnomalies',
|
||||
});
|
||||
}
|
||||
return { anomalyResults, ...(isLoggedRequestsEnabled ? { loggedRequests } : {}) };
|
||||
};
|
||||
|
|
|
|||
|
|
@ -57,9 +57,11 @@ describe('ml_executor', () => {
|
|||
ruleType: mlCompleteRule.ruleConfig.ruleTypeId,
|
||||
});
|
||||
(findMlSignals as jest.Mock).mockResolvedValue({
|
||||
_shards: {},
|
||||
hits: {
|
||||
hits: [],
|
||||
anomalyResults: {
|
||||
_shards: {},
|
||||
hits: {
|
||||
hits: [],
|
||||
},
|
||||
},
|
||||
});
|
||||
(bulkCreateMlSignals as jest.Mock).mockResolvedValue({
|
||||
|
|
@ -97,7 +99,7 @@ describe('ml_executor', () => {
|
|||
|
||||
it('should record a partial failure if Machine learning job summary was null', async () => {
|
||||
jobsSummaryMock.mockResolvedValue([]);
|
||||
const response = await mlExecutor({
|
||||
const { result } = await mlExecutor({
|
||||
completeRule: mlCompleteRule,
|
||||
tuple,
|
||||
ml: mlMock,
|
||||
|
|
@ -119,7 +121,7 @@ describe('ml_executor', () => {
|
|||
expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain(
|
||||
'Machine learning job(s) are not started'
|
||||
);
|
||||
expect(response.warningMessages.length).toEqual(1);
|
||||
expect(result.warningMessages.length).toEqual(1);
|
||||
});
|
||||
|
||||
it('should record a partial failure if Machine learning job was not started', async () => {
|
||||
|
|
@ -131,7 +133,7 @@ describe('ml_executor', () => {
|
|||
},
|
||||
]);
|
||||
|
||||
const response = await mlExecutor({
|
||||
const { result } = await mlExecutor({
|
||||
completeRule: mlCompleteRule,
|
||||
tuple,
|
||||
ml: mlMock,
|
||||
|
|
@ -153,7 +155,7 @@ describe('ml_executor', () => {
|
|||
expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain(
|
||||
'Machine learning job(s) are not started'
|
||||
);
|
||||
expect(response.warningMessages.length).toEqual(1);
|
||||
expect(result.warningMessages.length).toEqual(1);
|
||||
});
|
||||
|
||||
it('should report job missing errors as user errors', async () => {
|
||||
|
|
@ -161,7 +163,7 @@ describe('ml_executor', () => {
|
|||
message: 'my_test_job_name missing',
|
||||
});
|
||||
|
||||
const result = await mlExecutor({
|
||||
const { result } = await mlExecutor({
|
||||
completeRule: mlCompleteRule,
|
||||
tuple,
|
||||
ml: mlMock,
|
||||
|
|
@ -194,7 +196,7 @@ describe('ml_executor', () => {
|
|||
}))
|
||||
);
|
||||
|
||||
const result = await mlExecutor({
|
||||
const { result } = await mlExecutor({
|
||||
completeRule: mlCompleteRule,
|
||||
tuple,
|
||||
ml: mlMock,
|
||||
|
|
@ -220,7 +222,7 @@ describe('ml_executor', () => {
|
|||
);
|
||||
});
|
||||
it('should call scheduleNotificationResponseActionsService', async () => {
|
||||
const result = await mlExecutor({
|
||||
const { result } = await mlExecutor({
|
||||
completeRule: mlCompleteRule,
|
||||
tuple,
|
||||
ml: mlMock,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import type {
|
|||
} from '@kbn/alerting-plugin/server';
|
||||
import type { ListClient } from '@kbn/lists-plugin/server';
|
||||
import type { Filter } from '@kbn/es-query';
|
||||
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
|
||||
import { isJobStarted } from '../../../../../common/machine_learning/helpers';
|
||||
import type { ExperimentalFeatures } from '../../../../../common/experimental_features';
|
||||
import type { CompleteRule, MachineLearningRuleParams } from '../../rule_schema';
|
||||
|
|
@ -61,6 +62,7 @@ interface MachineLearningRuleExecutorParams {
|
|||
isAlertSuppressionActive: boolean;
|
||||
experimentalFeatures: ExperimentalFeatures;
|
||||
scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService'];
|
||||
isLoggedRequestsEnabled?: boolean;
|
||||
}
|
||||
|
||||
export const mlExecutor = async ({
|
||||
|
|
@ -80,9 +82,11 @@ export const mlExecutor = async ({
|
|||
alertWithSuppression,
|
||||
experimentalFeatures,
|
||||
scheduleNotificationResponseActionsService,
|
||||
isLoggedRequestsEnabled = false,
|
||||
}: MachineLearningRuleExecutorParams) => {
|
||||
const result = createSearchAfterReturnType();
|
||||
const ruleParams = completeRule.ruleParams;
|
||||
const loggedRequests: RulePreviewLoggedRequest[] = [];
|
||||
|
||||
return withSecuritySpan('mlExecutor', async () => {
|
||||
if (ml == null) {
|
||||
|
|
@ -122,7 +126,7 @@ export const mlExecutor = async ({
|
|||
|
||||
let anomalyResults: AnomalyResults;
|
||||
try {
|
||||
anomalyResults = await findMlSignals({
|
||||
const searchResults = await findMlSignals({
|
||||
ml,
|
||||
// Using fake KibanaRequest as it is needed to satisfy the ML Services API, but can be empty as it is
|
||||
// currently unused by the mlAnomalySearch function.
|
||||
|
|
@ -134,14 +138,17 @@ export const mlExecutor = async ({
|
|||
to: tuple.to.toISOString(),
|
||||
maxSignals: tuple.maxSignals,
|
||||
exceptionFilter,
|
||||
isLoggedRequestsEnabled,
|
||||
});
|
||||
anomalyResults = searchResults.anomalyResults;
|
||||
loggedRequests.push(...(searchResults.loggedRequests ?? []));
|
||||
} catch (error) {
|
||||
if (typeof error.message === 'string' && (error.message as string).endsWith('missing')) {
|
||||
result.userError = true;
|
||||
}
|
||||
result.errors.push(error.message);
|
||||
result.success = false;
|
||||
return result;
|
||||
return { result };
|
||||
}
|
||||
|
||||
// TODO we add the max_signals warning _before_ filtering the anomalies against the exceptions list. Is that correct?
|
||||
|
|
@ -204,12 +211,15 @@ export const mlExecutor = async ({
|
|||
signalsCount: result.createdSignalsCount,
|
||||
responseActions: completeRule.ruleParams.responseActions,
|
||||
});
|
||||
return mergeReturns([
|
||||
result,
|
||||
createSearchAfterReturnType({
|
||||
success: anomalyResults._shards.failed === 0,
|
||||
errors: searchErrors,
|
||||
}),
|
||||
]);
|
||||
return {
|
||||
result: mergeReturns([
|
||||
result,
|
||||
createSearchAfterReturnType({
|
||||
success: anomalyResults._shards.failed === 0,
|
||||
errors: searchErrors,
|
||||
}),
|
||||
]),
|
||||
...(isLoggedRequestsEnabled ? { loggedRequests } : {}),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -39,15 +39,18 @@ import {
|
|||
getUnprocessedExceptionsWarnings,
|
||||
getMaxSignalsWarning,
|
||||
getSuppressionMaxSignalsWarning,
|
||||
stringifyAfterKey,
|
||||
} from '../utils/utils';
|
||||
import { createEnrichEventsFunction } from '../utils/enrichments';
|
||||
import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active';
|
||||
import { multiTermsComposite } from './multi_terms_composite';
|
||||
import type { GenericBulkCreateResponse } from '../utils/bulk_create_with_suppression';
|
||||
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
|
||||
import * as i18n from '../translations';
|
||||
|
||||
export const createNewTermsAlertType = (
|
||||
createOptions: CreateRuleOptions
|
||||
): SecurityAlertType<NewTermsRuleParams, {}, {}, 'default'> => {
|
||||
): SecurityAlertType<NewTermsRuleParams, { isLoggedRequestsEnabled?: boolean }, {}, 'default'> => {
|
||||
const { logger, licensing, experimentalFeatures, scheduleNotificationResponseActionsService } =
|
||||
createOptions;
|
||||
return {
|
||||
|
|
@ -119,6 +122,9 @@ export const createNewTermsAlertType = (
|
|||
state,
|
||||
} = execOptions;
|
||||
|
||||
const isLoggedRequestsEnabled = Boolean(state?.isLoggedRequestsEnabled);
|
||||
const loggedRequests: RulePreviewLoggedRequest[] = [];
|
||||
|
||||
// Validate the history window size compared to `from` at runtime as well as in the `validate`
|
||||
// function because rule preview does not use the `validate` function defined on the rule type
|
||||
validateHistoryWindowStart({
|
||||
|
|
@ -157,6 +163,7 @@ export const createNewTermsAlertType = (
|
|||
if (exceptionsWarning) {
|
||||
result.warningMessages.push(exceptionsWarning);
|
||||
}
|
||||
let pageNumber = 0;
|
||||
|
||||
// There are 2 conditions that mean we're finished: either there were still too many alerts to create
|
||||
// after deduplication and the array of alerts was truncated before being submitted to ES, or there were
|
||||
|
|
@ -165,10 +172,16 @@ export const createNewTermsAlertType = (
|
|||
// in which case createdSignalsCount would still be less than maxSignals. Since valid alerts were truncated from
|
||||
// the array in that case, we stop and report the errors.
|
||||
while (result.createdSignalsCount <= params.maxSignals) {
|
||||
pageNumber++;
|
||||
// PHASE 1: Fetch a page of terms using a composite aggregation. This will collect a page from
|
||||
// all of the terms seen over the last rule interval. In the next phase we'll determine which
|
||||
// ones are new.
|
||||
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
|
||||
const {
|
||||
searchResult,
|
||||
searchDuration,
|
||||
searchErrors,
|
||||
loggedRequests: firstPhaseLoggedRequests = [],
|
||||
} = await singleSearchAfter({
|
||||
aggregations: buildRecentTermsAgg({
|
||||
fields: params.newTermsFields,
|
||||
after: afterKey,
|
||||
|
|
@ -185,7 +198,17 @@ export const createNewTermsAlertType = (
|
|||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
runtimeMappings,
|
||||
loggedRequestsConfig: isLoggedRequestsEnabled
|
||||
? {
|
||||
type: 'findAllTerms',
|
||||
description: i18n.FIND_ALL_NEW_TERMS_FIELDS_DESCRIPTION(
|
||||
stringifyAfterKey(afterKey)
|
||||
),
|
||||
skipRequestQuery: pageNumber > 2,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
loggedRequests.push(...firstPhaseLoggedRequests);
|
||||
const searchResultWithAggs = searchResult as RecentTermsAggResult;
|
||||
if (!searchResultWithAggs.aggregations) {
|
||||
throw new Error('Aggregations were missing on recent terms search result');
|
||||
|
|
@ -316,9 +339,11 @@ export const createNewTermsAlertType = (
|
|||
afterKey,
|
||||
createAlertsHook,
|
||||
isAlertSuppressionActive,
|
||||
isLoggedRequestsEnabled,
|
||||
});
|
||||
loggedRequests.push(...(bulkCreateResult?.loggedRequests ?? []));
|
||||
|
||||
if (bulkCreateResult?.alertsWereTruncated) {
|
||||
if (bulkCreateResult && 'alertsWereTruncated' in bulkCreateResult) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -330,6 +355,7 @@ export const createNewTermsAlertType = (
|
|||
searchResult: pageSearchResult,
|
||||
searchDuration: pageSearchDuration,
|
||||
searchErrors: pageSearchErrors,
|
||||
loggedRequests: pageSearchLoggedRequests = [],
|
||||
} = await singleSearchAfter({
|
||||
aggregations: buildNewTermsAgg({
|
||||
newValueWindowStart: tuple.from,
|
||||
|
|
@ -350,9 +376,17 @@ export const createNewTermsAlertType = (
|
|||
pageSize: 0,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
loggedRequestsConfig: isLoggedRequestsEnabled
|
||||
? {
|
||||
type: 'findNewTerms',
|
||||
description: i18n.FIND_NEW_TERMS_VALUES_DESCRIPTION(stringifyAfterKey(afterKey)),
|
||||
skipRequestQuery: pageNumber > 2,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
result.searchAfterTimes.push(pageSearchDuration);
|
||||
result.errors.push(...pageSearchErrors);
|
||||
loggedRequests.push(...pageSearchLoggedRequests);
|
||||
|
||||
logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`);
|
||||
|
||||
|
|
@ -374,6 +408,7 @@ export const createNewTermsAlertType = (
|
|||
searchResult: docFetchSearchResult,
|
||||
searchDuration: docFetchSearchDuration,
|
||||
searchErrors: docFetchSearchErrors,
|
||||
loggedRequests: docFetchLoggedRequests = [],
|
||||
} = await singleSearchAfter({
|
||||
aggregations: buildDocFetchAgg({
|
||||
timestampField: aggregatableTimestampField,
|
||||
|
|
@ -392,9 +427,19 @@ export const createNewTermsAlertType = (
|
|||
pageSize: 0,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
loggedRequestsConfig: isLoggedRequestsEnabled
|
||||
? {
|
||||
type: 'findDocuments',
|
||||
description: i18n.FIND_NEW_TERMS_EVENTS_DESCRIPTION(
|
||||
stringifyAfterKey(afterKey)
|
||||
),
|
||||
skipRequestQuery: pageNumber > 2,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
result.searchAfterTimes.push(docFetchSearchDuration);
|
||||
result.errors.push(...docFetchSearchErrors);
|
||||
loggedRequests.push(...docFetchLoggedRequests);
|
||||
|
||||
const docFetchResultWithAggs = docFetchSearchResult as DocFetchAggResult;
|
||||
|
||||
|
|
@ -424,7 +469,7 @@ export const createNewTermsAlertType = (
|
|||
responseActions: completeRule.ruleParams.responseActions,
|
||||
});
|
||||
|
||||
return { ...result, state };
|
||||
return { ...result, state, ...(isLoggedRequestsEnabled ? { loggedRequests } : {}) };
|
||||
},
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -22,10 +22,16 @@ import type {
|
|||
CreateAlertsHook,
|
||||
} from './build_new_terms_aggregation';
|
||||
import type { NewTermsFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts';
|
||||
import { getMaxSignalsWarning, getSuppressionMaxSignalsWarning } from '../utils/utils';
|
||||
import {
|
||||
getMaxSignalsWarning,
|
||||
getSuppressionMaxSignalsWarning,
|
||||
stringifyAfterKey,
|
||||
} from '../utils/utils';
|
||||
import type { GenericBulkCreateResponse } from '../utils/bulk_create_with_suppression';
|
||||
|
||||
import type { RuleServices, SearchAfterAndBulkCreateReturnType, RunOpts } from '../types';
|
||||
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
|
||||
import * as i18n from '../translations';
|
||||
|
||||
/**
|
||||
* composite aggregation page batch size set to 500 as it shows th best performance(refer https://github.com/elastic/kibana/pull/157413) and
|
||||
|
|
@ -49,12 +55,23 @@ interface MultiTermsCompositeArgsBase {
|
|||
afterKey: Record<string, string | number | null> | undefined;
|
||||
createAlertsHook: CreateAlertsHook;
|
||||
isAlertSuppressionActive: boolean;
|
||||
isLoggedRequestsEnabled: boolean;
|
||||
}
|
||||
|
||||
interface MultiTermsCompositeArgs extends MultiTermsCompositeArgsBase {
|
||||
batchSize: number;
|
||||
}
|
||||
|
||||
interface LoggedRequestsProps {
|
||||
loggedRequests?: RulePreviewLoggedRequest[];
|
||||
}
|
||||
|
||||
type MultiTermsCompositeResult =
|
||||
| (Omit<GenericBulkCreateResponse<NewTermsFieldsLatest>, 'suppressedItemsCount'> &
|
||||
LoggedRequestsProps)
|
||||
| LoggedRequestsProps
|
||||
| undefined;
|
||||
|
||||
/**
|
||||
* This helper does phase2/phase3(look README) got multiple new terms
|
||||
* It takes full page of results from phase 1 (10,000)
|
||||
|
|
@ -75,9 +92,8 @@ const multiTermsCompositeNonRetryable = async ({
|
|||
createAlertsHook,
|
||||
batchSize,
|
||||
isAlertSuppressionActive,
|
||||
}: MultiTermsCompositeArgs): Promise<
|
||||
Omit<GenericBulkCreateResponse<NewTermsFieldsLatest>, 'suppressedItemsCount'> | undefined
|
||||
> => {
|
||||
isLoggedRequestsEnabled,
|
||||
}: MultiTermsCompositeArgs): Promise<MultiTermsCompositeResult> => {
|
||||
const {
|
||||
ruleExecutionLogger,
|
||||
tuple,
|
||||
|
|
@ -87,11 +103,15 @@ const multiTermsCompositeNonRetryable = async ({
|
|||
secondaryTimestamp,
|
||||
} = runOpts;
|
||||
|
||||
const loggedRequests: RulePreviewLoggedRequest[] = [];
|
||||
|
||||
let internalAfterKey = afterKey ?? undefined;
|
||||
|
||||
let i = 0;
|
||||
let pageNumber = 0;
|
||||
|
||||
while (i < buckets.length) {
|
||||
pageNumber++;
|
||||
const batch = buckets.slice(i, i + batchSize);
|
||||
i += batchSize;
|
||||
const batchFilters = batch.map((b) => {
|
||||
|
|
@ -115,6 +135,7 @@ const multiTermsCompositeNonRetryable = async ({
|
|||
searchResult: pageSearchResult,
|
||||
searchDuration: pageSearchDuration,
|
||||
searchErrors: pageSearchErrors,
|
||||
loggedRequests: pageSearchLoggedRequests = [],
|
||||
} = await singleSearchAfter({
|
||||
aggregations: buildCompositeNewTermsAgg({
|
||||
newValueWindowStart: tuple.from,
|
||||
|
|
@ -136,10 +157,20 @@ const multiTermsCompositeNonRetryable = async ({
|
|||
pageSize: 0,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
loggedRequestsConfig: isLoggedRequestsEnabled
|
||||
? {
|
||||
type: 'findNewTerms',
|
||||
description: i18n.FIND_NEW_TERMS_VALUES_DESCRIPTION(
|
||||
stringifyAfterKey(internalAfterKey)
|
||||
),
|
||||
skipRequestQuery: Boolean(afterKey) || pageNumber > 2,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
|
||||
result.searchAfterTimes.push(pageSearchDuration);
|
||||
result.errors.push(...pageSearchErrors);
|
||||
loggedRequests.push(...pageSearchLoggedRequests);
|
||||
logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`);
|
||||
|
||||
const pageSearchResultWithAggs = pageSearchResult as CompositeNewTermsAggResult;
|
||||
|
|
@ -156,6 +187,7 @@ const multiTermsCompositeNonRetryable = async ({
|
|||
searchResult: docFetchSearchResult,
|
||||
searchDuration: docFetchSearchDuration,
|
||||
searchErrors: docFetchSearchErrors,
|
||||
loggedRequests: docFetchLoggedRequests = [],
|
||||
} = await singleSearchAfter({
|
||||
aggregations: buildCompositeDocFetchAgg({
|
||||
newValueWindowStart: tuple.from,
|
||||
|
|
@ -175,9 +207,19 @@ const multiTermsCompositeNonRetryable = async ({
|
|||
pageSize: 0,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
loggedRequestsConfig: isLoggedRequestsEnabled
|
||||
? {
|
||||
type: 'findDocuments',
|
||||
description: i18n.FIND_NEW_TERMS_EVENTS_DESCRIPTION(
|
||||
stringifyAfterKey(internalAfterKey)
|
||||
),
|
||||
skipRequestQuery: Boolean(afterKey) || pageNumber > 2,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
result.searchAfterTimes.push(docFetchSearchDuration);
|
||||
result.errors.push(...docFetchSearchErrors);
|
||||
loggedRequests.push(...docFetchLoggedRequests);
|
||||
|
||||
const docFetchResultWithAggs = docFetchSearchResult as CompositeDocFetchAggResult;
|
||||
|
||||
|
|
@ -191,12 +233,14 @@ const multiTermsCompositeNonRetryable = async ({
|
|||
result.warningMessages.push(
|
||||
isAlertSuppressionActive ? getSuppressionMaxSignalsWarning() : getMaxSignalsWarning()
|
||||
);
|
||||
return bulkCreateResult;
|
||||
return isLoggedRequestsEnabled ? { ...bulkCreateResult, loggedRequests } : bulkCreateResult;
|
||||
}
|
||||
}
|
||||
|
||||
internalAfterKey = batch[batch.length - 1]?.key;
|
||||
}
|
||||
|
||||
return { loggedRequests };
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -206,9 +250,7 @@ const multiTermsCompositeNonRetryable = async ({
|
|||
*/
|
||||
export const multiTermsComposite = async (
|
||||
args: MultiTermsCompositeArgsBase
|
||||
): Promise<
|
||||
Omit<GenericBulkCreateResponse<NewTermsFieldsLatest>, 'suppressedItemsCount'> | undefined
|
||||
> => {
|
||||
): Promise<MultiTermsCompositeResult> => {
|
||||
let retryBatchSize = BATCH_SIZE;
|
||||
const ruleExecutionLogger = args.runOpts.ruleExecutionLogger;
|
||||
return pRetry(
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../../c
|
|||
import type { ExperimentalFeatures } from '../../../../../../common';
|
||||
import { createEnrichEventsFunction } from '../../utils/enrichments';
|
||||
import { getNumberOfSuppressedAlerts } from '../../utils/get_number_of_suppressed_alerts';
|
||||
import * as i18n from '../../translations';
|
||||
|
||||
export interface BucketHistory {
|
||||
key: Record<string, string | number | null>;
|
||||
|
|
@ -49,11 +50,13 @@ export interface GroupAndBulkCreateParams {
|
|||
groupByFields: string[];
|
||||
eventsTelemetry: ITelemetryEventsSender | undefined;
|
||||
experimentalFeatures: ExperimentalFeatures;
|
||||
isLoggedRequestsEnabled: boolean;
|
||||
}
|
||||
|
||||
export interface GroupAndBulkCreateReturnType extends SearchAfterAndBulkCreateReturnType {
|
||||
state: {
|
||||
suppressionGroupHistory: BucketHistory[];
|
||||
isLoggedRequestsEnabled?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -128,6 +131,7 @@ export const groupAndBulkCreate = async ({
|
|||
groupByFields,
|
||||
eventsTelemetry,
|
||||
experimentalFeatures,
|
||||
isLoggedRequestsEnabled,
|
||||
}: GroupAndBulkCreateParams): Promise<GroupAndBulkCreateReturnType> => {
|
||||
return withSecuritySpan('groupAndBulkCreate', async () => {
|
||||
const tuple = runOpts.tuple;
|
||||
|
|
@ -149,6 +153,7 @@ export const groupAndBulkCreate = async ({
|
|||
errors: [],
|
||||
warningMessages: [],
|
||||
state: {
|
||||
isLoggedRequestsEnabled,
|
||||
suppressionGroupHistory: filteredBucketHistory,
|
||||
},
|
||||
};
|
||||
|
|
@ -197,11 +202,19 @@ export const groupAndBulkCreate = async ({
|
|||
secondaryTimestamp: runOpts.secondaryTimestamp,
|
||||
runtimeMappings: runOpts.runtimeMappings,
|
||||
additionalFilters: bucketHistoryFilter,
|
||||
loggedRequestsConfig: isLoggedRequestsEnabled
|
||||
? {
|
||||
type: 'findDocuments',
|
||||
description: i18n.FIND_EVENTS_DESCRIPTION,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter(
|
||||
eventsSearchParams
|
||||
);
|
||||
const { searchResult, searchDuration, searchErrors, loggedRequests } =
|
||||
await singleSearchAfter(eventsSearchParams);
|
||||
|
||||
if (isLoggedRequestsEnabled) {
|
||||
toReturn.loggedRequests = loggedRequests;
|
||||
}
|
||||
toReturn.searchAfterTimes.push(searchDuration);
|
||||
toReturn.errors.push(...searchErrors);
|
||||
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@ export const createQueryAlertType = (
|
|||
bucketHistory: state.suppressionGroupHistory,
|
||||
licensing,
|
||||
scheduleNotificationResponseActionsService,
|
||||
isLoggedRequestsEnabled: Boolean(state?.isLoggedRequestsEnabled),
|
||||
});
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ export const queryExecutor = async ({
|
|||
bucketHistory,
|
||||
scheduleNotificationResponseActionsService,
|
||||
licensing,
|
||||
isLoggedRequestsEnabled,
|
||||
}: {
|
||||
runOpts: RunOpts<UnifiedQueryRuleParams>;
|
||||
experimentalFeatures: ExperimentalFeatures;
|
||||
|
|
@ -44,6 +45,7 @@ export const queryExecutor = async ({
|
|||
bucketHistory?: BucketHistory[];
|
||||
scheduleNotificationResponseActionsService: CreateRuleOptions['scheduleNotificationResponseActionsService'];
|
||||
licensing: LicensingPluginSetup;
|
||||
isLoggedRequestsEnabled: boolean;
|
||||
}) => {
|
||||
const completeRule = runOpts.completeRule;
|
||||
const ruleParams = completeRule.ruleParams;
|
||||
|
|
@ -77,6 +79,7 @@ export const queryExecutor = async ({
|
|||
groupByFields: ruleParams.alertSuppression.groupBy,
|
||||
eventsTelemetry,
|
||||
experimentalFeatures,
|
||||
isLoggedRequestsEnabled,
|
||||
})
|
||||
: {
|
||||
...(await searchAfterAndBulkCreate({
|
||||
|
|
@ -95,8 +98,9 @@ export const queryExecutor = async ({
|
|||
runtimeMappings: runOpts.runtimeMappings,
|
||||
primaryTimestamp: runOpts.primaryTimestamp,
|
||||
secondaryTimestamp: runOpts.secondaryTimestamp,
|
||||
isLoggedRequestsEnabled,
|
||||
})),
|
||||
state: {},
|
||||
state: { isLoggedRequestsEnabled },
|
||||
};
|
||||
|
||||
scheduleNotificationResponseActionsService({
|
||||
|
|
|
|||
|
|
@ -31,7 +31,9 @@ import type {
|
|||
} from './types';
|
||||
import { shouldFilterByCardinality, searchResultHasAggs } from './utils';
|
||||
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
|
||||
import { getMaxSignalsWarning } from '../utils/utils';
|
||||
import { getMaxSignalsWarning, stringifyAfterKey } from '../utils/utils';
|
||||
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
|
||||
import * as i18n from '../translations';
|
||||
|
||||
interface FindThresholdSignalsParams {
|
||||
from: string;
|
||||
|
|
@ -46,6 +48,7 @@ interface FindThresholdSignalsParams {
|
|||
primaryTimestamp: TimestampOverride;
|
||||
secondaryTimestamp: TimestampOverride | undefined;
|
||||
aggregatableTimestampField: string;
|
||||
isLoggedRequestsEnabled?: boolean;
|
||||
}
|
||||
|
||||
const hasThresholdFields = (threshold: ThresholdNormalized) => !!threshold.field.length;
|
||||
|
|
@ -55,6 +58,7 @@ interface SearchAfterResults {
|
|||
searchErrors: string[];
|
||||
}
|
||||
|
||||
// eslint-disable-next-line complexity
|
||||
export const findThresholdSignals = async ({
|
||||
from,
|
||||
to,
|
||||
|
|
@ -68,11 +72,13 @@ export const findThresholdSignals = async ({
|
|||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
aggregatableTimestampField,
|
||||
isLoggedRequestsEnabled,
|
||||
}: FindThresholdSignalsParams): Promise<{
|
||||
buckets: ThresholdBucket[];
|
||||
searchDurations: string[];
|
||||
searchErrors: string[];
|
||||
warnings: string[];
|
||||
loggedRequests?: RulePreviewLoggedRequest[];
|
||||
}> => {
|
||||
// Leaf aggregations used below
|
||||
const buckets: ThresholdBucket[] = [];
|
||||
|
|
@ -81,13 +87,19 @@ export const findThresholdSignals = async ({
|
|||
searchErrors: [],
|
||||
};
|
||||
const warnings: string[] = [];
|
||||
const loggedRequests: RulePreviewLoggedRequest[] = [];
|
||||
|
||||
const includeCardinalityFilter = shouldFilterByCardinality(threshold);
|
||||
|
||||
if (hasThresholdFields(threshold)) {
|
||||
let sortKeys: Record<string, string | number | null> | undefined;
|
||||
do {
|
||||
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
|
||||
const {
|
||||
searchResult,
|
||||
searchDuration,
|
||||
searchErrors,
|
||||
loggedRequests: thresholdLoggedRequests,
|
||||
} = await singleSearchAfter({
|
||||
aggregations: buildThresholdMultiBucketAggregation({
|
||||
threshold,
|
||||
aggregatableTimestampField,
|
||||
|
|
@ -105,9 +117,18 @@ export const findThresholdSignals = async ({
|
|||
runtimeMappings,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
loggedRequestsConfig: isLoggedRequestsEnabled
|
||||
? {
|
||||
type: 'findThresholdBuckets',
|
||||
description: i18n.FIND_THRESHOLD_BUCKETS_DESCRIPTION(stringifyAfterKey(sortKeys)),
|
||||
skipRequestQuery: loggedRequests.length > 2,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
|
||||
searchAfterResults.searchDurations.push(searchDuration);
|
||||
loggedRequests.push(...(thresholdLoggedRequests ?? []));
|
||||
|
||||
if (!isEmpty(searchErrors)) {
|
||||
searchAfterResults.searchErrors.push(...searchErrors);
|
||||
sortKeys = undefined; // this will eject us out of the loop
|
||||
|
|
@ -116,7 +137,6 @@ export const findThresholdSignals = async ({
|
|||
} else if (searchResultHasAggs<ThresholdMultiBucketAggregationResult>(searchResult)) {
|
||||
const thresholdTerms = searchResult.aggregations?.thresholdTerms;
|
||||
sortKeys = thresholdTerms?.after_key;
|
||||
|
||||
buckets.push(
|
||||
...((searchResult.aggregations?.thresholdTerms.buckets as ThresholdBucket[]) ?? [])
|
||||
);
|
||||
|
|
@ -125,7 +145,12 @@ export const findThresholdSignals = async ({
|
|||
}
|
||||
} while (sortKeys && buckets.length <= maxSignals);
|
||||
} else {
|
||||
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
|
||||
const {
|
||||
searchResult,
|
||||
searchDuration,
|
||||
searchErrors,
|
||||
loggedRequests: thresholdLoggedRequests,
|
||||
} = await singleSearchAfter({
|
||||
aggregations: buildThresholdSingleBucketAggregation({
|
||||
threshold,
|
||||
aggregatableTimestampField,
|
||||
|
|
@ -143,10 +168,17 @@ export const findThresholdSignals = async ({
|
|||
runtimeMappings,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
loggedRequestsConfig: isLoggedRequestsEnabled
|
||||
? {
|
||||
type: 'findThresholdBuckets',
|
||||
description: i18n.FIND_THRESHOLD_BUCKETS_DESCRIPTION(),
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
|
||||
searchAfterResults.searchDurations.push(searchDuration);
|
||||
searchAfterResults.searchErrors.push(...searchErrors);
|
||||
loggedRequests.push(...(thresholdLoggedRequests ?? []));
|
||||
|
||||
if (
|
||||
!searchResultHasAggs<ThresholdSingleBucketAggregationResult>(searchResult) &&
|
||||
|
|
@ -182,5 +214,6 @@ export const findThresholdSignals = async ({
|
|||
buckets: buckets.slice(0, maxSignals),
|
||||
...searchAfterResults,
|
||||
warnings,
|
||||
...(isLoggedRequestsEnabled ? { loggedRequests } : {}),
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -96,6 +96,7 @@ export const thresholdExecutor = async ({
|
|||
}): Promise<SearchAfterAndBulkCreateReturnType & { state: ThresholdAlertState }> => {
|
||||
const result = createSearchAfterReturnType();
|
||||
const ruleParams = completeRule.ruleParams;
|
||||
const isLoggedRequestsEnabled = Boolean(state?.isLoggedRequestsEnabled);
|
||||
|
||||
return withSecuritySpan('thresholdExecutor', async () => {
|
||||
const exceptionsWarning = getUnprocessedExceptionsWarnings(unprocessedExceptions);
|
||||
|
|
@ -140,20 +141,22 @@ export const thresholdExecutor = async ({
|
|||
});
|
||||
|
||||
// Look for new events over threshold
|
||||
const { buckets, searchErrors, searchDurations, warnings } = await findThresholdSignals({
|
||||
inputIndexPattern: inputIndex,
|
||||
from: tuple.from.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
maxSignals: tuple.maxSignals,
|
||||
services,
|
||||
ruleExecutionLogger,
|
||||
filter: esFilter,
|
||||
threshold: ruleParams.threshold,
|
||||
runtimeMappings,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
aggregatableTimestampField,
|
||||
});
|
||||
const { buckets, searchErrors, searchDurations, warnings, loggedRequests } =
|
||||
await findThresholdSignals({
|
||||
inputIndexPattern: inputIndex,
|
||||
from: tuple.from.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
maxSignals: tuple.maxSignals,
|
||||
services,
|
||||
ruleExecutionLogger,
|
||||
filter: esFilter,
|
||||
threshold: ruleParams.threshold,
|
||||
runtimeMappings,
|
||||
primaryTimestamp,
|
||||
secondaryTimestamp,
|
||||
aggregatableTimestampField,
|
||||
isLoggedRequestsEnabled,
|
||||
});
|
||||
|
||||
const alertSuppression = completeRule.ruleParams.alertSuppression;
|
||||
|
||||
|
|
@ -227,6 +230,7 @@ export const thresholdExecutor = async ({
|
|||
...newSignalHistory,
|
||||
},
|
||||
},
|
||||
...(isLoggedRequestsEnabled ? { loggedRequests } : {}),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -27,3 +27,90 @@ export const EQL_SEARCH_REQUEST_DESCRIPTION = i18n.translate(
|
|||
defaultMessage: 'EQL request to find all matches',
|
||||
}
|
||||
);
|
||||
|
||||
export const FIND_THRESHOLD_BUCKETS_DESCRIPTION = (afterBucket?: string) =>
|
||||
afterBucket
|
||||
? i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.esqlRuleType.findThresholdRuleBucketsAfterDescription',
|
||||
{
|
||||
defaultMessage: 'Find all terms that exceeds threshold value after {afterBucket}',
|
||||
values: { afterBucket },
|
||||
}
|
||||
)
|
||||
: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.esqlRuleType.findThresholdRuleBucketsDescription',
|
||||
{
|
||||
defaultMessage: 'Find all terms that exceeds threshold value',
|
||||
}
|
||||
);
|
||||
|
||||
export const ML_SEARCH_ANOMALIES_DESCRIPTION = i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.esqlRuleType.mlSearchAnomaliesRequestDescription',
|
||||
{
|
||||
defaultMessage: 'Find all anomalies',
|
||||
}
|
||||
);
|
||||
|
||||
export const FIND_EVENTS_DESCRIPTION = i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.queryRuleType.findEventsDescription',
|
||||
{
|
||||
defaultMessage: 'Find events',
|
||||
}
|
||||
);
|
||||
|
||||
export const FIND_EVENTS_AFTER_CURSOR_DESCRIPTION = (cursor?: string) =>
|
||||
i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.queryRuleType.findEventsAfterCursorDescription',
|
||||
{
|
||||
defaultMessage: 'Find events after cursor {cursor}',
|
||||
values: { cursor },
|
||||
}
|
||||
);
|
||||
|
||||
export const FIND_ALL_NEW_TERMS_FIELDS_DESCRIPTION = (afterKey?: string) =>
|
||||
afterKey
|
||||
? i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.newTermsRuleType.findAllNewTermsFieldsAfterDescription',
|
||||
{
|
||||
defaultMessage: 'Find all values after {afterKey}',
|
||||
values: { afterKey },
|
||||
}
|
||||
)
|
||||
: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.newTermsRuleType.findAllNewTermsFieldsDescription',
|
||||
{
|
||||
defaultMessage: 'Find all values',
|
||||
}
|
||||
);
|
||||
|
||||
export const FIND_NEW_TERMS_VALUES_DESCRIPTION = (afterKey?: string) =>
|
||||
afterKey
|
||||
? i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.newTermsRuleType.findNewTermsValuesAfterDescription',
|
||||
{
|
||||
defaultMessage: 'Find new values after {afterKey}',
|
||||
values: { afterKey },
|
||||
}
|
||||
)
|
||||
: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.newTermsRuleType.findNewTermsValuesDescription',
|
||||
{
|
||||
defaultMessage: 'Find new values',
|
||||
}
|
||||
);
|
||||
|
||||
export const FIND_NEW_TERMS_EVENTS_DESCRIPTION = (afterKey?: string) =>
|
||||
afterKey
|
||||
? i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.newTermsRuleType.findNewTermsEventsAfterDescription',
|
||||
{
|
||||
defaultMessage: 'Find documents associated with new values after {afterKey}',
|
||||
values: { afterKey },
|
||||
}
|
||||
)
|
||||
: i18n.translate(
|
||||
'xpack.securitySolution.detectionEngine.newTermsRuleType.findNewTermsEventsDescription',
|
||||
{
|
||||
defaultMessage: 'Find documents associated with new values',
|
||||
}
|
||||
);
|
||||
|
|
|
|||
|
|
@ -390,6 +390,7 @@ export interface SearchAfterAndBulkCreateParams {
|
|||
primaryTimestamp: string;
|
||||
secondaryTimestamp?: string;
|
||||
additionalFilters?: estypes.QueryDslQueryContainer[];
|
||||
isLoggedRequestsEnabled?: boolean;
|
||||
}
|
||||
|
||||
export interface SearchAfterAndBulkCreateReturnType {
|
||||
|
|
@ -405,10 +406,17 @@ export interface SearchAfterAndBulkCreateReturnType {
|
|||
userError?: boolean;
|
||||
warningMessages: string[];
|
||||
suppressedAlertsCount?: number;
|
||||
loggedRequests?: RulePreviewLoggedRequest[];
|
||||
}
|
||||
|
||||
export interface LoggedRequestsConfig {
|
||||
type: string;
|
||||
description: string;
|
||||
skipRequestQuery?: boolean;
|
||||
}
|
||||
|
||||
// the new fields can be added later if needed
|
||||
export interface OverrideBodyQuery {
|
||||
_source?: estypes.SearchSourceConfig;
|
||||
fields?: estypes.Fields;
|
||||
fields?: Array<estypes.QueryDslFieldAndFormat | estypes.Field>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,9 +23,9 @@ describe('create_signals', () => {
|
|||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
index: ['auditbeat-*'],
|
||||
size: 100,
|
||||
ignore_unavailable: true,
|
||||
body: {
|
||||
size: 100,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
|
|
@ -79,9 +79,9 @@ describe('create_signals', () => {
|
|||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
index: ['auditbeat-*'],
|
||||
size: 100,
|
||||
ignore_unavailable: true,
|
||||
body: {
|
||||
size: 100,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
|
|
@ -176,9 +176,9 @@ describe('create_signals', () => {
|
|||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
index: ['auditbeat-*'],
|
||||
size: 100,
|
||||
ignore_unavailable: true,
|
||||
body: {
|
||||
size: 100,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
|
|
@ -233,9 +233,9 @@ describe('create_signals', () => {
|
|||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
index: ['auditbeat-*'],
|
||||
size: 100,
|
||||
ignore_unavailable: true,
|
||||
body: {
|
||||
size: 100,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
|
|
@ -290,9 +290,9 @@ describe('create_signals', () => {
|
|||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
index: ['auditbeat-*'],
|
||||
size: 100,
|
||||
ignore_unavailable: true,
|
||||
body: {
|
||||
size: 100,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
|
|
@ -346,9 +346,9 @@ describe('create_signals', () => {
|
|||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
index: ['auditbeat-*'],
|
||||
size: 100,
|
||||
ignore_unavailable: true,
|
||||
body: {
|
||||
size: 100,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
|
|
@ -409,9 +409,9 @@ describe('create_signals', () => {
|
|||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
index: ['auditbeat-*'],
|
||||
size: 100,
|
||||
ignore_unavailable: true,
|
||||
body: {
|
||||
size: 100,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
|
|
@ -470,7 +470,7 @@ describe('create_signals', () => {
|
|||
trackTotalHits: false,
|
||||
runtimeMappings: undefined,
|
||||
});
|
||||
expect(query.track_total_hits).toEqual(false);
|
||||
expect(query.body?.track_total_hits).toEqual(false);
|
||||
});
|
||||
|
||||
test('if sortOrder is provided it should be included', () => {
|
||||
|
|
@ -487,12 +487,14 @@ describe('create_signals', () => {
|
|||
trackTotalHits: false,
|
||||
runtimeMappings: undefined,
|
||||
});
|
||||
expect(query.body.sort[0]).toEqual({
|
||||
'@timestamp': {
|
||||
order: 'desc',
|
||||
unmapped_type: 'date',
|
||||
expect(query?.body?.sort).toEqual([
|
||||
{
|
||||
'@timestamp': {
|
||||
order: 'desc',
|
||||
unmapped_type: 'date',
|
||||
},
|
||||
},
|
||||
});
|
||||
]);
|
||||
});
|
||||
|
||||
test('it respects sort order for timestampOverride', () => {
|
||||
|
|
@ -508,18 +510,20 @@ describe('create_signals', () => {
|
|||
sortOrder: 'desc',
|
||||
runtimeMappings: undefined,
|
||||
});
|
||||
expect(query.body.sort[0]).toEqual({
|
||||
'event.ingested': {
|
||||
order: 'desc',
|
||||
unmapped_type: 'date',
|
||||
expect(query?.body?.sort).toEqual([
|
||||
{
|
||||
'event.ingested': {
|
||||
order: 'desc',
|
||||
unmapped_type: 'date',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(query.body.sort[1]).toEqual({
|
||||
'@timestamp': {
|
||||
order: 'desc',
|
||||
unmapped_type: 'date',
|
||||
{
|
||||
'@timestamp': {
|
||||
order: 'desc',
|
||||
unmapped_type: 'date',
|
||||
},
|
||||
},
|
||||
});
|
||||
]);
|
||||
});
|
||||
|
||||
test('it respects overriderBody params', () => {
|
||||
|
|
@ -541,11 +545,11 @@ describe('create_signals', () => {
|
|||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
index: ['auditbeat-*'],
|
||||
size: 100,
|
||||
runtime_mappings: undefined,
|
||||
track_total_hits: undefined,
|
||||
ignore_unavailable: true,
|
||||
body: {
|
||||
size: 100,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ export const buildEventsSearchQuery = ({
|
|||
trackTotalHits,
|
||||
additionalFilters,
|
||||
overrideBody,
|
||||
}: BuildEventsSearchQuery) => {
|
||||
}: BuildEventsSearchQuery): estypes.SearchRequest => {
|
||||
const timestamps = secondaryTimestamp
|
||||
? [primaryTimestamp, secondaryTimestamp]
|
||||
: [primaryTimestamp];
|
||||
|
|
@ -152,14 +152,13 @@ export const buildEventsSearchQuery = ({
|
|||
});
|
||||
}
|
||||
|
||||
const searchQuery = {
|
||||
const searchQuery: estypes.SearchRequest = {
|
||||
allow_no_indices: true,
|
||||
runtime_mappings: runtimeMappings,
|
||||
index,
|
||||
size,
|
||||
ignore_unavailable: true,
|
||||
track_total_hits: trackTotalHits,
|
||||
body: {
|
||||
track_total_hits: trackTotalHits,
|
||||
size,
|
||||
query: {
|
||||
bool: {
|
||||
filter: filterWithTime,
|
||||
|
|
|
|||
|
|
@ -8,3 +8,4 @@
|
|||
export * from './log_esql';
|
||||
export * from './log_eql';
|
||||
export * from './log_query';
|
||||
export * from './log_search_request';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,181 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { logSearchRequest } from './log_search_request';
|
||||
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
|
||||
describe('logSearchRequest', () => {
|
||||
it('should match inline snapshot when deprecated search request used', () => {
|
||||
const searchRequest = {
|
||||
allow_no_indices: true,
|
||||
index: ['close_alerts*'],
|
||||
ignore_unavailable: true,
|
||||
body: {
|
||||
size: 0,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
bool: {
|
||||
must: [],
|
||||
filter: [{ query_string: { query: '*' } }, { bool: { must_not: [] } }],
|
||||
should: [],
|
||||
must_not: [],
|
||||
},
|
||||
},
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
lte: '2024-12-09T17:26:48.786Z',
|
||||
gte: '2013-07-14T00:26:48.786Z',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
fields: [
|
||||
{ field: '*', include_unmapped: true },
|
||||
{ field: '@timestamp', format: 'strict_date_optional_time' },
|
||||
],
|
||||
aggregations: {
|
||||
thresholdTerms: {
|
||||
composite: {
|
||||
sources: [
|
||||
{ 'agent.name': { terms: { field: 'agent.name' } } },
|
||||
{ 'destination.ip': { terms: { field: 'destination.ip' } } },
|
||||
],
|
||||
after: { 'agent.name': 'test-6', 'destination.ip': '127.0.0.1' },
|
||||
size: 10000,
|
||||
},
|
||||
aggs: {
|
||||
max_timestamp: { max: { field: '@timestamp' } },
|
||||
min_timestamp: { min: { field: '@timestamp' } },
|
||||
count_check: {
|
||||
bucket_selector: {
|
||||
buckets_path: { docCount: '_count' },
|
||||
script: 'params.docCount >= 1',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
runtime_mappings: {},
|
||||
sort: [{ '@timestamp': { order: 'desc', unmapped_type: 'date' } }],
|
||||
},
|
||||
};
|
||||
|
||||
expect(logSearchRequest(searchRequest as unknown as estypes.SearchRequest))
|
||||
.toMatchInlineSnapshot(`
|
||||
"POST /close_alerts*/_search?allow_no_indices=true&ignore_unavailable=true
|
||||
{
|
||||
\\"size\\": 0,
|
||||
\\"query\\": {
|
||||
\\"bool\\": {
|
||||
\\"filter\\": [
|
||||
{
|
||||
\\"bool\\": {
|
||||
\\"must\\": [],
|
||||
\\"filter\\": [
|
||||
{
|
||||
\\"query_string\\": {
|
||||
\\"query\\": \\"*\\"
|
||||
}
|
||||
},
|
||||
{
|
||||
\\"bool\\": {
|
||||
\\"must_not\\": []
|
||||
}
|
||||
}
|
||||
],
|
||||
\\"should\\": [],
|
||||
\\"must_not\\": []
|
||||
}
|
||||
},
|
||||
{
|
||||
\\"range\\": {
|
||||
\\"@timestamp\\": {
|
||||
\\"lte\\": \\"2024-12-09T17:26:48.786Z\\",
|
||||
\\"gte\\": \\"2013-07-14T00:26:48.786Z\\",
|
||||
\\"format\\": \\"strict_date_optional_time\\"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
\\"fields\\": [
|
||||
{
|
||||
\\"field\\": \\"*\\",
|
||||
\\"include_unmapped\\": true
|
||||
},
|
||||
{
|
||||
\\"field\\": \\"@timestamp\\",
|
||||
\\"format\\": \\"strict_date_optional_time\\"
|
||||
}
|
||||
],
|
||||
\\"aggregations\\": {
|
||||
\\"thresholdTerms\\": {
|
||||
\\"composite\\": {
|
||||
\\"sources\\": [
|
||||
{
|
||||
\\"agent.name\\": {
|
||||
\\"terms\\": {
|
||||
\\"field\\": \\"agent.name\\"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
\\"destination.ip\\": {
|
||||
\\"terms\\": {
|
||||
\\"field\\": \\"destination.ip\\"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
\\"after\\": {
|
||||
\\"agent.name\\": \\"test-6\\",
|
||||
\\"destination.ip\\": \\"127.0.0.1\\"
|
||||
},
|
||||
\\"size\\": 10000
|
||||
},
|
||||
\\"aggs\\": {
|
||||
\\"max_timestamp\\": {
|
||||
\\"max\\": {
|
||||
\\"field\\": \\"@timestamp\\"
|
||||
}
|
||||
},
|
||||
\\"min_timestamp\\": {
|
||||
\\"min\\": {
|
||||
\\"field\\": \\"@timestamp\\"
|
||||
}
|
||||
},
|
||||
\\"count_check\\": {
|
||||
\\"bucket_selector\\": {
|
||||
\\"buckets_path\\": {
|
||||
\\"docCount\\": \\"_count\\"
|
||||
},
|
||||
\\"script\\": \\"params.docCount >= 1\\"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
\\"runtime_mappings\\": {},
|
||||
\\"sort\\": [
|
||||
{
|
||||
\\"@timestamp\\": {
|
||||
\\"order\\": \\"desc\\",
|
||||
\\"unmapped_type\\": \\"date\\"
|
||||
}
|
||||
}
|
||||
]
|
||||
}"
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
|
||||
export const logSearchRequest = (searchRequest: estypes.SearchRequest): string => {
|
||||
const { body, index, ...params } = searchRequest;
|
||||
const urlParams = Object.entries(params)
|
||||
.reduce<string[]>((acc, [key, value]) => {
|
||||
if (value != null) {
|
||||
acc.push(`${key}=${value}`);
|
||||
}
|
||||
|
||||
return acc;
|
||||
}, [])
|
||||
.join('&');
|
||||
|
||||
const url = `/${index}/_search${urlParams ? `?${urlParams}` : ''}`;
|
||||
|
||||
if (body) {
|
||||
return `POST ${url}\n${JSON.stringify({ ...body }, null, 2)}`;
|
||||
}
|
||||
|
||||
return `GET ${url}`;
|
||||
};
|
||||
|
|
@ -23,11 +23,33 @@ import type {
|
|||
SearchAfterAndBulkCreateParams,
|
||||
SearchAfterAndBulkCreateReturnType,
|
||||
SignalSourceHit,
|
||||
LoggedRequestsConfig,
|
||||
} from '../types';
|
||||
import { withSecuritySpan } from '../../../../utils/with_security_span';
|
||||
import type { GenericBulkCreateResponse } from '../factories';
|
||||
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
|
||||
|
||||
import type { BaseFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts';
|
||||
import * as i18n from '../translations';
|
||||
|
||||
const createLoggedRequestsConfig = (
|
||||
isLoggedRequestsEnabled: boolean | undefined,
|
||||
sortIds: estypes.SortResults | undefined,
|
||||
page: number
|
||||
): LoggedRequestsConfig | undefined => {
|
||||
if (!isLoggedRequestsEnabled) {
|
||||
return undefined;
|
||||
}
|
||||
const description = sortIds
|
||||
? i18n.FIND_EVENTS_AFTER_CURSOR_DESCRIPTION(JSON.stringify(sortIds))
|
||||
: i18n.FIND_EVENTS_DESCRIPTION;
|
||||
|
||||
return {
|
||||
type: 'findDocuments',
|
||||
description,
|
||||
skipRequestQuery: page > 2, // skipping query logging for performance reasons, so we won't overwhelm Kibana with large response size
|
||||
};
|
||||
};
|
||||
|
||||
export interface SearchAfterAndBulkCreateFactoryParams extends SearchAfterAndBulkCreateParams {
|
||||
bulkCreateExecutor: (params: {
|
||||
|
|
@ -56,10 +78,13 @@ export const searchAfterAndBulkCreateFactory = async ({
|
|||
additionalFilters,
|
||||
bulkCreateExecutor,
|
||||
getWarningMessage,
|
||||
isLoggedRequestsEnabled,
|
||||
}: SearchAfterAndBulkCreateFactoryParams): Promise<SearchAfterAndBulkCreateReturnType> => {
|
||||
// eslint-disable-next-line complexity
|
||||
return withSecuritySpan('searchAfterAndBulkCreate', async () => {
|
||||
let toReturn = createSearchAfterReturnType();
|
||||
let searchingIteration = 0;
|
||||
const loggedRequests: RulePreviewLoggedRequest[] = [];
|
||||
|
||||
// sortId tells us where to start our next consecutive search_after query
|
||||
let sortIds: estypes.SortResults | undefined;
|
||||
|
|
@ -88,7 +113,12 @@ export const searchAfterAndBulkCreateFactory = async ({
|
|||
);
|
||||
|
||||
if (hasSortId) {
|
||||
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
|
||||
const {
|
||||
searchResult,
|
||||
searchDuration,
|
||||
searchErrors,
|
||||
loggedRequests: singleSearchLoggedRequests = [],
|
||||
} = await singleSearchAfter({
|
||||
searchAfterSortIds: sortIds,
|
||||
index: inputIndexPattern,
|
||||
runtimeMappings,
|
||||
|
|
@ -103,6 +133,11 @@ export const searchAfterAndBulkCreateFactory = async ({
|
|||
trackTotalHits,
|
||||
sortOrder,
|
||||
additionalFilters,
|
||||
loggedRequestsConfig: createLoggedRequestsConfig(
|
||||
isLoggedRequestsEnabled,
|
||||
sortIds,
|
||||
searchingIteration
|
||||
),
|
||||
});
|
||||
mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]);
|
||||
toReturn = mergeReturns([
|
||||
|
|
@ -116,7 +151,7 @@ export const searchAfterAndBulkCreateFactory = async ({
|
|||
errors: searchErrors,
|
||||
}),
|
||||
]);
|
||||
|
||||
loggedRequests.push(...singleSearchLoggedRequests);
|
||||
// determine if there are any candidate signals to be processed
|
||||
const totalHits = getTotalHitsValue(mergedSearchResults.hits.total);
|
||||
const lastSortIds = getSafeSortIds(
|
||||
|
|
@ -211,6 +246,11 @@ export const searchAfterAndBulkCreateFactory = async ({
|
|||
}
|
||||
}
|
||||
ruleExecutionLogger.debug(`Completed bulk indexing of ${toReturn.createdSignalsCount} alert`);
|
||||
|
||||
if (isLoggedRequestsEnabled) {
|
||||
toReturn.loggedRequests = loggedRequests;
|
||||
}
|
||||
|
||||
return toReturn;
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -11,12 +11,19 @@ import type {
|
|||
AlertInstanceState,
|
||||
RuleExecutorServices,
|
||||
} from '@kbn/alerting-plugin/server';
|
||||
import type { SignalSearchResponse, SignalSource, OverrideBodyQuery } from '../types';
|
||||
import type {
|
||||
SignalSearchResponse,
|
||||
SignalSource,
|
||||
OverrideBodyQuery,
|
||||
LoggedRequestsConfig,
|
||||
} from '../types';
|
||||
import { buildEventsSearchQuery } from './build_events_query';
|
||||
import { createErrorsFromShard, makeFloatString } from './utils';
|
||||
import type { TimestampOverride } from '../../../../../common/api/detection_engine/model/rule_schema';
|
||||
import { withSecuritySpan } from '../../../../utils/with_security_span';
|
||||
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
|
||||
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
|
||||
import { logSearchRequest } from './logged_requests';
|
||||
|
||||
export interface SingleSearchAfterParams {
|
||||
aggregations?: Record<string, estypes.AggregationsAggregationContainer>;
|
||||
|
|
@ -35,6 +42,7 @@ export interface SingleSearchAfterParams {
|
|||
runtimeMappings: estypes.MappingRuntimeFields | undefined;
|
||||
additionalFilters?: estypes.QueryDslQueryContainer[];
|
||||
overrideBody?: OverrideBodyQuery;
|
||||
loggedRequestsConfig?: LoggedRequestsConfig;
|
||||
}
|
||||
|
||||
// utilize search_after for paging results into bulk.
|
||||
|
|
@ -57,12 +65,16 @@ export const singleSearchAfter = async <
|
|||
trackTotalHits,
|
||||
additionalFilters,
|
||||
overrideBody,
|
||||
loggedRequestsConfig,
|
||||
}: SingleSearchAfterParams): Promise<{
|
||||
searchResult: SignalSearchResponse<TAggregations>;
|
||||
searchDuration: string;
|
||||
searchErrors: string[];
|
||||
loggedRequests?: RulePreviewLoggedRequest[];
|
||||
}> => {
|
||||
return withSecuritySpan('singleSearchAfter', async () => {
|
||||
const loggedRequests: RulePreviewLoggedRequest[] = [];
|
||||
|
||||
try {
|
||||
const searchAfterQuery = buildEventsSearchQuery({
|
||||
aggregations,
|
||||
|
|
@ -88,7 +100,7 @@ export const singleSearchAfter = async <
|
|||
const start = performance.now();
|
||||
const { body: nextSearchAfterResult } =
|
||||
await services.scopedClusterClient.asCurrentUser.search<SignalSource, TAggregations>(
|
||||
searchAfterQuery as estypes.SearchRequest,
|
||||
searchAfterQuery,
|
||||
{ meta: true }
|
||||
);
|
||||
|
||||
|
|
@ -98,10 +110,22 @@ export const singleSearchAfter = async <
|
|||
errors: nextSearchAfterResult._shards.failures ?? [],
|
||||
});
|
||||
|
||||
if (loggedRequestsConfig) {
|
||||
loggedRequests.push({
|
||||
request: loggedRequestsConfig.skipRequestQuery
|
||||
? undefined
|
||||
: logSearchRequest(searchAfterQuery),
|
||||
description: loggedRequestsConfig.description,
|
||||
request_type: loggedRequestsConfig.type,
|
||||
duration: Math.round(end - start),
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
searchResult: nextSearchAfterResult,
|
||||
searchDuration: makeFloatString(end - start),
|
||||
searchErrors,
|
||||
loggedRequests,
|
||||
};
|
||||
} catch (exc) {
|
||||
ruleExecutionLogger.error(`Searching events operation failed: ${exc}`);
|
||||
|
|
@ -129,6 +153,7 @@ export const singleSearchAfter = async <
|
|||
searchResult: searchRes,
|
||||
searchDuration: '-1.0',
|
||||
searchErrors: exc.message,
|
||||
loggedRequests,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ import {
|
|||
getUnprocessedExceptionsWarnings,
|
||||
getDisabledActionsWarningText,
|
||||
calculateFromValue,
|
||||
stringifyAfterKey,
|
||||
} from './utils';
|
||||
import type { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from '../types';
|
||||
import {
|
||||
|
|
@ -1757,6 +1758,20 @@ describe('utils', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('stringifyAfterKey', () => {
|
||||
it('should stringify after_key object with single key value', () => {
|
||||
expect(stringifyAfterKey({ 'agent.name': 'test' })).toBe('agent.name: test');
|
||||
});
|
||||
it('should stringify after_key object with multiple key values', () => {
|
||||
expect(stringifyAfterKey({ 'agent.name': 'test', 'destination.ip': '127.0.0.1' })).toBe(
|
||||
'agent.name: test, destination.ip: 127.0.0.1'
|
||||
);
|
||||
});
|
||||
it('should return undefined if after_key is undefined', () => {
|
||||
expect(stringifyAfterKey(undefined)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getDisabledActionsWarningText', () => {
|
||||
const alertsCreated = true;
|
||||
const alertsNotCreated = false;
|
||||
|
|
|
|||
|
|
@ -1103,6 +1103,20 @@ export type SequenceSuppressionTermsAndFieldsFactory = (
|
|||
subAlerts: Array<WrappedFieldsLatest<EqlBuildingBlockFieldsLatest>>;
|
||||
};
|
||||
|
||||
/**
|
||||
* converts ES after_key object into string
|
||||
* for example: { "agent.name": "test" } would become `agent.name: test`
|
||||
*/
|
||||
export const stringifyAfterKey = (afterKey: Record<string, string | number | null> | undefined) => {
|
||||
if (!afterKey) {
|
||||
return;
|
||||
}
|
||||
|
||||
return Object.entries(afterKey)
|
||||
.map((entry) => entry.join(': '))
|
||||
.join(', ');
|
||||
};
|
||||
|
||||
export const buildShellAlertSuppressionTermsAndFields = ({
|
||||
shellAlert,
|
||||
buildingBlockAlerts,
|
||||
|
|
|
|||
|
|
@ -30,41 +30,43 @@ export const getAnomalies = async (
|
|||
params: AnomaliesSearchParams,
|
||||
mlAnomalySearch: MlAnomalySearch
|
||||
): Promise<AnomalyResults> => {
|
||||
const queryRequest = buildAnomalyQuery(params);
|
||||
return mlAnomalySearch(queryRequest, params.jobIds);
|
||||
};
|
||||
|
||||
export const buildAnomalyQuery = (params: AnomaliesSearchParams): estypes.SearchRequest => {
|
||||
const boolCriteria = buildCriteria(params);
|
||||
return mlAnomalySearch(
|
||||
{
|
||||
body: {
|
||||
size: params.maxRecords || 100,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
query_string: {
|
||||
query: 'result_type:record',
|
||||
analyze_wildcard: false,
|
||||
},
|
||||
return {
|
||||
body: {
|
||||
size: params.maxRecords || 100,
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
query_string: {
|
||||
query: 'result_type:record',
|
||||
analyze_wildcard: false,
|
||||
},
|
||||
{ term: { is_interim: false } },
|
||||
{
|
||||
bool: {
|
||||
must: boolCriteria,
|
||||
},
|
||||
},
|
||||
{ term: { is_interim: false } },
|
||||
{
|
||||
bool: {
|
||||
must: boolCriteria,
|
||||
},
|
||||
],
|
||||
must_not: params.exceptionFilter?.query,
|
||||
},
|
||||
},
|
||||
],
|
||||
must_not: params.exceptionFilter?.query,
|
||||
},
|
||||
fields: [
|
||||
{
|
||||
field: '*',
|
||||
include_unmapped: true,
|
||||
},
|
||||
],
|
||||
sort: [{ record_score: { order: 'desc' as const } }],
|
||||
},
|
||||
fields: [
|
||||
{
|
||||
field: '*',
|
||||
include_unmapped: true,
|
||||
},
|
||||
],
|
||||
sort: [{ record_score: { order: 'desc' as const } }],
|
||||
},
|
||||
params.jobIds
|
||||
);
|
||||
};
|
||||
};
|
||||
|
||||
const buildCriteria = (params: AnomaliesSearchParams): object[] => {
|
||||
|
|
|
|||
|
|
@ -345,5 +345,29 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('preview logged requests', () => {
|
||||
it('should not return requests property when not enabled', async () => {
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule,
|
||||
});
|
||||
|
||||
expect(logs[0].requests).toEqual(undefined);
|
||||
});
|
||||
it('should return requests property when enable_logged_requests set to true', async () => {
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule,
|
||||
enableLoggedRequests: true,
|
||||
});
|
||||
|
||||
const requests = logs[0].requests;
|
||||
|
||||
expect(requests).toHaveLength(1);
|
||||
expect(requests![0].description).toBe('Find all anomalies');
|
||||
expect(requests![0].request).toContain('POST /.ml-anomalies-*/_search');
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1364,5 +1364,117 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('preview logged requests', () => {
|
||||
const rule: NewTermsRuleCreateProps = {
|
||||
...getCreateNewTermsRulesSchemaMock('rule-1', true),
|
||||
index: ['new_terms'],
|
||||
new_terms_fields: ['host.name', 'host.ip'],
|
||||
from: ruleExecutionStart,
|
||||
history_window_start: historicalWindowStart,
|
||||
query: '*',
|
||||
};
|
||||
|
||||
it('should not return requests property when not enabled', async () => {
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule,
|
||||
});
|
||||
|
||||
expect(logs[0].requests).toEqual(undefined);
|
||||
});
|
||||
|
||||
it('should return requests property when enable_logged_requests set to true for single new term field', async () => {
|
||||
// historical window documents
|
||||
const historicalDocuments = [
|
||||
{
|
||||
host: { name: 'host-0', ip: '127.0.0.1' },
|
||||
},
|
||||
{
|
||||
host: { name: 'host-1', ip: '127.0.0.2' },
|
||||
},
|
||||
];
|
||||
|
||||
// rule execution documents
|
||||
const ruleExecutionDocuments = [
|
||||
{
|
||||
host: { name: 'host-0', ip: '127.0.0.2' },
|
||||
},
|
||||
{
|
||||
host: { name: 'host-2', ip: '127.0.0.1' },
|
||||
},
|
||||
];
|
||||
|
||||
const testId = await newTermsTestExecutionSetup({
|
||||
historicalDocuments,
|
||||
ruleExecutionDocuments,
|
||||
});
|
||||
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule: { ...rule, query: `id: "${testId}"`, new_terms_fields: ['host.name'] },
|
||||
enableLoggedRequests: true,
|
||||
});
|
||||
|
||||
expect(logs[0].requests?.length).toEqual(4);
|
||||
const requests = logs[0].requests ?? [];
|
||||
|
||||
expect(requests[0].description).toBe('Find all values');
|
||||
expect(requests[0].request_type).toBe('findAllTerms');
|
||||
|
||||
expect(requests[1].description).toBe('Find new values');
|
||||
expect(requests[1].request_type).toBe('findNewTerms');
|
||||
|
||||
expect(requests[2].description).toBe('Find documents associated with new values');
|
||||
expect(requests[2].request_type).toBe('findDocuments');
|
||||
|
||||
expect(requests[3].description).toBe('Find all values after host.name: host-2');
|
||||
});
|
||||
|
||||
it('should return requests property when enable_logged_requests set to true for multiple fields', async () => {
|
||||
// historical window documents
|
||||
const historicalDocuments = [
|
||||
{
|
||||
host: { name: 'host-0', ip: '127.0.0.1' },
|
||||
},
|
||||
{
|
||||
host: { name: 'host-1', ip: '127.0.0.2' },
|
||||
},
|
||||
];
|
||||
|
||||
// rule execution documents
|
||||
const ruleExecutionDocuments = [
|
||||
{
|
||||
host: { name: 'host-0', ip: '127.0.0.2' },
|
||||
},
|
||||
{
|
||||
host: { name: 'host-1', ip: '127.0.0.1' },
|
||||
},
|
||||
];
|
||||
|
||||
const testId = await newTermsTestExecutionSetup({
|
||||
historicalDocuments,
|
||||
ruleExecutionDocuments,
|
||||
});
|
||||
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule: { ...rule, query: `id: "${testId}"` },
|
||||
enableLoggedRequests: true,
|
||||
});
|
||||
|
||||
expect(logs[0].requests?.length).toEqual(4);
|
||||
const requests = logs[0].requests ?? [];
|
||||
|
||||
expect(requests[0].description).toBe('Find all values');
|
||||
expect(requests[0].request_type).toBe('findAllTerms');
|
||||
|
||||
expect(requests[1].description).toBe('Find new values');
|
||||
expect(requests[1].request_type).toBe('findNewTerms');
|
||||
|
||||
expect(requests[2].description).toBe('Find documents associated with new values');
|
||||
expect(requests[2].request_type).toBe('findDocuments');
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2833,5 +2833,76 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
expect(get(previewAlerts[0]?._source, 'event.dataset')).toEqual('network_traffic.tls');
|
||||
});
|
||||
});
|
||||
|
||||
describe('preview logged requests', () => {
|
||||
before(async () => {
|
||||
await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ecs_compliant');
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await esArchiver.unload(
|
||||
'x-pack/test/functional/es_archives/security_solution/ecs_compliant'
|
||||
);
|
||||
});
|
||||
|
||||
const rule: QueryRuleCreateProps = getRuleForAlertTesting(['ecs_compliant']);
|
||||
const ruleWithSuppression: QueryRuleCreateProps = {
|
||||
...rule,
|
||||
alert_suppression: {
|
||||
group_by: ['agent.name'],
|
||||
duration: {
|
||||
value: 500,
|
||||
unit: 'm',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
it('should not return requests property when not enabled', async () => {
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule,
|
||||
});
|
||||
|
||||
expect(logs[0].requests).toEqual(undefined);
|
||||
});
|
||||
it('should return requests property when enable_logged_requests set to true', async () => {
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule,
|
||||
enableLoggedRequests: true,
|
||||
});
|
||||
|
||||
const requests = logs[0].requests;
|
||||
|
||||
expect(requests).toHaveLength(1);
|
||||
expect(requests![0].description).toBe('Find events');
|
||||
expect(requests![0].request_type).toBe('findDocuments');
|
||||
expect(requests![0].request).toContain('POST /ecs_compliant/_search?allow_no_indices=true');
|
||||
});
|
||||
|
||||
it('should not return requests property when not enabled and suppression configured', async () => {
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule: ruleWithSuppression,
|
||||
});
|
||||
|
||||
expect(logs[0].requests).toEqual(undefined);
|
||||
});
|
||||
|
||||
it('should return requests property when enable_logged_requests set to true and suppression configured', async () => {
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule: ruleWithSuppression,
|
||||
enableLoggedRequests: true,
|
||||
});
|
||||
|
||||
const requests = logs[0].requests;
|
||||
|
||||
expect(requests).toHaveLength(1);
|
||||
expect(requests![0].description).toBe('Find events');
|
||||
expect(requests![0].request_type).toBe('findDocuments');
|
||||
expect(requests![0].request).toContain('POST /ecs_compliant/_search?allow_no_indices=true');
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -688,5 +688,46 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('preview logged requests', () => {
|
||||
const rule: ThresholdRuleCreateProps = {
|
||||
...getThresholdRuleForAlertTesting(['auditbeat-*']),
|
||||
threshold: {
|
||||
field: 'host.id',
|
||||
value: 100,
|
||||
},
|
||||
};
|
||||
|
||||
it('should not return requests property when not enabled', async () => {
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule,
|
||||
});
|
||||
|
||||
expect(logs[0].requests).toEqual(undefined);
|
||||
});
|
||||
it('should return requests property when enable_logged_requests set to true', async () => {
|
||||
const { logs } = await previewRule({
|
||||
supertest,
|
||||
rule,
|
||||
enableLoggedRequests: true,
|
||||
});
|
||||
|
||||
const requests = logs[0].requests;
|
||||
|
||||
expect(requests).toHaveLength(2);
|
||||
expect(requests![0].description).toBe('Find all terms that exceeds threshold value');
|
||||
|
||||
const requestWithoutSpaces = requests![0].request?.replace(/\s/g, '');
|
||||
expect(requestWithoutSpaces).toContain(
|
||||
`aggregations":{"thresholdTerms":{"composite":{"sources":[{"host.id":{"terms":{"field":"host.id"}}}],"size":10000},"aggs":{"max_timestamp":{"max":{"field":"@timestamp"}},"min_timestamp":{"min":{"field":"@timestamp"}},"count_check":{"bucket_selector":{"buckets_path":{"docCount":"_count"},"script":"params.docCount>=100"}}}}`
|
||||
);
|
||||
expect(requests![0].request).toContain('POST /auditbeat-*/_search?allow_no_indices=true');
|
||||
expect(requests![0].request_type).toBe('findThresholdBuckets');
|
||||
expect(requests![1].description).toBe(
|
||||
'Find all terms that exceeds threshold value after host.id: f9c7ca2d33f548a8b37667f6fffc59ce'
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { getEsqlRule, getSimpleCustomQueryRule } from '../../../../objects/rule';
|
||||
import { getEsqlRule, getNewThreatIndicatorRule } from '../../../../objects/rule';
|
||||
|
||||
import {
|
||||
PREVIEW_LOGGED_REQUEST_DESCRIPTION,
|
||||
|
|
@ -64,12 +64,12 @@ describe(
|
|||
|
||||
describe('does not support preview logged requests', () => {
|
||||
beforeEach(() => {
|
||||
createRule(getSimpleCustomQueryRule()).then((createdRule) => {
|
||||
createRule(getNewThreatIndicatorRule()).then((createdRule) => {
|
||||
visitEditRulePage(createdRule.body.id);
|
||||
});
|
||||
});
|
||||
|
||||
it('does not show preview logged requests checkbox', () => {
|
||||
it('does not show preview logged requests checkbox fro Indicator Match rule', () => {
|
||||
cy.get(RULES_CREATION_PREVIEW_REFRESH_BUTTON).should('be.visible');
|
||||
cy.get(PREVIEW_LOGGED_REQUESTS_CHECKBOX).should('not.exist');
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue