github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[SecuritySolution] Histogram IP legends error fixed (#99468)

Browse source 
* make sure stackByField exists

* fix types

* fix unit test

* skip extra request for non-ip queries

* elasticserach query changes to prevent corrupted data response bug

* client changes to split ip stacked histogram queries in two, inspect modal shows all requests and responses

* lint fixes

* test for useMatrixHistogramCombined added

* comment added on new multiple prop

* changed query to always contain value_type:ip for ip queries

Co-authored-by: Angela Chuang <yi-chun.chuang@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Sergi Massaneda 2021-05-10 17:54:55 +02:00 • committed by GitHub
parent 0ffe4c7a54
commit 518da5bcc1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 510 additions and 131 deletions
Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts
View file

@ -50,6 +50,7 @@ export interface MatrixHistogramRequestOptions extends RequestBasicOptions {
| undefined;
inspect?: Maybe<Inspect>;
isPtrIncluded?: boolean;
includeMissingData?: boolean;
}
export interface MatrixHistogramStrategyResponse extends IEsSearchResponse {

4
x-pack/plugins/security_solution/public/common/components/header_section/index.tsx
View file

@ -47,6 +47,7 @@ export interface HeaderSectionProps extends HeaderProps {
titleSize?: EuiTitleSize;
tooltip?: string;
growLeftSplit?: boolean;
inspectMultiple?: boolean;
}
const HeaderSectionComponent: React.FC<HeaderSectionProps> = ({
@ -60,6 +61,7 @@ const HeaderSectionComponent: React.FC<HeaderSectionProps> = ({
titleSize = 'm',
tooltip,
growLeftSplit = true,
inspectMultiple = false,
}) => (
<Header data-test-subj="header-section" border={border} height={height}>
<EuiFlexGroup alignItems="center">
@ -83,7 +85,7 @@ const HeaderSectionComponent: React.FC<HeaderSectionProps> = ({
{id && (
<EuiFlexItem grow={false}>
<InspectButton queryId={id} inspectIndex={0} title={title} />
<InspectButton queryId={id} multiple={inspectMultiple} title={title} />
</EuiFlexItem>
)}
</EuiFlexGroup>

30
x-pack/plugins/security_solution/public/common/components/inspect/index.tsx
View file

@ -59,6 +59,7 @@ interface OwnProps {
isDisabled?: boolean;
onCloseInspect?: () => void;
title: string | React.ReactElement | React.ReactNode;
multiple?: boolean;
}
type InspectButtonProps = OwnProps & PropsFromRedux;
@ -71,6 +72,7 @@ const InspectButtonComponent: React.FC<InspectButtonProps> = ({
isInspected,
loading,
inspectIndex = 0,
multiple = false, // If multiple = true we ignore the inspectIndex and pass all requests and responses to the inspect modal
onCloseInspect,
queryId = '',
selectedInspectIndex,
@ -99,6 +101,26 @@ const InspectButtonComponent: React.FC<InspectButtonProps> = ({
});
}, [onCloseInspect, setIsInspected, queryId, inputId, inspectIndex]);
let request: string | null = null;
let additionalRequests: string[] | null = null;
if (inspect != null && inspect.dsl.length > 0) {
if (multiple) {
[request, ...additionalRequests] = inspect.dsl;
} else {
request = inspect.dsl[inspectIndex];
}
}
let response: string | null = null;
let additionalResponses: string[] | null = null;
if (inspect != null && inspect.response.length > 0) {
if (multiple) {
[response, ...additionalResponses] = inspect.response;
} else {
response = inspect.response[inspectIndex];
}
}
return (
<>
{inputId === 'timeline' && !compact && (
@ -131,10 +153,10 @@ const InspectButtonComponent: React.FC<InspectButtonProps> = ({
<ModalInspectQuery
closeModal={handleCloseModal}
isShowing={isShowingModal}
request={inspect != null && inspect.dsl.length > 0 ? inspect.dsl[inspectIndex] : null}
response={
inspect != null && inspect.response.length > 0 ? inspect.response[inspectIndex] : null
}
request={request}
response={response}
additionalRequests={additionalRequests}
additionalResponses={additionalResponses}
title={title}
data-test-subj="inspect-modal"
/>

102
x-pack/plugins/security_solution/public/common/components/inspect/modal.tsx
View file

@ -19,7 +19,7 @@ import {
EuiTabbedContent,
} from '@elastic/eui';
import numeral from '@elastic/numeral';
import React, { ReactNode } from 'react';
import React, { Fragment, ReactNode } from 'react';
import styled from 'styled-components';
import { NO_ALERT_INDEX } from '../../../../common/constants';
@ -44,6 +44,8 @@ interface ModalInspectProps {
isShowing: boolean;
request: string | null;
response: string | null;
additionalRequests?: string[] | null;
additionalResponses?: string[] | null;
title: string | React.ReactElement | React.ReactNode;
}
@ -73,11 +75,11 @@ const MyEuiModal = styled(EuiModal)`
`;
MyEuiModal.displayName = 'MyEuiModal';
const parseInspectString = function <T>(objectStringify: string): T | null {
const parseInspectStrings = function <T>(stringsArray: string[]): T[] {
try {
return JSON.parse(objectStringify);
return stringsArray.map((objectStringify) => JSON.parse(objectStringify));
} catch {
return null;
return [];
}
};
@ -103,13 +105,23 @@ export const ModalInspectQuery = ({
isShowing = false,
request,
response,
additionalRequests,
additionalResponses,
title,
}: ModalInspectProps) => {
if (!isShowing || request == null || response == null) {
return null;
}
const inspectRequest: Request | null = parseInspectString(request);
const inspectResponse: Response | null = parseInspectString(response);
const requests: string[] = [request, ...(additionalRequests != null ? additionalRequests : [])];
const responses: string[] = [
response,
...(additionalResponses != null ? additionalResponses : []),
];
const inspectRequests: Request[] = parseInspectStrings(requests);
const inspectResponses: Response[] = parseInspectStrings(responses);
const statistics: Array<{
title: NonNullable<ReactNode | string>;
description: NonNullable<ReactNode | string>;
@ -123,7 +135,7 @@ export const ModalInspectQuery = ({
),
description: (
<span data-test-subj="index-pattern-description">
{formatIndexPatternRequested(inspectRequest?.index ?? [])}
{formatIndexPatternRequested(inspectRequests[0]?.index ?? [])}
</span>
),
},
@ -137,8 +149,8 @@ export const ModalInspectQuery = ({
),
description: (
<span data-test-subj="query-time-description">
{inspectResponse != null
? `${numeral(inspectResponse.took).format('0,0')}ms`
{inspectResponses[0]?.took
? `${numeral(inspectResponses[0].took).format('0,0')}ms`
: i18n.SOMETHING_WENT_WRONG}
</span>
),
@ -170,42 +182,50 @@ export const ModalInspectQuery = ({
{
id: 'request',
name: 'Request',
content: (
<>
<EuiSpacer />
<EuiCodeBlock
language="js"
fontSize="m"
paddingSize="m"
color="dark"
overflowHeight={300}
isCopyable
>
{inspectRequest != null
? manageStringify(inspectRequest.body)
: i18n.SOMETHING_WENT_WRONG}
</EuiCodeBlock>
</>
),
content:
inspectRequests.length > 0 ? (
inspectRequests.map((inspectRequest, index) => (
<Fragment key={index}>
<EuiSpacer />
<EuiCodeBlock
language="js"
fontSize="m"
paddingSize="m"
color="dark"
overflowHeight={300}
isCopyable
>
{manageStringify(inspectRequest.body)}
</EuiCodeBlock>
</Fragment>
))
) : (
<EuiCodeBlock>{i18n.SOMETHING_WENT_WRONG}</EuiCodeBlock>
),
},
{
id: 'response',
name: 'Response',
content: (
<>
<EuiSpacer />
<EuiCodeBlock
language="js"
fontSize="m"
paddingSize="m"
color="dark"
overflowHeight={300}
isCopyable
>
{response}
</EuiCodeBlock>
</>
),
content:
inspectResponses.length > 0 ? (
responses.map((responseText, index) => (
<Fragment key={index}>
<EuiSpacer />
<EuiCodeBlock
language="js"
fontSize="m"
paddingSize="m"
color="dark"
overflowHeight={300}
isCopyable
>
{responseText}
</EuiCodeBlock>
</Fragment>
))
) : (
<EuiCodeBlock>{i18n.SOMETHING_WENT_WRONG}</EuiCodeBlock>
),
},
];

9
x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.test.tsx
View file

@ -11,7 +11,7 @@ import { mount, ReactWrapper } from 'enzyme';
import React from 'react';
import { MatrixHistogram } from '.';
import { useMatrixHistogram } from '../../containers/matrix_histogram';
import { useMatrixHistogramCombined } from '../../containers/matrix_histogram';
import { MatrixHistogramType } from '../../../../common/search_strategy/security_solution';
import { TestProviders } from '../../mock';
@ -30,7 +30,7 @@ jest.mock('../charts/barchart', () => ({
}));
jest.mock('../../containers/matrix_histogram', () => ({
useMatrixHistogram: jest.fn(),
useMatrixHistogramCombined: jest.fn(),
}));
jest.mock('../../components/matrix_histogram/utils', () => ({
@ -63,7 +63,7 @@ describe('Matrix Histogram Component', () => {
};
beforeAll(() => {
(useMatrixHistogram as jest.Mock).mockReturnValue([
(useMatrixHistogramCombined as jest.Mock).mockReturnValue([
false,
{
data: null,
@ -75,6 +75,7 @@ describe('Matrix Histogram Component', () => {
wrappingComponent: TestProviders,
});
});
describe('on initial load', () => {
test('it renders MatrixLoader', () => {
expect(wrapper.find('MatrixLoader').exists()).toBe(true);
@ -99,7 +100,7 @@ describe('Matrix Histogram Component', () => {
describe('not initial load', () => {
beforeAll(() => {
(useMatrixHistogram as jest.Mock).mockReturnValue([
(useMatrixHistogramCombined as jest.Mock).mockReturnValue([
false,
{
data: [

14
x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx
View file

@ -17,7 +17,7 @@ import { HeaderSection } from '../header_section';
import { MatrixLoader } from './matrix_loader';
import { Panel } from '../panel';
import { getBarchartConfigs, getCustomChartData } from './utils';
import { useMatrixHistogram } from '../../containers/matrix_histogram';
import { useMatrixHistogramCombined } from '../../containers/matrix_histogram';
import { MatrixHistogramProps, MatrixHistogramOption, MatrixHistogramQueryProps } from './types';
import { InspectButtonContainer } from '../inspect';
import { MatrixHistogramType } from '../../../../common/search_strategy/security_solution';
@ -40,6 +40,7 @@ export type MatrixHistogramComponentProps = MatrixHistogramProps &
id: string;
legendPosition?: Position;
mapping?: MatrixHistogramMappingTypes;
onError?: () => void;
showSpacer?: boolean;
setQuery: GlobalTimeArgs['setQuery'];
setAbsoluteRangeDatePickerTarget?: InputsModelId;
@ -77,6 +78,7 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramComponentProps> =
isPtrIncluded,
legendPosition,
mapping,
onError,
panelHeight = DEFAULT_PANEL_HEIGHT,
setAbsoluteRangeDatePickerTarget = 'global',
setQuery,
@ -133,17 +135,22 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramComponentProps> =
[defaultStackByOption, stackByOptions]
);
const [loading, { data, inspect, totalCount, refetch }] = useMatrixHistogram({
const matrixHistogramRequest = {
endDate,
errorMessage,
filterQuery,
histogramType,
indexNames,
onError,
startDate,
stackByField: selectedStackByOption.value,
isPtrIncluded,
docValueFields,
});
};
const [loading, { data, inspect, totalCount, refetch }] = useMatrixHistogramCombined(
matrixHistogramRequest
);
const titleWithStackByField = useMemo(
() => (title != null && typeof title === 'function' ? title(selectedStackByOption) : title),
@ -208,6 +215,7 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramComponentProps> =
title={titleWithStackByField}
titleSize={titleSize}
subtitle={subtitleWithCounts}
inspectMultiple
>
<EuiFlexGroup alignItems="center" gutterSize="none">
<EuiFlexItem grow={false}>

2
x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts
View file

@ -66,6 +66,7 @@ export interface MatrixHistogramQueryProps {
errorMessage: string;
indexNames: string[];
filterQuery?: ESQuery | string | undefined;
onError?: () => void;
setAbsoluteRangeDatePicker?: ActionCreator<{
id: InputsModelId;
from: string;
@ -78,6 +79,7 @@ export interface MatrixHistogramQueryProps {
threshold?: Threshold;
skip?: boolean;
isPtrIncluded?: boolean;
includeMissingData?: boolean;
}
export interface MatrixHistogramProps extends MatrixHistogramBasicProps {

4
x-pack/plugins/security_solution/public/common/components/matrix_histogram/utils.test.ts
View file

@ -106,7 +106,7 @@ describe('utils', () => {
expect(result).toEqual([]);
});
test('shoule format data correctly', () => {
test('should format data correctly', () => {
const data = [
{ x: 1, y: 2, g: 'g1' },
{ x: 2, y: 4, g: 'g1' },
@ -120,6 +120,7 @@ describe('utils', () => {
expect(result).toEqual([
{
key: 'g1',
color: '#1EA593',
value: [
{ x: 1, y: 2, g: 'g1' },
{ x: 2, y: 4, g: 'g1' },
@ -128,6 +129,7 @@ describe('utils', () => {
},
{
key: 'g2',
color: '#2B70F7',
value: [
{ x: 1, y: 1, g: 'g2' },
{ x: 2, y: 3, g: 'g2' },

12
x-pack/plugins/security_solution/public/common/components/matrix_histogram/utils.ts
View file

@ -82,6 +82,7 @@ export const defaultLegendColors = [
'#B0916F',
'#7B000B',
'#34130C',
'#GGGGGG',
];
export const formatToChartDataItem = ([key, value]: [
@ -100,11 +101,8 @@ export const getCustomChartData = (
const dataGroupedByEvent = groupBy('g', data);
const dataGroupedEntries = toPairs(dataGroupedByEvent);
const formattedChartData = map(formatToChartDataItem, dataGroupedEntries);
if (mapping)
return map((item: ChartSeriesData) => {
const mapItem = get(item.key, mapping);
return { ...item, color: mapItem?.color };
}, formattedChartData);
else return formattedChartData;
return formattedChartData.map((item: ChartSeriesData, idx: number) => {
const mapItem = get(item.key, mapping);
return { ...item, color: mapItem?.color ?? defaultLegendColors[idx] };
});
};

1
x-pack/plugins/security_solution/public/common/components/top_n/top_n.tsx
View file

@ -129,6 +129,7 @@ const TopNComponent: React.FC<Props> = ({
setAbsoluteRangeDatePickerTarget={setAbsoluteRangeDatePickerTarget}
setQuery={setQuery}
showSpacer={false}
toggleTopN={toggleTopN}
timelineId={timelineId}
to={to}
/>

64
x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.test.ts
View file

@ -8,7 +8,7 @@
import { renderHook, act } from '@testing-library/react-hooks';
import { useKibana } from '../../../common/lib/kibana';
import { useMatrixHistogram } from '.';
import { useMatrixHistogram, useMatrixHistogramCombined } from '.';
import { MatrixHistogramType } from '../../../../common/search_strategy';
import { TestProviders } from '../../mock/test_providers';
@ -25,6 +25,10 @@ describe('useMatrixHistogram', () => {
startDate: new Date(Date.now()).toISOString(),
};
afterEach(() => {
(useKibana().services.data.search.search as jest.Mock).mockClear();
});
it('should update request when props has changed', async () => {
const localProps = { ...props };
const { rerender } = renderHook(() => useMatrixHistogram(localProps), {
@ -54,3 +58,61 @@ describe('useMatrixHistogram', () => {
expect(result1).toBe(result2);
});
});
describe('useMatrixHistogramCombined', () => {
const props = {
endDate: new Date(Date.now()).toISOString(),
errorMessage: '',
filterQuery: {},
histogramType: MatrixHistogramType.events,
indexNames: [],
stackByField: 'event.module',
startDate: new Date(Date.now()).toISOString(),
};
afterEach(() => {
(useKibana().services.data.search.search as jest.Mock).mockClear();
});
it('should update request when props has changed', async () => {
const localProps = { ...props };
const { rerender } = renderHook(() => useMatrixHistogramCombined(localProps), {
wrapper: TestProviders,
});
localProps.stackByField = 'event.action';
rerender();
const mockCalls = (useKibana().services.data.search.search as jest.Mock).mock.calls;
expect(mockCalls.length).toBe(2);
expect(mockCalls[0][0].stackByField).toBe('event.module');
expect(mockCalls[1][0].stackByField).toBe('event.action');
});
it('should do two request when stacking by ip field', async () => {
const localProps = { ...props, stackByField: 'source.ip' };
renderHook(() => useMatrixHistogramCombined(localProps), {
wrapper: TestProviders,
});
const mockCalls = (useKibana().services.data.search.search as jest.Mock).mock.calls;
expect(mockCalls.length).toBe(2);
expect(mockCalls[0][0].stackByField).toBe('source.ip');
expect(mockCalls[1][0].stackByField).toBe('source.ip');
});
it('returns a memoized value', async () => {
const { result, rerender } = renderHook(() => useMatrixHistogramCombined(props), {
wrapper: TestProviders,
});
const result1 = result.current[1];
act(() => rerender());
const result2 = result.current[1];
expect(result1).toBe(result2);
});
});

75
x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts
View file

@ -7,7 +7,7 @@
import deepEqual from 'fast-deep-equal';
import { getOr, isEmpty, noop } from 'lodash/fp';
import { useCallback, useEffect, useRef, useState } from 'react';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Subscription } from 'rxjs';
import { MatrixHistogramQueryProps } from '../../components/matrix_histogram/types';
@ -53,10 +53,12 @@ export const useMatrixHistogram = ({
histogramType,
indexNames,
isPtrIncluded,
onError,
stackByField,
startDate,
threshold,
skip = false,
includeMissingData = true,
}: MatrixHistogramQueryProps): [
boolean,
UseMatrixHistogramArgs,
@ -99,6 +101,7 @@ export const useMatrixHistogram = ({
threshold,
...(isPtrIncluded != null ? { isPtrIncluded } : {}),
...(!isEmpty(docValueFields) ? { docValueFields } : {}),
...(includeMissingData != null ? { includeMissingData } : {}),
});
const { addError, addWarning } = useAppToasts();
@ -215,6 +218,7 @@ export const useMatrixHistogram = ({
]);
useEffect(() => {
// We want to search if it is not skipped, stackByField ends with ip and include missing data
if (!skip) {
hostsSearch(matrixHistogramRequest);
}
@ -240,3 +244,72 @@ export const useMatrixHistogram = ({
return [loading, matrixHistogramResponse, runMatrixHistogramSearch];
};
/* function needed to split ip histogram data requests due to elasticsearch bug https://github.com/elastic/kibana/issues/89205
* using includeMissingData parameter to do the "missing data" query separately
**/
export const useMatrixHistogramCombined = (
matrixHistogramQueryProps: MatrixHistogramQueryProps
): [boolean, UseMatrixHistogramArgs] => {
const [mainLoading, mainResponse] = useMatrixHistogram({
...matrixHistogramQueryProps,
includeMissingData: true,
});
const skipMissingData = useMemo(() => !matrixHistogramQueryProps.stackByField.endsWith('.ip'), [
matrixHistogramQueryProps.stackByField,
]);
const [missingDataLoading, missingDataResponse] = useMatrixHistogram({
...matrixHistogramQueryProps,
includeMissingData: false,
skip: skipMissingData,
});
const combinedLoading = useMemo<boolean>(() => mainLoading || missingDataLoading, [
mainLoading,
missingDataLoading,
]);
const combinedResponse = useMemo<UseMatrixHistogramArgs>(() => {
if (skipMissingData) return mainResponse;
const { data, inspect, totalCount, refetch, buckets } = mainResponse;
const {
data: extraData,
inspect: extraInspect,
totalCount: extraTotalCount,
refetch: extraRefetch,
} = missingDataResponse;
const combinedRefetch = () => {
refetch();
extraRefetch();
};
if (combinedLoading) {
return {
data: [],
inspect: {
dsl: [],
response: [],
},
refetch: combinedRefetch,
totalCount: -1,
buckets: [],
};
}
return {
data: [...data, ...extraData],
inspect: {
dsl: [...inspect.dsl, ...extraInspect.dsl],
response: [...inspect.response, ...extraInspect.response],
},
totalCount: totalCount + extraTotalCount,
refetch: combinedRefetch,
buckets,
};
}, [combinedLoading, mainResponse, missingDataResponse, skipMissingData]);
return [combinedLoading, combinedResponse];
};

8
x-pack/plugins/security_solution/public/overview/components/alerts_by_category/index.test.tsx
View file

@ -10,7 +10,7 @@ import React from 'react';
import '../../../common/mock/match_media';
import '../../../common/mock/react_beautiful_dnd';
import { useMatrixHistogram } from '../../../common/containers/matrix_histogram';
import { useMatrixHistogramCombined } from '../../../common/containers/matrix_histogram';
import { waitFor } from '@testing-library/react';
import { mockIndexPattern, TestProviders } from '../../../common/mock';
@ -19,7 +19,7 @@ import { AlertsByCategory } from '.';
jest.mock('../../../common/components/link_to');
jest.mock('../../../common/lib/kibana');
jest.mock('../../../common/containers/matrix_histogram', () => ({
useMatrixHistogram: jest.fn(),
useMatrixHistogramCombined: jest.fn(),
}));
const from = '2020-03-31T06:00:00.000Z';
@ -42,7 +42,7 @@ describe('Alerts by category', () => {
};
describe('before loading data', () => {
beforeAll(async () => {
(useMatrixHistogram as jest.Mock).mockReturnValue([
(useMatrixHistogramCombined as jest.Mock).mockReturnValue([
false,
{
data: null,
@ -101,7 +101,7 @@ describe('Alerts by category', () => {
describe('after loading data', () => {
beforeAll(async () => {
(useMatrixHistogram as jest.Mock).mockReturnValue([
(useMatrixHistogramCombined as jest.Mock).mockReturnValue([
false,
{
data: [

3
x-pack/plugins/security_solution/public/overview/components/events_by_dataset/index.tsx
View file

@ -52,6 +52,7 @@ interface Props extends Pick<GlobalTimeArgs, 'from' | 'to' | 'deleteQuery' | 'se
setAbsoluteRangeDatePickerTarget?: InputsModelId;
showSpacer?: boolean;
timelineId?: string;
toggleTopN?: () => void;
}
const getHistogramOption = (fieldName: string): MatrixHistogramOption => ({
@ -74,6 +75,7 @@ const EventsByDatasetComponent: React.FC<Props> = ({
showSpacer = true,
timelineId,
to,
toggleTopN,
}) => {
// create a unique, but stable (across re-renders) query id
const uniqueQueryId = useMemo(() => `${ID}-${uuid.v4()}`, []);
@ -164,6 +166,7 @@ const EventsByDatasetComponent: React.FC<Props> = ({
headerChildren={headerContent}
id={uniqueQueryId}
indexNames={indexNames}
onError={toggleTopN}
setAbsoluteRangeDatePickerTarget={setAbsoluteRangeDatePickerTarget}
setQuery={setQuery}
showSpacer={showSpacer}

182
x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts
View file

@ -308,3 +308,185 @@ export const expectedThresholdWithGroupFieldsAndCardinalityDsl = {
size: 0,
},
};
export const expectedThresholdGroupWithCardinalityDsl = {
allowNoIndices: true,
body: {
aggregations: {
eventActionGroup: {
aggs: {
cardinality_check: {
bucket_selector: {
buckets_path: { cardinalityCount: 'cardinality_count' },
script: 'params.cardinalityCount >= 10',
},
},
cardinality_count: { cardinality: { field: 'agent.name' } },
events: {
date_histogram: {
extended_bounds: { max: 1599667886215, min: 1599581486215 },
field: '@timestamp',
fixed_interval: '2700000ms',
min_doc_count: 200,
},
},
},
terms: {
order: { _count: 'desc' },
script: {
lang: 'painless',
source: "doc['host.name'].value + ':' + doc['agent.name'].value",
},
size: 10,
},
},
},
query: {
bool: {
filter: [
{ bool: { filter: [{ match_all: {} }], must: [], must_not: [], should: [] } },
{
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2020-09-08T16:11:26.215Z',
lte: '2020-09-09T16:11:26.215Z',
},
},
},
],
},
},
size: 0,
},
ignoreUnavailable: true,
index: [
'apm-*-transaction*',
'auditbeat-*',
'endgame-*',
'filebeat-*',
'logs-*',
'packetbeat-*',
'winlogbeat-*',
],
track_total_hits: true,
};
export const expectedIpIncludingMissingDataDsl = {
index: [
'apm-*-transaction*',
'auditbeat-*',
'endgame-*',
'filebeat-*',
'logs-*',
'packetbeat-*',
'winlogbeat-*',
],
allowNoIndices: true,
ignoreUnavailable: true,
track_total_hits: true,
body: {
aggregations: {
eventActionGroup: {
terms: {
field: 'source.ip',
missing: '0.0.0.0',
value_type: 'ip',
order: { _count: 'desc' },
size: 10,
},
aggs: {
events: {
date_histogram: {
field: '@timestamp',
fixed_interval: '2700000ms',
min_doc_count: 0,
extended_bounds: { min: 1599581486215, max: 1599667886215 },
},
},
},
},
},
query: {
bool: {
filter: [
{
bool: {
must: [],
filter: [{ match_all: {} }],
should: [],
must_not: [{ exists: { field: 'source.ip' } }],
},
},
{
range: {
'@timestamp': {
gte: '2020-09-08T16:11:26.215Z',
lte: '2020-09-09T16:11:26.215Z',
format: 'strict_date_optional_time',
},
},
},
],
},
},
size: 0,
},
};
export const expectedIpNotIncludingMissingDataDsl = {
index: [
'apm-*-transaction*',
'auditbeat-*',
'endgame-*',
'filebeat-*',
'logs-*',
'packetbeat-*',
'winlogbeat-*',
],
allowNoIndices: true,
ignoreUnavailable: true,
track_total_hits: true,
body: {
aggregations: {
eventActionGroup: {
terms: { field: 'source.ip', order: { _count: 'desc' }, size: 10, value_type: 'ip' },
aggs: {
events: {
date_histogram: {
field: '@timestamp',
fixed_interval: '2700000ms',
min_doc_count: 0,
extended_bounds: { min: 1599581486215, max: 1599667886215 },
},
},
},
},
},
query: {
bool: {
filter: [
{
bool: {
must: [],
filter: [{ match_all: {} }],
should: [],
must_not: [],
},
},
{ exists: { field: 'source.ip' } },
{
range: {
'@timestamp': {
gte: '2020-09-08T16:11:26.215Z',
lte: '2020-09-09T16:11:26.215Z',
format: 'strict_date_optional_time',
},
},
},
],
},
},
size: 0,
},
};

85
x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/query.events_histogram.dsl.test.ts
View file

@ -12,6 +12,9 @@ import {
expectedThresholdDsl,
expectedThresholdMissingFieldDsl,
expectedThresholdWithCardinalityDsl,
expectedThresholdGroupWithCardinalityDsl,
expectedIpIncludingMissingDataDsl,
expectedIpNotIncludingMissingDataDsl,
} from './__mocks__/';
describe('buildEventsHistogramQuery', () => {
@ -63,67 +66,25 @@ describe('buildEventsHistogramQuery', () => {
cardinality: { field: ['agent.name'], value: '10' },
},
})
).toEqual({
allowNoIndices: true,
body: {
aggregations: {
eventActionGroup: {
aggs: {
cardinality_check: {
bucket_selector: {
buckets_path: { cardinalityCount: 'cardinality_count' },
script: 'params.cardinalityCount >= 10',
},
},
cardinality_count: { cardinality: { field: 'agent.name' } },
events: {
date_histogram: {
extended_bounds: { max: 1599667886215, min: 1599581486215 },
field: '@timestamp',
fixed_interval: '2700000ms',
min_doc_count: 200,
},
},
},
terms: {
order: { _count: 'desc' },
script: {
lang: 'painless',
source: "doc['host.name'].value + ':' + doc['agent.name'].value",
},
size: 10,
},
},
},
query: {
bool: {
filter: [
{ bool: { filter: [{ match_all: {} }], must: [], must_not: [], should: [] } },
{
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2020-09-08T16:11:26.215Z',
lte: '2020-09-09T16:11:26.215Z',
},
},
},
],
},
},
size: 0,
},
ignoreUnavailable: true,
index: [
'apm-*-transaction*',
'auditbeat-*',
'endgame-*',
'filebeat-*',
'logs-*',
'packetbeat-*',
'winlogbeat-*',
],
track_total_hits: true,
});
).toEqual(expectedThresholdGroupWithCardinalityDsl);
});
test('builds query with stack by ip and including missing data', () => {
expect(
buildEventsHistogramQuery({
...mockOptions,
stackByField: 'source.ip',
})
).toEqual(expectedIpIncludingMissingDataDsl);
});
test('builds query with stack by ip and not including missing data', () => {
expect(
buildEventsHistogramQuery({
...mockOptions,
includeMissingData: false,
stackByField: 'source.ip',
})
).toEqual(expectedIpNotIncludingMissingDataDsl);
});
});

45
x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/query.events_histogram.dsl.ts
View file

@ -22,9 +22,45 @@ export const buildEventsHistogramQuery = ({
defaultIndex,
stackByField = 'event.action',
threshold,
includeMissingData = true,
}: MatrixHistogramRequestOptions) => {
const [queryFilterFirstClause, ...queryFilterClauses] = createQueryFilterClauses(filterQuery);
const stackByIpField =
stackByField != null &&
showAllOthersBucket.includes(stackByField) &&
stackByField.endsWith('.ip');
const filter = [
...createQueryFilterClauses(filterQuery),
...[
{
...queryFilterFirstClause,
bool: {
...(queryFilterFirstClause.bool || {}),
must_not: [
...(queryFilterFirstClause.bool?.must_not || []),
...(stackByIpField && includeMissingData
? [
{
exists: {
field: stackByField,
},
},
]
: []),
],
},
},
...queryFilterClauses,
],
...(stackByIpField && !includeMissingData
? [
{
exists: {
field: stackByField,
},
},
]
: []),
{
range: {
'@timestamp': {
@ -54,7 +90,12 @@ export const buildEventsHistogramQuery = ({
const missing =
stackByField != null && showAllOthersBucket.includes(stackByField)
? {
missing: stackByField?.endsWith('.ip') ? '0.0.0.0' : i18n.ALL_OTHERS,
...(includeMissingData
? stackByField?.endsWith('.ip')
? { missing: '0.0.0.0' }
: { missing: i18n.ALL_OTHERS }
: {}),
...(stackByField?.endsWith('.ip') ? { value_type: 'ip' } : {}),
}
: {};