github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Performance][Security Solution][1/4] - Field Browser Performance (#212469)

Browse source 
## Summary
Part 1 of https://github.com/elastic/kibana/pull/212173

### Testing
For setup see testing section here:
https://github.com/elastic/kibana/pull/212173#issue-2870522020

**Areas to test:**
- Alert Table (Alerts, Rule Detail, Rule Preview pages)
- Security solution field browser component
- Flyout table tab.

### Background

When investigating the performance of the security solution application,
one of the issues that was observed was locking of the page and field
browser component when the total number of fields returned were
significantly high.

This led to cell values not rendering in the alert table, and the field
browser in all the security solution pages causing the page to crash.
The relevant images can be seen at the bottom of this description

In short: The `push(...fields)` is non-performant at scale, and at a
significant enough scale (Testing was done with 500k mapped fields),
fails to run due to excessive arguments provided to the `push` method.
In this PR improvements are made in the `browserFields` transformations
that are done for the field browser component, expandable flyout table
tab, and alert/rule tables via `CellValue` component.

This work was done to get immediate improvements in the security
solution UI, but a longer term consideration will be whether or not the
`browserFields` is even necessary anymore as a concept based on what is
available via the `fields` api. We will revisit once our Sourcerer
refactoring work is done.

<img width="1728" alt="Screenshot 2025-02-26 at 10 15 29 AM"
src="https://github.com/user-attachments/assets/a25f577f-f758-415e-9c93-5452eadb8020"
/>

<img width="1445" alt="Screenshot 2025-02-26 at 10 18 36 AM"
src="https://github.com/user-attachments/assets/d70970d3-991a-47ba-b617-5862d18101b6"
/>

<img width="1469" alt="Screenshot 2025-02-26 at 10 19 48 AM"
src="https://github.com/user-attachments/assets/1767aa9b-66ab-46be-bc1a-5311630c2765"
/>


![image](https://github.com/user-attachments/assets/5d746b21-fa9b-425b-826a-cc7abd444f21)


![image](https://github.com/user-attachments/assets/4dff2378-d61b-4770-b46b-41cb37d6ead4)


### After the fix
(Done on [this branch](https://github.com/elastic/kibana/pull/212173)
that has the other changes as well)


https://github.com/user-attachments/assets/da992296-4eb8-49d4-96ca-b0a19a00f1f0


### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
This commit is contained in:
Michael Olorunnisola 2025-03-03 20:22:25 -05:00 • committed by GitHub
parent e9813b8c72
commit 750e156c26
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 210 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

53
src/platform/packages/shared/response-ops/alerts-fields-browser/components/field_items/field_items.tsx
View file

@ -17,7 +17,6 @@ import {
EuiBadge,
EuiScreenReaderOnly,
} from '@elastic/eui';
import { uniqBy } from 'lodash/fp';
import type { BrowserFields } from '@kbn/rule-registry-plugin/common';
import { EcsFlat } from '@elastic/ecs';
import { EcsMetadata } from '@kbn/alerts-as-data-utils/src/field_maps/types';
@ -67,29 +66,41 @@ export const getFieldItemsData = ({
selectedCategoryIds.length > 0 ? selectedCategoryIds : Object.keys(browserFields);
const selectedFieldIds = new Set(columnIds);
const fieldItems = uniqBy(
'name',
categoryIds.reduce<BrowserFieldItem[]>((fieldItemsAcc, categoryId) => {
const getFieldItems = () => {
const fieldItemsAcc: BrowserFieldItem[] = [];
const fieldsSeen: Set<string> = new Set();
/**
* Both categoryIds and the fields in browserFields can be significantly large. Technically speaking,
* there is no upper cap on how many fields a customer can have. We are using a for loop here to avoid
* the performance issues that can arise from using map/filter/reduce.
*/
for (let i = 0; i < categoryIds.length; i += 1) {
const categoryId = categoryIds[i];
const categoryBrowserFields = Object.values(browserFields[categoryId]?.fields ?? {});
if (categoryBrowserFields.length > 0) {
fieldItemsAcc.push(
...categoryBrowserFields.map(({ name = '', ...field }) => {
return {
name,
type: field.type,
description: getDescription(name, EcsFlat as Record<string, EcsMetadata>),
example: field.example?.toString(),
category: getCategory(name),
selected: selectedFieldIds.has(name),
isRuntime: !!field.runtimeField,
};
})
);
for (let j = 0; j < categoryBrowserFields.length; j += 1) {
const field = categoryBrowserFields[j];
const name = field.name !== undefined ? field.name : '';
if (fieldsSeen.has(name)) continue;
fieldsSeen.add(name);
const categoryFieldItem = {
name,
type: field.type,
description: getDescription(name, EcsFlat as Record<string, EcsMetadata>),
example: field.example?.toString(),
category: getCategory(name),
selected: selectedFieldIds.has(name),
isRuntime: !!field.runtimeField,
};
fieldItemsAcc.push(categoryFieldItem);
}
}
return fieldItemsAcc;
}, [])
);
return { fieldItems };
}
return fieldItemsAcc;
};
return { fieldItems: getFieldItems() };
};
const getDefaultFieldTableColumns = ({ highlight }: { highlight: string }): FieldTableColumns => {

150
x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.test.tsx
View file

@ -5,151 +5,19 @@
* 2.0.
*/
import type { IndexFieldSearch } from './use_data_view';
import { useDataView } from './use_data_view';
import { mocksSource } from './mock';
import { mockGlobalState, TestProviders } from '../../mock';
import { act, renderHook } from '@testing-library/react';
import { useKibana } from '../../lib/kibana';
const mockDispatch = jest.fn();
jest.mock('react-redux', () => {
const original = jest.requireActual('react-redux');
return {
...original,
useDispatch: () => mockDispatch,
};
});
jest.mock('../../lib/kibana');
jest.mock('../../lib/apm/use_track_http_request');
import { mockBrowserFields, mockIndexFields, mockIndexFieldsByName } from './mock';
import * as indexUtils from '.';
describe('source/index.tsx', () => {
describe('useDataView hook', () => {
const mockSearchResponse = {
...mocksSource,
indicesExist: ['auditbeat-*', mockGlobalState.sourcerer.signalIndexName],
isRestore: false,
rawResponse: {},
runtimeMappings: {},
};
beforeEach(() => {
jest.clearAllMocks();
const mock = {
subscribe: ({ next }: { next: Function }) => next(mockSearchResponse),
unsubscribe: jest.fn(),
};
(useKibana as jest.Mock).mockReturnValue({
services: {
data: {
dataViews: {
...useKibana().services.data.dataViews,
get: async (dataViewId: string, displayErrors?: boolean, refreshFields = false) => {
const dataViewMock = {
id: dataViewId,
matchedIndices: refreshFields
? ['hello', 'world', 'refreshed']
: ['hello', 'world'],
fields: mocksSource.indexFields,
getIndexPattern: () =>
refreshFields ? 'hello*,world*,refreshed*' : 'hello*,world*',
getRuntimeMappings: () => ({
myfield: {
type: 'keyword',
},
}),
};
return Promise.resolve({
toSpec: () => dataViewMock,
...dataViewMock,
});
},
getFieldsForWildcard: async () => Promise.resolve(),
getExistingIndices: async (indices: string[]) => Promise.resolve(indices),
},
search: {
search: jest.fn().mockReturnValue({
subscribe: ({ next }: { next: Function }) => {
next(mockSearchResponse);
return mock;
},
unsubscribe: jest.fn(),
}),
},
},
},
});
describe('getAllBrowserFields', () => {
it('should return the expected browser fields list', () => {
expect(indexUtils.getAllBrowserFields(mockBrowserFields)).toEqual(mockIndexFields);
});
it('sets field data for data view', async () => {
const { result } = renderHook(() => useDataView(), {
wrapper: TestProviders,
});
});
await act(async () => {
await result.current.indexFieldsSearch({ dataViewId: 'neato' });
});
expect(mockDispatch.mock.calls[0][0]).toEqual({
type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING',
payload: { id: 'neato', loading: true },
});
const { type: sourceType, payload } = mockDispatch.mock.calls[1][0];
expect(sourceType).toEqual('x-pack/security_solution/local/sourcerer/SET_DATA_VIEW');
expect(payload.id).toEqual('neato');
});
it('should reuse the result for dataView info when cleanCache not passed', async () => {
let indexFieldsSearch: IndexFieldSearch;
const { result } = renderHook(() => useDataView(), {
wrapper: TestProviders,
});
await act(async () => {
indexFieldsSearch = result.current.indexFieldsSearch;
});
await indexFieldsSearch!({ dataViewId: 'neato' });
const {
payload: { browserFields, indexFields },
} = mockDispatch.mock.calls[1][0];
mockDispatch.mockClear();
await indexFieldsSearch!({ dataViewId: 'neato' });
const {
payload: { browserFields: newBrowserFields, indexFields: newIndexFields },
} = mockDispatch.mock.calls[1][0];
expect(browserFields).toBe(newBrowserFields);
expect(indexFields).toBe(newIndexFields);
});
it('should not reuse the result for dataView info when cleanCache passed', async () => {
let indexFieldsSearch: IndexFieldSearch;
const { result } = renderHook(() => useDataView(), {
wrapper: TestProviders,
});
await act(async () => {
indexFieldsSearch = result.current.indexFieldsSearch;
});
await indexFieldsSearch!({ dataViewId: 'neato' });
const {
payload: { patternList },
} = mockDispatch.mock.calls[1][0];
mockDispatch.mockClear();
await indexFieldsSearch!({ dataViewId: 'neato', cleanCache: true });
const {
payload: { patternList: newPatternList },
} = mockDispatch.mock.calls[1][0];
expect(patternList).not.toBe(newPatternList);
expect(patternList).not.toContain('refreshed*');
expect(newPatternList).toContain('refreshed*');
describe('getAllFieldsByName', () => {
it('should return the expected browser fields list', () => {
expect(indexUtils.getAllFieldsByName(mockBrowserFields)).toEqual(mockIndexFieldsByName);
});
});
});

13
x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.tsx
View file

@ -22,10 +22,11 @@ import type { ENDPOINT_FIELDS_SEARCH_STRATEGY } from '../../../../common/endpoin
export type { BrowserFields };
export function getAllBrowserFields(browserFields: BrowserFields): Array<Partial<FieldSpec>> {
const result: Array<Partial<FieldSpec>> = [];
let result: Array<Partial<FieldSpec>> = [];
for (const namespace of Object.values(browserFields)) {
if (namespace.fields) {
result.push(...Object.values(namespace.fields));
const namespaceFields = Object.values(namespace.fields);
result = result.concat(namespaceFields);
}
}
return result;
@ -36,9 +37,11 @@ export function getAllBrowserFields(browserFields: BrowserFields): Array<Partial
* @param browserFields
* @returns
*/
export const getAllFieldsByName = (
browserFields: BrowserFields
): { [fieldName: string]: Partial<FieldSpec> } => keyBy('name', getAllBrowserFields(browserFields));
export const getAllFieldsByName = memoizeOne(
(browserFields: BrowserFields): { [fieldName: string]: Partial<FieldSpec> } =>
keyBy('name', getAllBrowserFields(browserFields)),
(newArgs, lastArgs) => newArgs[0] === lastArgs[0]
);
export const getIndexFields = memoizeOne(
(title: string, fields: IIndexPatternFieldList): DataViewBase =>

8
x-pack/solutions/security/plugins/security_solution/public/common/containers/source/mock.ts
View file

@ -7,6 +7,7 @@
import type { MappingRuntimeFieldType } from '@elastic/elasticsearch/lib/api/types';
import { flatten } from 'lodash';
import type { FieldSpec } from '@kbn/data-views-plugin/common';
import type { BrowserFields } from '../../../../common/search_strategy/index_fields';
export const mocksSource = {
@ -387,6 +388,13 @@ export const mockIndexFields = flatten(
Object.values(mockBrowserFields).map((fieldItem) => Object.values(fieldItem.fields ?? {}))
);
export const mockIndexFieldsByName = mockIndexFields.reduce((acc, indexFieldObj) => {
if (indexFieldObj.name) {
acc[indexFieldObj.name] = indexFieldObj;
}
return acc;
}, {} as { [fieldName: string]: Partial<FieldSpec> });
const runTimeType: MappingRuntimeFieldType = 'keyword' as const;
export const mockRuntimeMappings = {

153
x-pack/solutions/security/plugins/security_solution/public/common/containers/source/use_data_view.test.tsx Normal file
View file

@ -0,0 +1,153 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IndexFieldSearch } from './use_data_view';
import { useDataView } from './use_data_view';
import { mocksSource } from './mock';
import { mockGlobalState, TestProviders } from '../../mock';
import { act, renderHook } from '@testing-library/react';
import { useKibana } from '../../lib/kibana';
const mockDispatch = jest.fn();
jest.mock('react-redux', () => {
const original = jest.requireActual('react-redux');
return {
...original,
useDispatch: () => mockDispatch,
};
});
jest.mock('../../lib/kibana');
jest.mock('../../lib/apm/use_track_http_request');
describe('useDataView', () => {
const mockSearchResponse = {
...mocksSource,
indicesExist: ['auditbeat-*', mockGlobalState.sourcerer.signalIndexName],
isRestore: false,
rawResponse: {},
runtimeMappings: {},
};
beforeEach(() => {
jest.clearAllMocks();
const mock = {
subscribe: ({ next }: { next: Function }) => next(mockSearchResponse),
unsubscribe: jest.fn(),
};
(useKibana as jest.Mock).mockReturnValue({
services: {
data: {
dataViews: {
...useKibana().services.data.dataViews,
get: async (dataViewId: string, displayErrors?: boolean, refreshFields = false) => {
const dataViewMock = {
id: dataViewId,
matchedIndices: refreshFields
? ['hello', 'world', 'refreshed']
: ['hello', 'world'],
fields: mocksSource.indexFields,
getIndexPattern: () =>
refreshFields ? 'hello*,world*,refreshed*' : 'hello*,world*',
getRuntimeMappings: () => ({
myfield: {
type: 'keyword',
},
}),
};
return Promise.resolve({
toSpec: () => dataViewMock,
...dataViewMock,
});
},
getFieldsForWildcard: async () => Promise.resolve(),
getExistingIndices: async (indices: string[]) => Promise.resolve(indices),
},
search: {
search: jest.fn().mockReturnValue({
subscribe: ({ next }: { next: Function }) => {
next(mockSearchResponse);
return mock;
},
unsubscribe: jest.fn(),
}),
},
},
},
});
});
it('sets field data for data view', async () => {
const { result } = renderHook(() => useDataView(), {
wrapper: TestProviders,
});
await act(async () => {
await result.current.indexFieldsSearch({ dataViewId: 'neato' });
});
expect(mockDispatch.mock.calls[0][0]).toEqual({
type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING',
payload: { id: 'neato', loading: true },
});
const { type: sourceType, payload } = mockDispatch.mock.calls[1][0];
expect(sourceType).toEqual('x-pack/security_solution/local/sourcerer/SET_DATA_VIEW');
expect(payload.id).toEqual('neato');
});
it('should reuse the result for dataView info when cleanCache not passed', async () => {
let indexFieldsSearch: IndexFieldSearch;
const { result } = renderHook(() => useDataView(), {
wrapper: TestProviders,
});
await act(async () => {
indexFieldsSearch = result.current.indexFieldsSearch;
});
await indexFieldsSearch!({ dataViewId: 'neato' });
const {
payload: { browserFields, indexFields },
} = mockDispatch.mock.calls[1][0];
mockDispatch.mockClear();
await indexFieldsSearch!({ dataViewId: 'neato' });
const {
payload: { browserFields: newBrowserFields, indexFields: newIndexFields },
} = mockDispatch.mock.calls[1][0];
expect(browserFields).toBe(newBrowserFields);
expect(indexFields).toBe(newIndexFields);
});
it('should not reuse the result for dataView info when cleanCache passed', async () => {
let indexFieldsSearch: IndexFieldSearch;
const { result } = renderHook(() => useDataView(), {
wrapper: TestProviders,
});
await act(async () => {
indexFieldsSearch = result.current.indexFieldsSearch;
});
await indexFieldsSearch!({ dataViewId: 'neato' });
const {
payload: { patternList },
} = mockDispatch.mock.calls[1][0];
mockDispatch.mockClear();
await indexFieldsSearch!({ dataViewId: 'neato', cleanCache: true });
const {
payload: { patternList: newPatternList },
} = mockDispatch.mock.calls[1][0];
expect(patternList).not.toBe(newPatternList);
expect(patternList).not.toContain('refreshed*');
expect(newPatternList).toContain('refreshed*');
});
});