[8.17] [Performance][Security Solution][1/4] - Field Browser Performance (#212469) (#213025)

# Backport This will backport the following commits from `main` to `8.17`: - [[Performance][Security Solution][1/4] - Field Browser Performance (#212469)](https://github.com/elastic/kibana/pull/212469)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)
2025-04-24 09:48:58 -04:00 · 2025-03-04 12:56:13 -05:00 · 2025-03-04 12:56:13 -05:00 · 543bf80673
commit 543bf80673
parent 72460c6d64
5 changed files with 211 additions and 177 deletions
--- a/x-pack/plugins/security_solution/public/common/containers/source/index.test.tsx
+++ b/x-pack/plugins/security_solution/public/common/containers/source/index.test.tsx
@ -5,160 +5,19 @@
 * 2.0.
 */

-import type { PropsWithChildren } from 'react';
-import type { IndexFieldSearch } from './use_data_view';
-import { useDataView } from './use_data_view';
-import { mocksSource } from './mock';
-import { mockGlobalState, TestProviders } from '../../mock';
-import { act, renderHook } from '@testing-library/react-hooks';
-import { useKibana } from '../../lib/kibana';
-
-const mockDispatch = jest.fn();
-jest.mock('react-redux', () => {
-  const original = jest.requireActual('react-redux');
-
-  return {
-    ...original,
-    useDispatch: () => mockDispatch,
-  };
-});
-jest.mock('../../lib/kibana');
-jest.mock('../../lib/apm/use_track_http_request');
+import { mockBrowserFields, mockIndexFields, mockIndexFieldsByName } from './mock';
+import * as indexUtils from '.';

 describe('source/index.tsx', () => {
-  describe('useDataView hook', () => {
-    const mockSearchResponse = {
-      ...mocksSource,
-      indicesExist: ['auditbeat-*', mockGlobalState.sourcerer.signalIndexName],
-      isRestore: false,
-      rawResponse: {},
-      runtimeMappings: {},
-    };
-
-    beforeEach(() => {
-      jest.clearAllMocks();
-      const mock = {
-        subscribe: ({ next }: { next: Function }) => next(mockSearchResponse),
-        unsubscribe: jest.fn(),
-      };
-
-      (useKibana as jest.Mock).mockReturnValue({
-        services: {
-          data: {
-            dataViews: {
-              ...useKibana().services.data.dataViews,
-              get: async (dataViewId: string, displayErrors?: boolean, refreshFields = false) => {
-                const dataViewMock = {
-                  id: dataViewId,
-                  matchedIndices: refreshFields
-                    ? ['hello', 'world', 'refreshed']
-                    : ['hello', 'world'],
-                  fields: mocksSource.indexFields,
-                  getIndexPattern: () =>
-                    refreshFields ? 'hello*,world*,refreshed*' : 'hello*,world*',
-                  getRuntimeMappings: () => ({
-                    myfield: {
-                      type: 'keyword',
-                    },
-                  }),
-                };
-                return Promise.resolve({
-                  toSpec: () => dataViewMock,
-                  ...dataViewMock,
-                });
-              },
-              getFieldsForWildcard: async () => Promise.resolve(),
-              getExistingIndices: async (indices: string[]) => Promise.resolve(indices),
-            },
-            search: {
-              search: jest.fn().mockReturnValue({
-                subscribe: ({ next }: { next: Function }) => {
-                  next(mockSearchResponse);
-                  return mock;
-                },
-                unsubscribe: jest.fn(),
-              }),
-            },
-          },
-        },
-      });
-    });
-    it('sets field data for data view', async () => {
-      await act(async () => {
-        const { waitForNextUpdate, result } = renderHook<
-          PropsWithChildren<{}>,
-          { indexFieldsSearch: IndexFieldSearch }
-        >(() => useDataView(), {
-          wrapper: TestProviders,
-        });
-        await waitForNextUpdate();
-        await result.current.indexFieldsSearch({ dataViewId: 'neato' });
-      });
-      expect(mockDispatch.mock.calls[0][0]).toEqual({
-        type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING',
-        payload: { id: 'neato', loading: true },
-      });
-      const { type: sourceType, payload } = mockDispatch.mock.calls[1][0];
-      expect(sourceType).toEqual('x-pack/security_solution/local/sourcerer/SET_DATA_VIEW');
-      expect(payload.id).toEqual('neato');
+  describe('getAllBrowserFields', () => {
+    it('should return the expected browser fields list', () => {
+      expect(indexUtils.getAllBrowserFields(mockBrowserFields)).toEqual(mockIndexFields);
    });
+  });

-    it('should reuse the result for dataView info when cleanCache not passed', async () => {
-      let indexFieldsSearch: IndexFieldSearch;
-      await act(async () => {
-        const { waitForNextUpdate, result } = renderHook<
-          PropsWithChildren<{}>,
-          { indexFieldsSearch: IndexFieldSearch }
-        >(() => useDataView(), {
-          wrapper: TestProviders,
-        });
-        await waitForNextUpdate();
-        indexFieldsSearch = result.current.indexFieldsSearch;
-      });
-
-      await indexFieldsSearch!({ dataViewId: 'neato' });
-      const {
-        payload: { browserFields, indexFields },
-      } = mockDispatch.mock.calls[1][0];
-
-      mockDispatch.mockClear();
-
-      await indexFieldsSearch!({ dataViewId: 'neato' });
-      const {
-        payload: { browserFields: newBrowserFields, indexFields: newIndexFields },
-      } = mockDispatch.mock.calls[1][0];
-
-      expect(browserFields).toBe(newBrowserFields);
-      expect(indexFields).toBe(newIndexFields);
-    });
-
-    it('should not reuse the result for dataView info when cleanCache passed', async () => {
-      let indexFieldsSearch: IndexFieldSearch;
-      await act(async () => {
-        const { waitForNextUpdate, result } = renderHook<
-          PropsWithChildren<{}>,
-          { indexFieldsSearch: IndexFieldSearch }
-        >(() => useDataView(), {
-          wrapper: TestProviders,
-        });
-        await waitForNextUpdate();
-        indexFieldsSearch = result.current.indexFieldsSearch;
-      });
-
-      await indexFieldsSearch!({ dataViewId: 'neato' });
-      const {
-        payload: { patternList },
-      } = mockDispatch.mock.calls[1][0];
-
-      mockDispatch.mockClear();
-
-      await indexFieldsSearch!({ dataViewId: 'neato', cleanCache: true });
-      const {
-        payload: { patternList: newPatternList },
-      } = mockDispatch.mock.calls[1][0];
-      expect(patternList).not.toBe(newPatternList);
-      expect(patternList).not.toContain('refreshed*');
-      expect(newPatternList).toContain('refreshed*');
+  describe('getAllFieldsByName', () => {
+    it('should return the expected browser fields list', () => {
+      expect(indexUtils.getAllFieldsByName(mockBrowserFields)).toEqual(mockIndexFieldsByName);
    });
  });
 });
--- a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx
+++ b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx
@ -22,10 +22,11 @@ import type { ENDPOINT_FIELDS_SEARCH_STRATEGY } from '../../../../common/endpoin
 export type { BrowserFields };

 export function getAllBrowserFields(browserFields: BrowserFields): Array<Partial<FieldSpec>> {
-  const result: Array<Partial<FieldSpec>> = [];
+  let result: Array<Partial<FieldSpec>> = [];
  for (const namespace of Object.values(browserFields)) {
    if (namespace.fields) {
-      result.push(...Object.values(namespace.fields));
+      const namespaceFields = Object.values(namespace.fields);
+      result = result.concat(namespaceFields);
    }
  }
  return result;
@ -36,9 +37,11 @@ export function getAllBrowserFields(browserFields: BrowserFields): Array<Partial
 * @param browserFields
 * @returns
 */
-export const getAllFieldsByName = (
-  browserFields: BrowserFields
-): { [fieldName: string]: Partial<FieldSpec> } => keyBy('name', getAllBrowserFields(browserFields));
+export const getAllFieldsByName = memoizeOne(
+  (browserFields: BrowserFields): { [fieldName: string]: Partial<FieldSpec> } =>
+    keyBy('name', getAllBrowserFields(browserFields)),
+  (newArgs, lastArgs) => newArgs[0] === lastArgs[0]
+);

 export const getIndexFields = memoizeOne(
  (title: string, fields: IIndexPatternFieldList): DataViewBase =>
--- a/x-pack/plugins/security_solution/public/common/containers/source/mock.ts
+++ b/x-pack/plugins/security_solution/public/common/containers/source/mock.ts
@ -7,6 +7,7 @@

 import type { MappingRuntimeFieldType } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { flatten } from 'lodash';
+import type { FieldSpec } from '@kbn/data-views-plugin/common';
 import type { BrowserFields } from '../../../../common/search_strategy/index_fields';

 export const mocksSource = {
@ -387,6 +388,13 @@ export const mockIndexFields = flatten(
  Object.values(mockBrowserFields).map((fieldItem) => Object.values(fieldItem.fields ?? {}))
 );

+export const mockIndexFieldsByName = mockIndexFields.reduce((acc, indexFieldObj) => {
+  if (indexFieldObj.name) {
+    acc[indexFieldObj.name] = indexFieldObj;
+  }
+  return acc;
+}, {} as { [fieldName: string]: Partial<FieldSpec> });
+
 const runTimeType: MappingRuntimeFieldType = 'keyword' as const;

 export const mockRuntimeMappings = {
--- a/x-pack/plugins/security_solution/public/common/containers/source/use_data_view.test.tsx
+++ b/x-pack/plugins/security_solution/public/common/containers/source/use_data_view.test.tsx
@ -0,0 +1,153 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IndexFieldSearch } from './use_data_view';
+import { useDataView } from './use_data_view';
+import { mocksSource } from './mock';
+import { mockGlobalState, TestProviders } from '../../mock';
+import { act, renderHook } from '@testing-library/react';
+import { useKibana } from '../../lib/kibana';
+
+const mockDispatch = jest.fn();
+jest.mock('react-redux', () => {
+  const original = jest.requireActual('react-redux');
+
+  return {
+    ...original,
+    useDispatch: () => mockDispatch,
+  };
+});
+jest.mock('../../lib/kibana');
+jest.mock('../../lib/apm/use_track_http_request');
+
+describe('useDataView', () => {
+  const mockSearchResponse = {
+    ...mocksSource,
+    indicesExist: ['auditbeat-*', mockGlobalState.sourcerer.signalIndexName],
+    isRestore: false,
+    rawResponse: {},
+    runtimeMappings: {},
+  };
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    const mock = {
+      subscribe: ({ next }: { next: Function }) => next(mockSearchResponse),
+      unsubscribe: jest.fn(),
+    };
+
+    (useKibana as jest.Mock).mockReturnValue({
+      services: {
+        data: {
+          dataViews: {
+            ...useKibana().services.data.dataViews,
+            get: async (dataViewId: string, displayErrors?: boolean, refreshFields = false) => {
+              const dataViewMock = {
+                id: dataViewId,
+                matchedIndices: refreshFields
+                  ? ['hello', 'world', 'refreshed']
+                  : ['hello', 'world'],
+                fields: mocksSource.indexFields,
+                getIndexPattern: () =>
+                  refreshFields ? 'hello*,world*,refreshed*' : 'hello*,world*',
+                getRuntimeMappings: () => ({
+                  myfield: {
+                    type: 'keyword',
+                  },
+                }),
+              };
+              return Promise.resolve({
+                toSpec: () => dataViewMock,
+                ...dataViewMock,
+              });
+            },
+            getFieldsForWildcard: async () => Promise.resolve(),
+            getExistingIndices: async (indices: string[]) => Promise.resolve(indices),
+          },
+          search: {
+            search: jest.fn().mockReturnValue({
+              subscribe: ({ next }: { next: Function }) => {
+                next(mockSearchResponse);
+                return mock;
+              },
+              unsubscribe: jest.fn(),
+            }),
+          },
+        },
+      },
+    });
+  });
+  it('sets field data for data view', async () => {
+    const { result } = renderHook(() => useDataView(), {
+      wrapper: TestProviders,
+    });
+
+    await act(async () => {
+      await result.current.indexFieldsSearch({ dataViewId: 'neato' });
+    });
+    expect(mockDispatch.mock.calls[0][0]).toEqual({
+      type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING',
+      payload: { id: 'neato', loading: true },
+    });
+    const { type: sourceType, payload } = mockDispatch.mock.calls[1][0];
+    expect(sourceType).toEqual('x-pack/security_solution/local/sourcerer/SET_DATA_VIEW');
+    expect(payload.id).toEqual('neato');
+  });
+
+  it('should reuse the result for dataView info when cleanCache not passed', async () => {
+    let indexFieldsSearch: IndexFieldSearch;
+
+    const { result } = renderHook(() => useDataView(), {
+      wrapper: TestProviders,
+    });
+
+    await act(async () => {
+      indexFieldsSearch = result.current.indexFieldsSearch;
+    });
+
+    await indexFieldsSearch!({ dataViewId: 'neato' });
+    const {
+      payload: { browserFields, indexFields },
+    } = mockDispatch.mock.calls[1][0];
+
+    mockDispatch.mockClear();
+
+    await indexFieldsSearch!({ dataViewId: 'neato' });
+    const {
+      payload: { browserFields: newBrowserFields, indexFields: newIndexFields },
+    } = mockDispatch.mock.calls[1][0];
+
+    expect(browserFields).toBe(newBrowserFields);
+    expect(indexFields).toBe(newIndexFields);
+  });
+
+  it('should not reuse the result for dataView info when cleanCache passed', async () => {
+    let indexFieldsSearch: IndexFieldSearch;
+    const { result } = renderHook(() => useDataView(), {
+      wrapper: TestProviders,
+    });
+
+    await act(async () => {
+      indexFieldsSearch = result.current.indexFieldsSearch;
+    });
+
+    await indexFieldsSearch!({ dataViewId: 'neato' });
+    const {
+      payload: { patternList },
+    } = mockDispatch.mock.calls[1][0];
+
+    mockDispatch.mockClear();
+
+    await indexFieldsSearch!({ dataViewId: 'neato', cleanCache: true });
+    const {
+      payload: { patternList: newPatternList },
+    } = mockDispatch.mock.calls[1][0];
+    expect(patternList).not.toBe(newPatternList);
+    expect(patternList).not.toContain('refreshed*');
+    expect(newPatternList).toContain('refreshed*');
+  });
+});
--- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.tsx
+++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.tsx
@ -14,8 +14,7 @@ import {
  EuiBadge,
  EuiScreenReaderOnly,
 } from '@elastic/eui';
-import { uniqBy } from 'lodash/fp';
-import { BrowserFields } from '@kbn/rule-registry-plugin/common';
+import type { BrowserFields } from '@kbn/rule-registry-plugin/common';
 import { EcsFlat } from '@elastic/ecs';
 import { EcsMetadata } from '@kbn/alerts-as-data-utils/src/field_maps/types';

@ -64,29 +63,41 @@ export const getFieldItemsData = ({
    selectedCategoryIds.length > 0 ? selectedCategoryIds : Object.keys(browserFields);
  const selectedFieldIds = new Set(columnIds);

-  const fieldItems = uniqBy(
-    'name',
-    categoryIds.reduce<BrowserFieldItem[]>((fieldItemsAcc, categoryId) => {
+  const getFieldItems = () => {
+    const fieldItemsAcc: BrowserFieldItem[] = [];
+    const fieldsSeen: Set<string> = new Set();
+
+    /**
+     * Both categoryIds and the fields in browserFields can be significantly large. Technically speaking,
+     * there is no upper cap on how many fields a customer can have. We are using a for loop here to avoid
+     * the performance issues that can arise from using map/filter/reduce.
+     */
+    for (let i = 0; i < categoryIds.length; i += 1) {
+      const categoryId = categoryIds[i];
      const categoryBrowserFields = Object.values(browserFields[categoryId]?.fields ?? {});
      if (categoryBrowserFields.length > 0) {
-        fieldItemsAcc.push(
-          ...categoryBrowserFields.map(({ name = '', ...field }) => {
-            return {
-              name,
-              type: field.type,
-              description: getDescription(name, EcsFlat as Record<string, EcsMetadata>),
-              example: field.example?.toString(),
-              category: getCategory(name),
-              selected: selectedFieldIds.has(name),
-              isRuntime: !!field.runtimeField,
-            };
-          })
-        );
+        for (let j = 0; j < categoryBrowserFields.length; j += 1) {
+          const field = categoryBrowserFields[j];
+          const name = field.name !== undefined ? field.name : '';
+          if (fieldsSeen.has(name)) continue;
+          fieldsSeen.add(name);
+          const categoryFieldItem = {
+            name,
+            type: field.type,
+            description: getDescription(name, EcsFlat as Record<string, EcsMetadata>),
+            example: field.example?.toString(),
+            category: getCategory(name),
+            selected: selectedFieldIds.has(name),
+            isRuntime: !!field.runtimeField,
+          };
+          fieldItemsAcc.push(categoryFieldItem);
+        }
      }
-      return fieldItemsAcc;
-    }, [])
-  );
-  return { fieldItems };
+    }
+    return fieldItemsAcc;
+  };
+
+  return { fieldItems: getFieldItems() };
 };

 const getDefaultFieldTableColumns = ({ highlight }: { highlight: string }): FieldTableColumns => {