[8.11] [Fleet] Replace dataviews suggestions in KQL searchboxes with internal ones (#172190) (#172437)

# Backport This will backport the following commits from `main` to `8.11`: - [[Fleet] Replace dataviews suggestions in KQL searchboxes with internal ones (#172190)](https://github.com/elastic/kibana/pull/172190)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
2025-04-23 01:13:23 -04:00 · 2023-12-04 16:22:55 +01:00 · 2023-12-04 16:22:55 +01:00 · 1258c38a8e
commit 1258c38a8e
parent be9d20037d
6 changed files with 328 additions and 146 deletions
--- a/x-pack/plugins/fleet/common/constants/index.ts
+++ b/x-pack/plugins/fleet/common/constants/index.ts
@ -55,3 +55,5 @@ export const FLEET_SERVER_INDICES = [
 export const DATA_TIERS = ['data_hot'];

 export const FLEET_ENROLLMENT_API_PREFIX = 'fleet-enrollment-api-keys';
+
+export * from './mappings';
--- a/x-pack/plugins/fleet/common/constants/mappings.ts
+++ b/x-pack/plugins/fleet/common/constants/mappings.ts
@ -6,7 +6,7 @@
 */

 /**
- * ATTENTION: New mappings for Fleet are defined in the ElasticSearch repo.
+ * ATTENTION: Mappings for Fleet are defined in the ElasticSearch repo.
 *
 * The following mappings declared here closely mirror them
 * But they are only used to perform validation on the endpoints using ListWithKuery
--- a/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.test.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.test.tsx
@ -6,12 +6,20 @@
 */

 import React from 'react';
+import { act } from '@testing-library/react';

 import type { FieldSpec } from '@kbn/data-plugin/common';

 import { createFleetTestRendererMock } from '../../../mock';

-import { SearchBar, filterAndConvertFields } from './search_bar';
+import {
+  AGENTS_PREFIX,
+  FLEET_ENROLLMENT_API_PREFIX,
+  PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+  AGENT_POLICY_SAVED_OBJECT_TYPE,
+} from '../constants';
+
+import { SearchBar, getFieldSpecs } from './search_bar';

 const fields = [
  {
@ -36,39 +44,6 @@ const fields = [
  },
 ] as FieldSpec[];

-const allFields = [
-  {
-    name: 'test-index._id',
-    type: 'string',
-    esTypes: ['_id'],
-  },
-  {
-    name: 'test-index.api_key',
-    type: 'string',
-    esTypes: ['keyword'],
-  },
-  {
-    name: 'test-index.name',
-    type: 'string',
-    esTypes: ['keyword'],
-  },
-  {
-    name: 'another-index.version',
-    type: 'string',
-    esTypes: ['keyword'],
-  },
-  {
-    name: 'test2-index.name',
-    type: 'string',
-    esTypes: ['keyword'],
-  },
-  {
-    name: 'fleet-agents.actions',
-    type: 'string',
-    esTypes: ['keyword'],
-  },
-] as FieldSpec[];
-
 jest.mock('../hooks', () => {
  return {
    ...jest.requireActual('../hooks'),
@ -87,23 +62,6 @@ jest.mock('../hooks', () => {
      },
      data: {
        dataViews: {
-          getFieldsForWildcard: jest.fn().mockResolvedValue([
-            {
-              name: '_id',
-              type: 'string',
-              esTypes: ['_id'],
-            },
-            {
-              name: 'api_key',
-              type: 'string',
-              esTypes: ['keyword'],
-            },
-            {
-              name: 'name',
-              type: 'string',
-              esTypes: ['keyword'],
-            },
-          ]),
          create: jest.fn().mockResolvedValue({
            fields,
          }),
@ -197,6 +155,9 @@ describe('SearchBar', () => {
  );

  it('renders the search box', async () => {
+    await act(async () => {
+      result.queryByTestId('queryInput');
+    });
    const textArea = result.queryByTestId('queryInput');
    expect(textArea).not.toBeNull();
    expect(textArea?.getAttribute('placeholder')).toEqual('Filter your data using KQL syntax');
@ -207,60 +168,243 @@ describe('SearchBar', () => {
  });
 });

-describe('filterAndConvertFields', () => {
-  it('leaves the fields names unchanged and does not hide any fields if fieldPrefix is not passed', async () => {
-    expect(filterAndConvertFields(fields, '.test-index')).toEqual({
-      _id: { esTypes: ['_id'], name: '_id', type: 'string' },
-      api_key: { esTypes: ['keyword'], name: 'api_key', type: 'string' },
-      name: { esTypes: ['keyword'], name: 'name', type: 'string' },
-      version: { esTypes: ['keyword'], name: 'version', type: 'string' },
-    });
+describe('getFieldSpecs', () => {
+  it('returns fieldSpecs for fleet-agents', () => {
+    expect(getFieldSpecs(`.${AGENTS_PREFIX}`)).toHaveLength(57);
+  });
+  it('returns getFieldSpecs for fleet-enrollment-api-keys', () => {
+    const indexPattern = `.${FLEET_ENROLLMENT_API_PREFIX}`;
+    expect(getFieldSpecs(indexPattern)).toHaveLength(8);
+    expect(getFieldSpecs(indexPattern)).toEqual([
+      {
+        aggregatable: true,
+        esTypes: ['boolean'],
+        name: 'active',
+        searchable: true,
+        type: 'boolean',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'api_key',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'api_key_id',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['date'],
+        name: 'created_at',
+        searchable: true,
+        type: 'date',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['date'],
+        name: 'expire_at',
+        searchable: true,
+        type: 'date',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'name',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'policy_id',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['date'],
+        name: 'updated_at',
+        searchable: true,
+        type: 'date',
+      },
+    ]);
  });

-  it('filters out the fields from other indices if indexPattern === .kibana-ingest', async () => {
-    expect(filterAndConvertFields(allFields, '.kibana_ingest', 'test-index')).toEqual({
-      'test-index._id': { esTypes: ['_id'], name: 'test-index._id', type: 'string' },
-      'test-index.api_key': { esTypes: ['keyword'], name: 'test-index.api_key', type: 'string' },
-      'test-index.name': { esTypes: ['keyword'], name: 'test-index.name', type: 'string' },
-    });
+  it('returns getFieldSpecs for fleet-agent-policy', () => {
+    const indexPattern = `.${AGENT_POLICY_SAVED_OBJECT_TYPE}`;
+    expect(getFieldSpecs(indexPattern)).toHaveLength(23);
+    expect(getFieldSpecs(indexPattern)).toEqual([
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'agent_features.name',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['boolean'],
+        name: 'agent_features.enabled',
+        searchable: true,
+        type: 'boolean',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'data_output_id',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['text'],
+        name: 'description',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'download_source_id',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'fleet_server_host_id',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['integer'],
+        name: 'inactivity_timeout',
+        searchable: true,
+        type: 'number',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['boolean'],
+        name: 'is_default',
+        searchable: true,
+        type: 'boolean',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['boolean'],
+        name: 'is_default_fleet_server',
+        searchable: true,
+        type: 'boolean',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['boolean'],
+        name: 'is_managed',
+        searchable: true,
+        type: 'boolean',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'is_preconfigured',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['boolean'],
+        name: 'is_protected',
+        searchable: true,
+        type: 'boolean',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'monitoring_enabled',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['false'],
+        name: 'monitoring_enabled.index',
+        searchable: true,
+        type: 'false',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'monitoring_output_id',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'name',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'namespace',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['integer'],
+        name: 'revision',
+        searchable: true,
+        type: 'number',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['version'],
+        name: 'schema_version',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'status',
+        searchable: true,
+        type: 'string',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['integer'],
+        name: 'unenroll_timeout',
+        searchable: true,
+        type: 'number',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['date'],
+        name: 'updated_at',
+        searchable: true,
+        type: 'date',
+      },
+      {
+        aggregatable: true,
+        esTypes: ['keyword'],
+        name: 'updated_by',
+        searchable: true,
+        type: 'string',
+      },
+    ]);
  });
+  expect(getFieldSpecs(`.${PACKAGE_POLICY_SAVED_OBJECT_TYPE}`)).toHaveLength(18);

-  it('returns fields unchanged if fieldPrefix and indexPattern are not passed', async () => {
-    expect(filterAndConvertFields(allFields, undefined, undefined)).toEqual({
-      'another-index.version': {
-        esTypes: ['keyword'],
-        name: 'another-index.version',
-        type: 'string',
-      },
-      'fleet-agents.actions': {
-        esTypes: ['keyword'],
-        name: 'fleet-agents.actions',
-        type: 'string',
-      },
-      'test-index._id': {
-        esTypes: ['_id'],
-        name: 'test-index._id',
-        type: 'string',
-      },
-      'test-index.api_key': {
-        esTypes: ['keyword'],
-        name: 'test-index.api_key',
-        type: 'string',
-      },
-      'test-index.name': {
-        esTypes: ['keyword'],
-        name: 'test-index.name',
-        type: 'string',
-      },
-      'test2-index.name': {
-        esTypes: ['keyword'],
-        name: 'test2-index.name',
-        type: 'string',
-      },
-    });
-  });
-
-  it('returns empty object if fields is empty', async () => {
-    expect(filterAndConvertFields([], '.kibana_ingest', 'test-index')).toEqual({});
+  it('returns empty array if indexPattern is not one of the previous', async () => {
+    expect(getFieldSpecs('.kibana_ingest')).toEqual([]);
  });
 });
--- a/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.tsx
@ -16,9 +16,19 @@ import type { DataView } from '@kbn/data-views-plugin/public';
 import { i18n } from '@kbn/i18n';

 import { useStartServices } from '../hooks';
-import { INDEX_NAME, AGENTS_PREFIX } from '../constants';
-
-const HIDDEN_FIELDS = [`${AGENTS_PREFIX}.actions`, '_id', '_index'];
+import {
+  INDEX_NAME,
+  AGENTS_PREFIX,
+  FLEET_ENROLLMENT_API_PREFIX,
+  PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+  AGENT_POLICY_SAVED_OBJECT_TYPE,
+} from '../constants';
+import {
+  AGENT_POLICY_MAPPINGS,
+  PACKAGE_POLICIES_MAPPINGS,
+  AGENT_MAPPINGS,
+  ENROLLMENT_API_KEY_MAPPINGS,
+} from '../../../../common/constants';

 const NoWrapQueryStringInput = styled(QueryStringInput)`
  .kbnQueryBar__textarea {
@ -35,39 +45,69 @@ interface Props {
  dataTestSubj?: string;
 }

-/** Exported for testing only **/
-export const filterAndConvertFields = (
-  fields: FieldSpec[],
-  indexPattern?: string,
-  fieldPrefix?: string
-) => {
-  if (!fields) return {};
-  let filteredFields: FieldSpec[] = [];
-
-  if (fieldPrefix) {
-    // exclude fields from different indices
-    if (indexPattern === INDEX_NAME) {
-      filteredFields = fields.filter((field) => field.name.startsWith(fieldPrefix));
-    } else {
-      // filter out fields that have names to be hidden
-      filteredFields = fields.filter((field) => {
-        for (const hiddenField of HIDDEN_FIELDS) {
-          if (field.name.includes(hiddenField)) {
-            return false;
-          }
-        }
-        return true;
-      });
-    }
-  } else {
-    filteredFields = fields;
+const getMappings = (indexPattern: string) => {
+  switch (indexPattern) {
+    case `.${AGENTS_PREFIX}`:
+      return AGENT_MAPPINGS;
+    case `.${AGENT_POLICY_SAVED_OBJECT_TYPE}`:
+      return AGENT_POLICY_MAPPINGS;
+    case `.${PACKAGE_POLICY_SAVED_OBJECT_TYPE}`:
+      return PACKAGE_POLICIES_MAPPINGS;
+    case `.${FLEET_ENROLLMENT_API_PREFIX}`:
+      return ENROLLMENT_API_KEY_MAPPINGS;
+    default:
+      return {};
  }
+};

-  const fieldsMap = filteredFields.reduce((acc: Record<string, FieldSpec>, curr: FieldSpec) => {
-    acc[curr.name] = curr;
-    return acc;
-  }, {});
-  return fieldsMap;
+const getType = (type: string) => {
+  switch (type) {
+    case 'keyword':
+      return 'string';
+    case 'text':
+      return 'string';
+    case 'version':
+      return 'string';
+    case 'integer':
+      return 'number';
+    case 'double':
+      return 'number';
+    default:
+      return type;
+  }
+};
+
+const concatKeys = (obj: any, parentKey = '') => {
+  let result: string[] = [];
+  for (const key in obj) {
+    if (typeof obj[key] === 'object') {
+      result = result.concat(concatKeys(obj[key], `${parentKey}${key}.`));
+    } else {
+      result.push(`${parentKey}${key}:${obj[key]}`);
+    }
+  }
+  return result;
+};
+/** Exported for testing only **/
+export const getFieldSpecs = (indexPattern: string) => {
+  const mapping = getMappings(indexPattern);
+  // @ts-ignore-next-line
+  const rawFields = concatKeys(mapping?.properties) || [];
+  const fields = rawFields
+    .map((field) => field.replaceAll(/.properties/g, ''))
+    .map((field) => field.replace(/.type/g, ''))
+    .map((field) => field.split(':'));
+
+  const fieldSpecs: FieldSpec[] = fields.map((field) => {
+    return {
+      name: field[0],
+      type: getType(field[1]),
+      searchable: true,
+      aggregatable: true,
+      esTypes: [field[1]],
+    };
+  });
+  return fieldSpecs;
 };

 export const SearchBar: React.FunctionComponent<Props> = ({
@ -108,16 +148,11 @@ export const SearchBar: React.FunctionComponent<Props> = ({
  useEffect(() => {
    const fetchFields = async () => {
      try {
-        const fields: FieldSpec[] = await data.dataViews.getFieldsForWildcard({
-          pattern: indexPattern,
-        });
-        const fieldsMap = filterAndConvertFields(fields, indexPattern, fieldPrefix);
-        // Refetch only if fieldsMap is empty
-        const skipFetchField = !!fieldsMap;
-
+        const fieldSpecs = getFieldSpecs(indexPattern);
+        const fieldsMap = data.dataViews.fieldArrayToMap(fieldSpecs);
        const newDataView = await data.dataViews.create(
          { title: indexPattern, fields: fieldsMap },
-          skipFetchField
+          true
        );
        setDataView(newDataView);
      } catch (err) {
--- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/list_page/index.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/list_page/index.tsx
@ -260,6 +260,7 @@ export const AgentPolicyListPage: React.FunctionComponent<{}> = () => {
              });
              setSearch(newSearch);
            }}
+            indexPattern={`.${AGENT_POLICY_SAVED_OBJECT_TYPE}`}
            fieldPrefix={AGENT_POLICY_SAVED_OBJECT_TYPE}
            dataTestSubj="agentPolicyList.queryInput"
          />
--- a/x-pack/plugins/fleet/server/constants/index.ts
+++ b/x-pack/plugins/fleet/server/constants/index.ts
@ -98,4 +98,4 @@ export {
 } from './fleet_es_assets';
 export { FILE_STORAGE_DATA_AGENT_INDEX } from './fleet_es_assets';
 export { FILE_STORAGE_METADATA_AGENT_INDEX } from './fleet_es_assets';
-export * from './mappings';
+export * from '../../common/constants/mappings';