[OnWeek] Show ECS field descriptions in Discover. Add markdown support for field descriptions. (#187160)

- Closes #186818
- Closes #97246

## Summary

This PR adds fetching and rendering of ECS field descriptions to:
- field list sidebar
- doc viewer

<img width="664" alt="Screenshot 2024-07-12 at 17 04 36"
src="https://github.com/user-attachments/assets/e9984797-1bc4-4651-8924-d90d734d76f5">
<img width="629" alt="Screenshot 2024-07-12 at 17 05 07"
src="https://github.com/user-attachments/assets/dd472f7e-0ec8-4d5d-b96f-afc19b52a478">


It's based on the new `fieldsMetadata` service
https://github.com/elastic/kibana/pull/183806

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [x] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-field-utils/src/components/field_description/field_description.test.tsx

@@ -9,17 +9,18 @@
 import React from 'react';
 import { FieldDescription } from './field_description';
 import { render, screen } from '@testing-library/react';
+import { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
 
 describe('FieldDescription', () => {
   it('should render correctly when no custom description', async () => {
-    render(<FieldDescription field={{ name: 'bytes' }} />);
+    render(<FieldDescription field={{ name: 'bytes', type: 'number' }} />);
     const desc = screen.queryByTestId('fieldDescription-bytes');
     expect(desc).toBeNull();
   });
 
   it('should render correctly with a short custom description', async () => {
     const customDescription = 'test this desc';
-    render(<FieldDescription field={{ name: 'bytes', customDescription }} />);
+    render(<FieldDescription field={{ name: 'bytes', type: 'number', customDescription }} />);
     const desc = screen.queryByTestId('fieldDescription-bytes');
     expect(desc).toHaveTextContent(customDescription);
     const button = screen.queryByTestId('toggleFieldDescription-bytes');
@@ -28,7 +29,7 @@ describe('FieldDescription', () => {
 
   it('should render correctly with a long custom description', async () => {
     const customDescription = 'test this long desc '.repeat(8).trim();
-    render(<FieldDescription field={{ name: 'bytes', customDescription }} />);
+    render(<FieldDescription field={{ name: 'bytes', type: 'number', customDescription }} />);
     expect(screen.queryByTestId('fieldDescription-bytes')).toHaveTextContent(customDescription);
     screen.queryByTestId('toggleFieldDescription-bytes')?.click();
     expect(screen.queryByTestId('fieldDescription-bytes')).toHaveTextContent(
@@ -40,9 +41,106 @@ describe('FieldDescription', () => {
 
   it('should render a long custom description without truncation', async () => {
     const customDescription = 'test this long desc '.repeat(8).trim();
-    render(<FieldDescription field={{ name: 'bytes', customDescription }} truncate={false} />);
+    render(
+      <FieldDescription
+        field={{ name: 'bytes', type: 'number', customDescription }}
+        truncate={false}
+      />
+    );
     expect(screen.queryByTestId('fieldDescription-bytes')).toHaveTextContent(customDescription);
     const button = screen.queryByTestId('toggleFieldDescription-bytes');
     expect(button).toBeNull();
   });
+
+  it('should render correctly with markdown', async () => {
+    const fieldsMetadataService: Partial<FieldsMetadataPublicStart> = {
+      useFieldsMetadata: jest.fn(() => ({
+        fieldsMetadata: {
+          bytes: { description: 'ESC desc', type: 'long' },
+        },
+        loading: false,
+        error: undefined,
+        reload: jest.fn(),
+      })),
+    };
+    const customDescription = 'test this `markdown` desc';
+    render(
+      <FieldDescription
+        field={{ name: 'bytes', type: 'number', customDescription }}
+        fieldsMetadataService={fieldsMetadataService as FieldsMetadataPublicStart}
+      />
+    );
+    const desc = screen.queryByTestId('fieldDescription-bytes');
+    expect(desc).toHaveTextContent('test this markdown desc');
+    expect(fieldsMetadataService.useFieldsMetadata).not.toHaveBeenCalled();
+  });
+
+  it('should fetch ECS metadata', async () => {
+    const fieldsMetadataService: Partial<FieldsMetadataPublicStart> = {
+      useFieldsMetadata: jest.fn(() => ({
+        fieldsMetadata: {
+          bytes: { description: 'ESC desc', type: 'long' },
+        },
+        loading: false,
+        error: undefined,
+        reload: jest.fn(),
+      })),
+    };
+    render(
+      <FieldDescription
+        field={{ name: 'bytes', type: 'number', customDescription: undefined }}
+        fieldsMetadataService={fieldsMetadataService as FieldsMetadataPublicStart}
+      />
+    );
+    const desc = screen.queryByTestId('fieldDescription-bytes');
+    expect(desc).toHaveTextContent('ESC desc');
+    expect(fieldsMetadataService.useFieldsMetadata).toHaveBeenCalledWith({
+      attributes: ['description', 'type'],
+      fieldNames: ['bytes'],
+    });
+  });
+
+  it('should not show ECS metadata if types do not match', async () => {
+    const fieldsMetadataService: Partial<FieldsMetadataPublicStart> = {
+      useFieldsMetadata: jest.fn(() => ({
+        fieldsMetadata: {
+          bytes: { description: 'ESC desc', type: 'keyword' },
+        },
+        loading: false,
+        error: undefined,
+        reload: jest.fn(),
+      })),
+    };
+    render(
+      <FieldDescription
+        field={{ name: 'bytes', type: 'number', customDescription: undefined }}
+        fieldsMetadataService={fieldsMetadataService as FieldsMetadataPublicStart}
+      />
+    );
+    const desc = screen.queryByTestId('fieldDescription-bytes');
+    expect(desc).toBeNull();
+  });
+
+  it('should not show ECS metadata if none found', async () => {
+    const fieldsMetadataService: Partial<FieldsMetadataPublicStart> = {
+      useFieldsMetadata: jest.fn(() => ({
+        fieldsMetadata: {},
+        loading: false,
+        error: undefined,
+        reload: jest.fn(),
+      })),
+    };
+    render(
+      <FieldDescription
+        field={{ name: 'extension.keyword', type: 'keyword', customDescription: undefined }}
+        fieldsMetadataService={fieldsMetadataService as FieldsMetadataPublicStart}
+      />
+    );
+    const desc = screen.queryByTestId('fieldDescription-extension.keyword');
+    expect(desc).toBeNull();
+    expect(fieldsMetadataService.useFieldsMetadata).toHaveBeenCalledWith({
+      attributes: ['description', 'type'],
+      fieldNames: ['extension'],
+    });
+  });
 });

packages/kbn-field-utils/src/components/field_description/field_description.tsx

@@ -8,27 +8,81 @@
 
 import React, { useState } from 'react';
 import { i18n } from '@kbn/i18n';
-import { EuiText, EuiButtonEmpty, EuiTextBlockTruncate, useEuiTheme } from '@elastic/eui';
+import { Markdown } from '@kbn/shared-ux-markdown';
+import {
+  EuiText,
+  EuiButtonEmpty,
+  EuiTextBlockTruncate,
+  EuiSkeletonText,
+  useEuiTheme,
+} from '@elastic/eui';
 import { css } from '@emotion/react';
+import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
+import { esFieldTypeToKibanaFieldType } from '@kbn/field-types';
 
 const MAX_VISIBLE_LENGTH = 110;
 
-export interface FieldDescriptionProps {
+const removeKeywordSuffix = (name: string) => {
+  return name.endsWith('.keyword') ? name.slice(0, -8) : name;
+};
+
+export interface FieldDescriptionContentProps {
   field: {
     name: string;
     customDescription?: string;
+    type: string;
   };
   color?: 'subdued';
   truncate?: boolean;
+  Wrapper?: React.FC<{ children: React.ReactNode }>;
+}
+
+export interface FieldDescriptionProps extends FieldDescriptionContentProps {
+  fieldsMetadataService?: FieldsMetadataPublicStart;
 }
 
 export const FieldDescription: React.FC<FieldDescriptionProps> = ({
-  field,
-  color,
-  truncate = true,
+  fieldsMetadataService,
+  ...props
 }) => {
+  if (fieldsMetadataService && !props.field.customDescription) {
+    return <EcsFieldDescriptionFallback fieldsMetadataService={fieldsMetadataService} {...props} />;
+  }
+
+  return <FieldDescriptionContent {...props} />;
+};
+
+const EcsFieldDescriptionFallback: React.FC<
+  FieldDescriptionProps & { fieldsMetadataService: FieldsMetadataPublicStart }
+> = ({ fieldsMetadataService, ...props }) => {
+  const fieldName = removeKeywordSuffix(props.field.name);
+  const { fieldsMetadata, loading } = fieldsMetadataService.useFieldsMetadata({
+    attributes: ['description', 'type'],
+    fieldNames: [fieldName],
+  });
+
+  const escFieldDescription = fieldsMetadata?.[fieldName]?.description;
+  const escFieldType = fieldsMetadata?.[fieldName]?.type;
+
+  return (
+    <EuiSkeletonText isLoading={loading} size="s">
+      <FieldDescriptionContent
+        {...props}
+        ecsFieldDescription={
+          escFieldType && esFieldTypeToKibanaFieldType(escFieldType) === props.field.type
+            ? escFieldDescription
+            : undefined
+        }
+      />
+    </EuiSkeletonText>
+  );
+};
+
+export const FieldDescriptionContent: React.FC<
+  FieldDescriptionContentProps & { ecsFieldDescription?: string }
+> = ({ field, color, truncate = true, ecsFieldDescription, Wrapper }) => {
   const { euiTheme } = useEuiTheme();
-  const customDescription = (field?.customDescription || '').trim();
+  const customDescription = (field?.customDescription || ecsFieldDescription || '').trim();
   const isTooLong = Boolean(truncate && customDescription.length > MAX_VISIBLE_LENGTH);
   const [isTruncated, setIsTruncated] = useState<boolean>(isTooLong);
 
@@ -36,7 +90,7 @@ export const FieldDescription: React.FC<FieldDescriptionProps> = ({
     return null;
   }
 
-  return (
+  const result = (
     <div data-test-subj={`fieldDescription-${field.name}`}>
       {isTruncated ? (
         <EuiText color={color} size="xs" className="eui-textBreakWord eui-textLeft">
@@ -61,13 +115,15 @@ export const FieldDescription: React.FC<FieldDescriptionProps> = ({
               }
             `}
           >
-            <EuiTextBlockTruncate lines={2}>{customDescription}</EuiTextBlockTruncate>
+            <EuiTextBlockTruncate lines={2}>
+              <Markdown readOnly>{customDescription}</Markdown>
+            </EuiTextBlockTruncate>
           </button>
         </EuiText>
       ) : (
         <>
           <EuiText color={color} size="xs" className="eui-textBreakWord eui-textLeft">
-            {customDescription}
+            <Markdown readOnly>{customDescription}</Markdown>
           </EuiText>
           {isTooLong && (
             <EuiButtonEmpty
@@ -85,4 +141,6 @@ export const FieldDescription: React.FC<FieldDescriptionProps> = ({
       )}
     </div>
   );
+
+  return Wrapper ? <Wrapper>{result}</Wrapper> : result;
 };

packages/kbn-field-utils/tsconfig.json

@@ -11,6 +11,8 @@
     "@kbn/field-types",
     "@kbn/expressions-plugin",
     "@kbn/data-view-utils",
+    "@kbn/fields-metadata-plugin",
+    "@kbn/shared-ux-markdown",
   ],
   "exclude": ["target/**/*"]
 }

packages/kbn-unified-field-list/src/components/field_popover/field_popover_header.tsx

@@ -20,6 +20,7 @@ import {
 import { i18n } from '@kbn/i18n';
 import { FieldDescription } from '@kbn/field-utils';
 import type { DataViewField } from '@kbn/data-views-plugin/common';
+import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
 import type { AddFieldFilterHandler } from '../../types';
 
 export interface FieldPopoverHeaderProps {
@@ -33,6 +34,9 @@ export interface FieldPopoverHeaderProps {
   onAddFilter?: AddFieldFilterHandler;
   onEditField?: (fieldName: string) => unknown;
   onDeleteField?: (fieldName: string) => unknown;
+  services?: {
+    fieldsMetadata?: FieldsMetadataPublicStart;
+  };
 }
 
 export const FieldPopoverHeader: React.FC<FieldPopoverHeaderProps> = ({
@@ -46,6 +50,7 @@ export const FieldPopoverHeader: React.FC<FieldPopoverHeaderProps> = ({
   onAddFilter,
   onEditField,
   onDeleteField,
+  services,
 }) => {
   if (!field) {
     return null;
@@ -153,12 +158,20 @@ export const FieldPopoverHeader: React.FC<FieldPopoverHeaderProps> = ({
           </EuiFlexItem>
         )}
       </EuiFlexGroup>
-      {field.customDescription ? (
-        <>
-          <EuiSpacer size="xs" />
-          <FieldDescription field={field} />
-        </>
-      ) : null}
+      <FieldDescription
+        field={field}
+        Wrapper={FieldDescriptionWrapper}
+        fieldsMetadataService={services?.fieldsMetadata}
+      />
+    </>
+  );
+};
+
+const FieldDescriptionWrapper: React.FC = ({ children }) => {
+  return (
+    <>
+      <EuiSpacer size="xs" />
+      {children}
     </>
   );
 };

packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx

@@ -10,6 +10,7 @@ import React, { memo, useCallback, useMemo, useState } from 'react';
 import { EuiSpacer, EuiTitle } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { UiCounterMetricType } from '@kbn/analytics';
+import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
 import { Draggable } from '@kbn/dom-drag-drop';
 import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
 import type { SearchMode } from '../../types';
@@ -119,6 +120,7 @@ export interface UnifiedFieldListItemProps {
    */
   services: UnifiedFieldListItemStatsProps['services'] & {
     uiActions?: FieldPopoverFooterProps['uiActions'];
+    fieldsMetadata?: FieldsMetadataPublicStart;
   };
   /**
    * Current search mode
@@ -367,6 +369,7 @@ function UnifiedFieldListItemComponent({
       data-test-subj={stateService.creationOptions.dataTestSubj?.fieldListItemPopoverDataTestSubj}
       renderHeader={() => (
         <FieldPopoverHeader
+          services={services}
           field={field}
           closePopover={closePopover}
           onAddFieldToWorkspace={!isSelected ? toggleDisplay : undefined}

packages/kbn-unified-field-list/tsconfig.json

@@ -33,7 +33,8 @@
     "@kbn/field-utils",
     "@kbn/visualization-utils",
     "@kbn/esql-utils",
-    "@kbn/search-types"
+    "@kbn/search-types",
+    "@kbn/fields-metadata-plugin"
   ],
   "exclude": ["target/**/*"]
 }

src/plugins/discover/kibana.jsonc

@@ -40,7 +40,8 @@
       "noDataPage",
       "globalSearch",
       "observabilityAIAssistant",
-      "aiops"
+      "aiops",
+      "fieldsMetadata"
     ],
     "requiredBundles": ["kibanaUtils", "kibanaReact", "unifiedSearch", "savedObjects"],
     "extraPublicDirs": ["common"]

src/plugins/discover/public/build_services.ts

@@ -56,6 +56,7 @@ import { memoize, noop } from 'lodash';
 import type { NoDataPagePluginStart } from '@kbn/no-data-page-plugin/public';
 import type { AiopsPluginStart } from '@kbn/aiops-plugin/public';
 import type { DataVisualizerPluginStart } from '@kbn/data-visualizer-plugin/public';
+import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
 import type { DiscoverStartPlugins } from './types';
 import type { DiscoverContextAppLocator } from './application/context/services/locator';
 import type { DiscoverSingleDocLocator } from './application/doc/locator';
@@ -128,6 +129,7 @@ export interface DiscoverServices {
   noDataPage?: NoDataPagePluginStart;
   observabilityAIAssistant?: ObservabilityAIAssistantPublicStart;
   profilesManager: ProfilesManager;
+  fieldsMetadata?: FieldsMetadataPublicStart;
 }
 
 export const buildServices = memoize(
@@ -214,6 +216,7 @@ export const buildServices = memoize(
       noDataPage: plugins.noDataPage,
       observabilityAIAssistant: plugins.observabilityAIAssistant,
       profilesManager,
+      fieldsMetadata: plugins.fieldsMetadata,
     };
   }
 );

src/plugins/discover/public/types.ts

@@ -39,6 +39,7 @@ import type {
 } from '@kbn/observability-ai-assistant-plugin/public';
 import type { AiopsPluginStart } from '@kbn/aiops-plugin/public';
 import type { DataVisualizerPluginStart } from '@kbn/data-visualizer-plugin/public';
+import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
 import { DiscoverAppLocator } from '../common';
 import { DiscoverCustomizationContext } from './customizations';
 import { type DiscoverContainerProps } from './components/discover_container';
@@ -167,4 +168,5 @@ export interface DiscoverStartPlugins {
   unifiedSearch: UnifiedSearchPublicPluginStart;
   urlForwarding: UrlForwardingStart;
   usageCollection?: UsageCollectionSetup;
+  fieldsMetadata: FieldsMetadataPublicStart;
 }

src/plugins/discover/tsconfig.json

@@ -92,7 +92,8 @@
     "@kbn/aiops-plugin",
     "@kbn/data-visualizer-plugin",
     "@kbn/search-types",
-    "@kbn/observability-ai-assistant-plugin"
+    "@kbn/observability-ai-assistant-plugin",
+    "@kbn/fields-metadata-plugin"
   ],
   "exclude": ["target/**/*"]
 }

src/plugins/unified_doc_viewer/public/components/doc_viewer_table/table.tsx

@@ -141,7 +141,7 @@ export const DocViewerTable = ({
   onRemoveColumn,
 }: DocViewRenderProps) => {
   const [containerRef, setContainerRef] = useState<HTMLDivElement | null>(null);
-  const { fieldFormats, storage, uiSettings } = getUnifiedDocViewerServices();
+  const { fieldFormats, storage, uiSettings, fieldsMetadata } = getUnifiedDocViewerServices();
   const showMultiFields = uiSettings.get(SHOW_MULTIFIELDS);
   const currentDataViewId = dataView.id!;
 
@@ -387,9 +387,13 @@ export const DocViewerTable = ({
               isPinned={pinned}
             />
 
-            {isDetails && fieldMapping?.customDescription ? (
+            {isDetails && !!fieldMapping ? (
               <div>
-                <FieldDescription field={fieldMapping} truncate={false} />
+                <FieldDescription
+                  fieldsMetadataService={fieldsMetadata}
+                  field={fieldMapping}
+                  truncate={false}
+                />
               </div>
             ) : null}
           </div>
@@ -409,7 +413,7 @@ export const DocViewerTable = ({
 
       return null;
     },
-    [rows, searchText]
+    [rows, searchText, fieldsMetadata]
   );
 
   const renderCellPopover = useCallback(

test/functional/apps/discover/group7/_runtime_fields_editor.ts

@@ -113,6 +113,51 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       await dataGrid.closeFlyout();
     });
 
+    it('allows to replace ECS description with a custom field description', async function () {
+      await PageObjects.unifiedFieldList.clickFieldListItem('@timestamp');
+      await retry.waitFor('field popover text', async () => {
+        return (await testSubjects.getVisibleText('fieldDescription-@timestamp')).startsWith(
+          'Date'
+        );
+      });
+      await PageObjects.unifiedFieldList.closeFieldPopover();
+      // check it in the doc viewer too
+      await dataGrid.clickRowToggle({ rowIndex: 0 });
+      await dataGrid.expandFieldNameCellInFlyout('@timestamp');
+      await retry.waitFor('doc viewer popover text', async () => {
+        return (await testSubjects.getVisibleText('fieldDescription-@timestamp')).startsWith(
+          'Date'
+        );
+      });
+      await dataGrid.closeFlyout();
+
+      const customDescription = 'custom @timestamp description here';
+      // set a custom description
+      await PageObjects.discover.editField('@timestamp');
+      await fieldEditor.enableCustomDescription();
+      await fieldEditor.setCustomDescription(customDescription);
+      await fieldEditor.save();
+      await fieldEditor.waitUntilClosed();
+      await PageObjects.header.waitUntilLoadingHasFinished();
+      await PageObjects.unifiedFieldList.clickFieldListItem('@timestamp');
+      await retry.waitFor('field popover text', async () => {
+        return (
+          (await testSubjects.getVisibleText('fieldDescription-@timestamp')) === customDescription
+        );
+      });
+      await PageObjects.unifiedFieldList.closeFieldPopover();
+      // check it in the doc viewer too
+      await dataGrid.clickRowToggle({ rowIndex: 0 });
+      await dataGrid.expandFieldNameCellInFlyout('@timestamp');
+      await retry.waitFor('doc viewer popover text', async () => {
+        return (
+          (await testSubjects.getVisibleText('fieldDescription-@timestamp')) === customDescription
+        );
+      });
+
+      await dataGrid.closeFlyout();
+    });
+
     it('should show a validation error when adding a too long custom description to existing fields', async function () {
       const customDescription = 'custom bytes long description here'.repeat(10);
       // set a custom description