[Discover] Show correct data for top level object columns (#91954)

* [Discover] Show correct data for top level object columns

* Fix bug with missing fields

* Fix bug in data grid

* Fix remaining bug in datagrid

* Change use of API to work with any type

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

src/plugins/discover/public/__mocks__/index_pattern.ts

@@ -52,6 +52,13 @@ const fields = [
     scripted: true,
     filterable: false,
   },
+  {
+    name: 'object.value',
+    type: 'number',
+    scripted: false,
+    filterable: true,
+    aggregatable: true,
+  },
 ] as IIndexPatternFieldList;
 
 fields.getByName = (name: string) => {
@@ -64,13 +71,14 @@ const indexPattern = ({
   metaFields: ['_index', '_score'],
   formatField: jest.fn(),
   flattenHit: undefined,
-  formatHit: jest.fn((hit) => hit._source),
+  formatHit: jest.fn((hit) => (hit.fields ? hit.fields : hit._source)),
   fields,
   getComputedFields: () => ({ docvalueFields: [], scriptFields: {}, storedFields: ['*'] }),
   getSourceFiltering: () => ({}),
-  getFieldByName: () => ({}),
+  getFieldByName: jest.fn(() => ({})),
   timeFieldName: '',
   docvalueFields: [],
+  getFormatterForField: () => ({ convert: () => 'formatted' }),
 } as unknown) as IndexPattern;
 
 indexPattern.flattenHit = indexPatterns.flattenHitWrapper(indexPattern, indexPattern.metaFields);

src/plugins/discover/public/application/angular/doc_table/components/table_row.ts

@@ -16,7 +16,7 @@ import cellTemplateHtml from '../components/table_row/cell.html';
 import truncateByHeightTemplateHtml from '../components/table_row/truncate_by_height.html';
 import { getServices } from '../../../../kibana_services';
 import { getContextUrl } from '../../../helpers/get_context_url';
-import { formatRow } from '../../helpers';
+import { formatRow, formatTopLevelObject } from '../../helpers';
 
 const TAGS_WITH_WS = />\s+</g;
 
@@ -145,16 +145,32 @@ export function createTableRowDirective($compile: ng.ICompileService) {
         } else {
           $scope.columns.forEach(function (column: string) {
             const isFilterable = mapping(column) && mapping(column).filterable && $scope.filter;
-
+            if ($scope.useNewFieldsApi && !mapping(column) && !row.fields[column]) {
-            newHtmls.push(
+              const innerColumns = Object.fromEntries(
-              cellTemplate({
+                Object.entries(row.fields).filter(([key]) => {
-                timefield: false,
+                  return key.indexOf(`${column}.`) === 0;
-                sourcefield: column === '_source',
+                })
-                formatted: _displayField(row, column, true),
+              );
-                filterable: isFilterable,
+              newHtmls.push(
-                column,
+                cellTemplate({
-              })
+                  timefield: false,
-            );
+                  sourcefield: true,
+                  formatted: formatTopLevelObject(row, innerColumns, indexPattern),
+                  filterable: false,
+                  column,
+                })
+              );
+            } else {
+              newHtmls.push(
+                cellTemplate({
+                  timefield: false,
+                  sourcefield: column === '_source',
+                  formatted: _displayField(row, column, true),
+                  filterable: isFilterable,
+                  column,
+                })
+              );
+            }
           });
         }
 

src/plugins/discover/public/application/angular/helpers/index.ts

@@ -7,5 +7,5 @@
  */
 
 export { buildPointSeriesData } from './point_series';
-export { formatRow } from './row_formatter';
+export { formatRow, formatTopLevelObject } from './row_formatter';
 export { handleSourceColumnState } from './state_helpers';

src/plugins/discover/public/application/angular/helpers/row_formatter.test.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { formatRow } from './row_formatter';
+import { formatRow, formatTopLevelObject } from './row_formatter';
 import { stubbedSavedObjectIndexPattern } from '../../../__mocks__/stubbed_saved_object_index_pattern';
 import { IndexPattern } from '../../../../../data/common/index_patterns/index_patterns';
 import { fieldFormatsMock } from '../../../../../data/common/field_formats/mocks';
@@ -43,16 +43,97 @@ describe('Row formatter', () => {
     foo: 'bar',
     hello: '<h1>World</h1>',
   };
-  const formatHitMock = jest.fn().mockReturnValueOnce(formatHitReturnValue);
+  const formatHitMock = jest.fn().mockReturnValue(formatHitReturnValue);
 
   beforeEach(() => {
-    // @ts-ignore
+    // @ts-expect-error
     indexPattern.formatHit = formatHitMock;
   });
 
   it('formats document properly', () => {
-    expect(formatRow(hit, indexPattern).trim()).toBe(
+    expect(formatRow(hit, indexPattern).trim()).toMatchInlineSnapshot(
-      '<dl class="source truncate-by-height"><dt>also:</dt><dd>with \\&quot;quotes\\&quot; or &#39;single qoutes&#39;</dd> <dt>number:</dt><dd>42</dd> <dt>foo:</dt><dd>bar</dd> <dt>hello:</dt><dd>&lt;h1&gt;World&lt;/h1&gt;</dd> </dl>'
+      `"<dl class=\\"source truncate-by-height\\"><dt>also:</dt><dd>with \\\\&quot;quotes\\\\&quot; or &#39;single qoutes&#39;</dd> <dt>number:</dt><dd>42</dd> <dt>foo:</dt><dd>bar</dd> <dt>hello:</dt><dd>&lt;h1&gt;World&lt;/h1&gt;</dd> </dl>"`
+    );
+  });
+
+  it('formats document with highlighted fields first', () => {
+    expect(
+      formatRow({ ...hit, highlight: { number: '42' } }, indexPattern).trim()
+    ).toMatchInlineSnapshot(
+      `"<dl class=\\"source truncate-by-height\\"><dt>number:</dt><dd>42</dd> <dt>also:</dt><dd>with \\\\&quot;quotes\\\\&quot; or &#39;single qoutes&#39;</dd> <dt>foo:</dt><dd>bar</dd> <dt>hello:</dt><dd>&lt;h1&gt;World&lt;/h1&gt;</dd> </dl>"`
+    );
+  });
+
+  it('formats top level objects using formatter', () => {
+    indexPattern.getFieldByName = jest.fn().mockReturnValue({
+      name: 'subfield',
+    });
+    indexPattern.getFormatterForField = jest.fn().mockReturnValue({
+      convert: () => 'formatted',
+    });
+    expect(
+      formatTopLevelObject(
+        {
+          fields: {
+            'object.value': [5, 10],
+          },
+        },
+        {
+          'object.value': [5, 10],
+        },
+        indexPattern
+      ).trim()
+    ).toMatchInlineSnapshot(
+      `"<dl class=\\"source truncate-by-height\\"><dt>object.value:</dt><dd>formatted, formatted</dd> </dl>"`
+    );
+  });
+
+  it('formats top level objects with subfields and highlights', () => {
+    indexPattern.getFieldByName = jest.fn().mockReturnValue({
+      name: 'subfield',
+    });
+    indexPattern.getFormatterForField = jest.fn().mockReturnValue({
+      convert: () => 'formatted',
+    });
+    expect(
+      formatTopLevelObject(
+        {
+          fields: {
+            'object.value': [5, 10],
+            'object.keys': ['a', 'b'],
+          },
+          highlight: {
+            'object.keys': 'a',
+          },
+        },
+        {
+          'object.value': [5, 10],
+          'object.keys': ['a', 'b'],
+        },
+        indexPattern
+      ).trim()
+    ).toMatchInlineSnapshot(
+      `"<dl class=\\"source truncate-by-height\\"><dt>object.keys:</dt><dd>formatted, formatted</dd> <dt>object.value:</dt><dd>formatted, formatted</dd> </dl>"`
+    );
+  });
+
+  it('formats top level objects, converting unknown fields to string', () => {
+    indexPattern.getFieldByName = jest.fn();
+    indexPattern.getFormatterForField = jest.fn();
+    expect(
+      formatTopLevelObject(
+        {
+          fields: {
+            'object.value': [5, 10],
+          },
+        },
+        {
+          'object.value': [5, 10],
+        },
+        indexPattern
+      ).trim()
+    ).toMatchInlineSnapshot(
+      `"<dl class=\\"source truncate-by-height\\"><dt>object.value:</dt><dd>5, 10</dd> </dl>"`
     );
   });
 });

src/plugins/discover/public/application/angular/helpers/row_formatter.ts

@@ -35,3 +35,31 @@ export const formatRow = (hit: Record<string, any>, indexPattern: IndexPattern)
   });
   return doTemplate({ defPairs: [...highlightPairs, ...sourcePairs] });
 };
+
+export const formatTopLevelObject = (
+  row: Record<string, any>,
+  fields: Record<string, any>,
+  indexPattern: IndexPattern
+) => {
+  const highlights = row.highlight ?? {};
+  const highlightPairs: Array<[string, unknown]> = [];
+  const sourcePairs: Array<[string, unknown]> = [];
+  Object.entries(fields).forEach(([key, values]) => {
+    const field = indexPattern.getFieldByName(key);
+    const formatter = field
+      ? indexPattern.getFormatterForField(field)
+      : { convert: (v: string, ...rest: unknown[]) => String(v) };
+    const formatted = values
+      .map((val: unknown) =>
+        formatter.convert(val, 'html', {
+          field,
+          hit: row,
+          indexPattern,
+        })
+      )
+      .join(', ');
+    const pairs = highlights[key] ? highlightPairs : sourcePairs;
+    pairs.push([key, formatted]);
+  });
+  return doTemplate({ defPairs: [...highlightPairs, ...sourcePairs] });
+};

src/plugins/discover/public/application/components/discover.tsx

@@ -410,6 +410,7 @@ export function Discover({
                               onSetColumns={onSetColumns}
                               onSort={onSort}
                               onResize={onResize}
+                              useNewFieldsApi={useNewFieldsApi}
                             />
                           </div>
                         )}

src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx

@@ -120,6 +120,10 @@ export interface DiscoverGridProps {
    * Current sort setting
    */
   sort: SortPairArr[];
+  /**
+   * How the data is fetched
+   */
+  useNewFieldsApi: boolean;
 }
 
 export const EuiDataGridMemoized = React.memo((props: EuiDataGridProps) => {
@@ -146,6 +150,7 @@ export const DiscoverGrid = ({
   settings,
   showTimeCol,
   sort,
+  useNewFieldsApi,
 }: DiscoverGridProps) => {
   const displayedColumns = getDisplayedColumns(columns, indexPattern);
   const defaultColumns = displayedColumns.includes('_source');
@@ -197,9 +202,10 @@ export const DiscoverGrid = ({
       getRenderCellValueFn(
         indexPattern,
         rows,
-        rows ? rows.map((hit) => indexPattern.flattenHit(hit)) : []
+        rows ? rows.map((hit) => indexPattern.flattenHit(hit)) : [],
+        useNewFieldsApi
       ),
-    [rows, indexPattern]
+    [rows, indexPattern, useNewFieldsApi]
   );
 
   /**

src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.test.tsx

@@ -10,13 +10,45 @@ import React from 'react';
 import { shallow } from 'enzyme';
 import { getRenderCellValueFn } from './get_render_cell_value';
 import { indexPatternMock } from '../../../__mocks__/index_pattern';
-const rows = [
+
+const rowsSource = [
   {
     _id: '1',
     _index: 'test',
     _type: 'test',
     _score: 1,
-    _source: { bytes: 100 },
+    _source: { bytes: 100, extension: '.gz' },
+    highlight: {
+      extension: '@kibana-highlighted-field.gz@/kibana-highlighted-field',
+    },
+  },
+];
+
+const rowsFields = [
+  {
+    _id: '1',
+    _index: 'test',
+    _type: 'test',
+    _score: 1,
+    _source: undefined,
+    fields: { bytes: [100], extension: ['.gz'] },
+    highlight: {
+      extension: '@kibana-highlighted-field.gz@/kibana-highlighted-field',
+    },
+  },
+];
+
+const rowsFieldsWithTopLevelObject = [
+  {
+    _id: '1',
+    _index: 'test',
+    _type: 'test',
+    _score: 1,
+    _source: undefined,
+    fields: { 'object.value': [100], extension: ['.gz'] },
+    highlight: {
+      extension: '@kibana-highlighted-field.gz@/kibana-highlighted-field',
+    },
   },
 ];
 
@@ -24,8 +56,9 @@ describe('Discover grid cell rendering', function () {
   it('renders bytes column correctly', () => {
     const DiscoverGridCellValue = getRenderCellValueFn(
       indexPatternMock,
-      rows,
+      rowsSource,
-      rows.map((row) => indexPatternMock.flattenHit(row))
+      rowsSource.map((row) => indexPatternMock.flattenHit(row)),
+      false
     );
     const component = shallow(
       <DiscoverGridCellValue
@@ -39,11 +72,13 @@ describe('Discover grid cell rendering', function () {
     );
     expect(component.html()).toMatchInlineSnapshot(`"<span>100</span>"`);
   });
+
   it('renders _source column correctly', () => {
     const DiscoverGridCellValue = getRenderCellValueFn(
       indexPatternMock,
-      rows,
+      rowsSource,
-      rows.map((row) => indexPatternMock.flattenHit(row))
+      rowsSource.map((row) => indexPatternMock.flattenHit(row)),
+      false
     );
     const component = shallow(
       <DiscoverGridCellValue
@@ -55,16 +90,44 @@ describe('Discover grid cell rendering', function () {
         setCellProps={jest.fn()}
       />
     );
-    expect(component.html()).toMatchInlineSnapshot(
+    expect(component).toMatchInlineSnapshot(`
-      `"<dl class=\\"euiDescriptionList euiDescriptionList--inline euiDescriptionList--compressed dscDiscoverGrid__descriptionList\\"><dt class=\\"euiDescriptionList__title\\">bytes</dt><dd class=\\"euiDescriptionList__description dscDiscoverGrid__descriptionListDescription\\">100</dd></dl>"`
+      <EuiDescriptionList
-    );
+        className="dscDiscoverGrid__descriptionList"
+        compressed={true}
+        type="inline"
+      >
+        <EuiDescriptionListTitle>
+          extension
+        </EuiDescriptionListTitle>
+        <EuiDescriptionListDescription
+          className="dscDiscoverGrid__descriptionListDescription"
+          dangerouslySetInnerHTML={
+            Object {
+              "__html": ".gz",
+            }
+          }
+        />
+        <EuiDescriptionListTitle>
+          bytes
+        </EuiDescriptionListTitle>
+        <EuiDescriptionListDescription
+          className="dscDiscoverGrid__descriptionListDescription"
+          dangerouslySetInnerHTML={
+            Object {
+              "__html": 100,
+            }
+          }
+        />
+      </EuiDescriptionList>
+    `);
   });
 
   it('renders _source column correctly when isDetails is set to true', () => {
     const DiscoverGridCellValue = getRenderCellValueFn(
       indexPatternMock,
-      rows,
+      rowsSource,
-      rows.map((row) => indexPatternMock.flattenHit(row))
+      rowsSource.map((row) => indexPatternMock.flattenHit(row)),
+      false
     );
     const component = shallow(
       <DiscoverGridCellValue
@@ -83,17 +146,247 @@ describe('Discover grid cell rendering', function () {
         &quot;_type&quot;: &quot;test&quot;,
         &quot;_score&quot;: 1,
         &quot;_source&quot;: {
-          &quot;bytes&quot;: 100
+          &quot;bytes&quot;: 100,
+          &quot;extension&quot;: &quot;.gz&quot;
+        },
+        &quot;highlight&quot;: {
+          &quot;extension&quot;: &quot;@kibana-highlighted-field.gz@/kibana-highlighted-field&quot;
         }
       }</span>"
     `);
   });
 
+  it('renders fields-based column correctly', () => {
+    const DiscoverGridCellValue = getRenderCellValueFn(
+      indexPatternMock,
+      rowsFields,
+      rowsFields.map((row) => indexPatternMock.flattenHit(row)),
+      true
+    );
+    const component = shallow(
+      <DiscoverGridCellValue
+        rowIndex={0}
+        columnId="_source"
+        isDetails={false}
+        isExpanded={false}
+        isExpandable={true}
+        setCellProps={jest.fn()}
+      />
+    );
+    expect(component).toMatchInlineSnapshot(`
+      <EuiDescriptionList
+        className="dscDiscoverGrid__descriptionList"
+        compressed={true}
+        type="inline"
+      >
+        <EuiDescriptionListTitle>
+          extension
+        </EuiDescriptionListTitle>
+        <EuiDescriptionListDescription
+          className="dscDiscoverGrid__descriptionListDescription"
+          dangerouslySetInnerHTML={
+            Object {
+              "__html": Array [
+                ".gz",
+              ],
+            }
+          }
+        />
+        <EuiDescriptionListTitle>
+          bytes
+        </EuiDescriptionListTitle>
+        <EuiDescriptionListDescription
+          className="dscDiscoverGrid__descriptionListDescription"
+          dangerouslySetInnerHTML={
+            Object {
+              "__html": Array [
+                100,
+              ],
+            }
+          }
+        />
+      </EuiDescriptionList>
+    `);
+  });
+
+  it('renders fields-based column correctly when isDetails is set to true', () => {
+    const DiscoverGridCellValue = getRenderCellValueFn(
+      indexPatternMock,
+      rowsFields,
+      rowsFields.map((row) => indexPatternMock.flattenHit(row)),
+      true
+    );
+    const component = shallow(
+      <DiscoverGridCellValue
+        rowIndex={0}
+        columnId="_source"
+        isDetails={true}
+        isExpanded={false}
+        isExpandable={true}
+        setCellProps={jest.fn()}
+      />
+    );
+    expect(component.html()).toMatchInlineSnapshot(`
+      "<span>{
+        &quot;_id&quot;: &quot;1&quot;,
+        &quot;_index&quot;: &quot;test&quot;,
+        &quot;_type&quot;: &quot;test&quot;,
+        &quot;_score&quot;: 1,
+        &quot;fields&quot;: {
+          &quot;bytes&quot;: [
+            100
+          ],
+          &quot;extension&quot;: [
+            &quot;.gz&quot;
+          ]
+        },
+        &quot;highlight&quot;: {
+          &quot;extension&quot;: &quot;@kibana-highlighted-field.gz@/kibana-highlighted-field&quot;
+        }
+      }</span>"
+    `);
+  });
+
+  it('collect object fields and renders them like _source', () => {
+    const DiscoverGridCellValue = getRenderCellValueFn(
+      indexPatternMock,
+      rowsFieldsWithTopLevelObject,
+      rowsFieldsWithTopLevelObject.map((row) => indexPatternMock.flattenHit(row)),
+      true
+    );
+    const component = shallow(
+      <DiscoverGridCellValue
+        rowIndex={0}
+        columnId="object"
+        isDetails={false}
+        isExpanded={false}
+        isExpandable={true}
+        setCellProps={jest.fn()}
+      />
+    );
+    expect(component).toMatchInlineSnapshot(`
+      <EuiDescriptionList
+        className="dscDiscoverGrid__descriptionList"
+        compressed={true}
+        type="inline"
+      >
+        <EuiDescriptionListTitle>
+          object.value
+        </EuiDescriptionListTitle>
+        <EuiDescriptionListDescription
+          className="dscDiscoverGrid__descriptionListDescription"
+          dangerouslySetInnerHTML={
+            Object {
+              "__html": "formatted",
+            }
+          }
+        />
+      </EuiDescriptionList>
+    `);
+  });
+
+  it('collect object fields and renders them like _source with fallback for unmapped', () => {
+    (indexPatternMock.getFieldByName as jest.Mock).mockReturnValueOnce(undefined);
+    const DiscoverGridCellValue = getRenderCellValueFn(
+      indexPatternMock,
+      rowsFieldsWithTopLevelObject,
+      rowsFieldsWithTopLevelObject.map((row) => indexPatternMock.flattenHit(row)),
+      true
+    );
+    const component = shallow(
+      <DiscoverGridCellValue
+        rowIndex={0}
+        columnId="object"
+        isDetails={false}
+        isExpanded={false}
+        isExpandable={true}
+        setCellProps={jest.fn()}
+      />
+    );
+    expect(component).toMatchInlineSnapshot(`
+      <EuiDescriptionList
+        className="dscDiscoverGrid__descriptionList"
+        compressed={true}
+        type="inline"
+      >
+        <EuiDescriptionListTitle>
+          object.value
+        </EuiDescriptionListTitle>
+        <EuiDescriptionListDescription
+          className="dscDiscoverGrid__descriptionListDescription"
+          dangerouslySetInnerHTML={
+            Object {
+              "__html": "100",
+            }
+          }
+        />
+      </EuiDescriptionList>
+    `);
+  });
+
+  it('collect object fields and renders them as json in details', () => {
+    const DiscoverGridCellValue = getRenderCellValueFn(
+      indexPatternMock,
+      rowsFieldsWithTopLevelObject,
+      rowsFieldsWithTopLevelObject.map((row) => indexPatternMock.flattenHit(row)),
+      true
+    );
+    const component = shallow(
+      <DiscoverGridCellValue
+        rowIndex={0}
+        columnId="object"
+        isDetails={true}
+        isExpanded={false}
+        isExpandable={true}
+        setCellProps={jest.fn()}
+      />
+    );
+    expect(component).toMatchInlineSnapshot(`
+      <span>
+        {
+        "object.value": [
+          100
+        ]
+      }
+      </span>
+    `);
+  });
+
+  it('does not collect subfields when the the column is unmapped but part of fields response', () => {
+    (indexPatternMock.getFieldByName as jest.Mock).mockReturnValueOnce(undefined);
+    const DiscoverGridCellValue = getRenderCellValueFn(
+      indexPatternMock,
+      rowsFieldsWithTopLevelObject,
+      rowsFieldsWithTopLevelObject.map((row) => indexPatternMock.flattenHit(row)),
+      true
+    );
+    const component = shallow(
+      <DiscoverGridCellValue
+        rowIndex={0}
+        columnId="object.value"
+        isDetails={false}
+        isExpanded={false}
+        isExpandable={true}
+        setCellProps={jest.fn()}
+      />
+    );
+    expect(component).toMatchInlineSnapshot(`
+      <span
+        dangerouslySetInnerHTML={
+          Object {
+            "__html": 100,
+          }
+        }
+      />
+    `);
+  });
+
   it('renders correctly when invalid row is given', () => {
     const DiscoverGridCellValue = getRenderCellValueFn(
       indexPatternMock,
-      rows,
+      rowsSource,
-      rows.map((row) => indexPatternMock.flattenHit(row))
+      rowsSource.map((row) => indexPatternMock.flattenHit(row)),
+      false
     );
     const component = shallow(
       <DiscoverGridCellValue
@@ -107,11 +400,13 @@ describe('Discover grid cell rendering', function () {
     );
     expect(component.html()).toMatchInlineSnapshot(`"<span>-</span>"`);
   });
+
   it('renders correctly when invalid column is given', () => {
     const DiscoverGridCellValue = getRenderCellValueFn(
       indexPatternMock,
-      rows,
+      rowsSource,
-      rows.map((row) => indexPatternMock.flattenHit(row))
+      rowsSource.map((row) => indexPatternMock.flattenHit(row)),
+      false
     );
     const component = shallow(
       <DiscoverGridCellValue

src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.tsx

@@ -23,7 +23,8 @@ import { DiscoverGridContext } from './discover_grid_context';
 export const getRenderCellValueFn = (
   indexPattern: IndexPattern,
   rows: ElasticSearchHit[] | undefined,
-  rowsFlattened: Array<Record<string, unknown>>
+  rowsFlattened: Array<Record<string, unknown>>,
+  useNewFieldsApi: boolean
 ) => ({ rowIndex, columnId, isDetails, setCellProps }: EuiDataGridCellValueElementProps) => {
   const row = rows ? (rows[rowIndex] as Record<string, unknown>) : undefined;
   const rowFlattened = rowsFlattened
@@ -51,6 +52,60 @@ export const getRenderCellValueFn = (
     return <span>-</span>;
   }
 
+  if (
+    useNewFieldsApi &&
+    !field &&
+    row &&
+    row.fields &&
+    !(row.fields as Record<string, unknown[]>)[columnId]
+  ) {
+    const innerColumns = Object.fromEntries(
+      Object.entries(row.fields as Record<string, unknown[]>).filter(([key]) => {
+        return key.indexOf(`${columnId}.`) === 0;
+      })
+    );
+    if (isDetails) {
+      // nicely formatted JSON for the expanded view
+      return <span>{JSON.stringify(innerColumns, null, 2)}</span>;
+    }
+
+    // Put the most important fields first
+    const highlights: Record<string, unknown> = (row.highlight as Record<string, unknown>) ?? {};
+    const highlightPairs: Array<[string, string]> = [];
+    const sourcePairs: Array<[string, string]> = [];
+    Object.entries(innerColumns).forEach(([key, values]) => {
+      const subField = indexPattern.getFieldByName(key);
+      const formatter = subField
+        ? indexPattern.getFormatterForField(subField)
+        : { convert: (v: string, ...rest: unknown[]) => String(v) };
+      const formatted = (values as unknown[])
+        .map((val: unknown) =>
+          formatter.convert(val, 'html', {
+            field: subField,
+            hit: row,
+            indexPattern,
+          })
+        )
+        .join(', ');
+      const pairs = highlights[key] ? highlightPairs : sourcePairs;
+      pairs.push([key, formatted]);
+    });
+
+    return (
+      <EuiDescriptionList type="inline" compressed className="dscDiscoverGrid__descriptionList">
+        {[...highlightPairs, ...sourcePairs].map(([key, value]) => (
+          <Fragment key={key}>
+            <EuiDescriptionListTitle>{key}</EuiDescriptionListTitle>
+            <EuiDescriptionListDescription
+              dangerouslySetInnerHTML={{ __html: value }}
+              className="dscDiscoverGrid__descriptionListDescription"
+            />
+          </Fragment>
+        ))}
+      </EuiDescriptionList>
+    );
+  }
+
   if (field && field.type === '_source') {
     if (isDetails) {
       // nicely formatted JSON for the expanded view
@@ -58,13 +113,23 @@ export const getRenderCellValueFn = (
     }
     const formatted = indexPattern.formatHit(row);
 
+    // Put the most important fields first
+    const highlights: Record<string, unknown> = (row.highlight as Record<string, unknown>) ?? {};
+    const highlightPairs: Array<[string, string]> = [];
+    const sourcePairs: Array<[string, string]> = [];
+
+    Object.entries(formatted).forEach(([key, val]) => {
+      const pairs = highlights[key] ? highlightPairs : sourcePairs;
+      pairs.push([key, val as string]);
+    });
+
     return (
       <EuiDescriptionList type="inline" compressed className="dscDiscoverGrid__descriptionList">
-        {Object.keys(formatted).map((key) => (
+        {[...highlightPairs, ...sourcePairs].map(([key, value]) => (
           <Fragment key={key}>
             <EuiDescriptionListTitle>{key}</EuiDescriptionListTitle>
             <EuiDescriptionListDescription
-              dangerouslySetInnerHTML={{ __html: formatted[key] }}
+              dangerouslySetInnerHTML={{ __html: value }}
               className="dscDiscoverGrid__descriptionListDescription"
             />
           </Fragment>

test/functional/apps/discover/_field_data.ts

@@ -16,6 +16,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const kibanaServer = getService('kibanaServer');
   const toasts = getService('toasts');
   const queryBar = getService('queryBar');
+  const browser = getService('browser');
   const PageObjects = getPageObjects(['common', 'header', 'discover', 'visualize', 'timePicker']);
 
   describe('discover tab', function describeIndexTests() {
@@ -89,6 +90,27 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         expect(message).to.contain(expectedError);
         await toasts.dismissToast();
       });
+
+      it('shows top-level object keys', async function () {
+        await queryBar.setQuery('election');
+        await queryBar.submitQuery();
+        const currentUrl = await browser.getCurrentUrl();
+        const [, hash] = currentUrl.split('#/');
+        await PageObjects.common.navigateToUrl(
+          'discover',
+          hash.replace('columns:!(_source)', 'columns:!(relatedContent)'),
+          { useActualUrl: true }
+        );
+        await retry.try(async function tryingForTime() {
+          expect(await PageObjects.discover.getDocHeader()).to.be('Time relatedContent');
+        });
+
+        const field = await PageObjects.discover.getDocTableField(1, 1);
+        expect(field).to.include.string('"og:description":');
+
+        const marks = await PageObjects.discover.getMarks();
+        expect(marks.length).to.be(0);
+      });
     });
   });
 }

test/functional/apps/discover/_field_data_with_fields_api.ts

@@ -16,6 +16,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const kibanaServer = getService('kibanaServer');
   const toasts = getService('toasts');
   const queryBar = getService('queryBar');
+  const browser = getService('browser');
   const PageObjects = getPageObjects(['common', 'header', 'discover', 'visualize', 'timePicker']);
 
   describe('discover tab with new fields API', function describeIndexTests() {
@@ -89,6 +90,28 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         expect(message).to.contain(expectedError);
         await toasts.dismissToast();
       });
+
+      it('shows top-level object keys', async function () {
+        await queryBar.setQuery('election');
+        await queryBar.submitQuery();
+        const currentUrl = await browser.getCurrentUrl();
+        const [, hash] = currentUrl.split('#/');
+        await PageObjects.common.navigateToUrl(
+          'discover',
+          hash.replace('columns:!()', 'columns:!(relatedContent)'),
+          { useActualUrl: true }
+        );
+        await retry.try(async function tryingForTime() {
+          expect(await PageObjects.discover.getDocHeader()).to.be('Time relatedContent');
+        });
+
+        const field = await PageObjects.discover.getDocTableField(1, 1);
+        expect(field).to.include.string('relatedContent.url:');
+
+        const marks = await PageObjects.discover.getMarks();
+        expect(marks.length).to.be(172);
+        expect(marks.indexOf('election')).to.be(0);
+      });
     });
   });
 }

test/functional/page_objects/discover_page.ts

@@ -194,11 +194,11 @@ export function DiscoverPageProvider({ getService, getPageObjects }: FtrProvider
       return await row.getVisibleText();
     }
 
-    public async getDocTableField(index: number) {
+    public async getDocTableField(index: number, cellIndex = 0) {
-      const field = await find.byCssSelector(
+      const fields = await find.allByCssSelector(
-        `tr.kbnDocTable__row:nth-child(${index}) > [data-test-subj='docTableField']`
+        `tr.kbnDocTable__row:nth-child(${index}) [data-test-subj='docTableField']`
       );
-      return await field.getVisibleText();
+      return await fields[cellIndex].getVisibleText();
     }
 
     public async skipToEndOfDocTable() {