[Discover] Fix formatting and sorting for custom ES|QL vars (#209360)

- Closes https://github.com/elastic/kibana/issues/208020

## Summary

By default the data view (which is used in ES|QL mode) has only mapped
fields as per field caps for the current index pattern. This PR
dynamically extends with additional fields based on ES|QL meta
information.

This change helps to fix the formatting for ES|QL var values and fixes
sorting on them.

<img width="1673" alt="Screenshot 2025-02-03 at 18 42 14"
src="https://github.com/user-attachments/assets/3647a375-f0f5-43e6-815d-d5a4292c637a"
/>
<img width="643" alt="Screenshot 2025-02-03 at 18 42 50"
src="https://github.com/user-attachments/assets/9d84bc23-7665-43c1-8ac2-d67174b68c31"
/>


### Checklist


- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>

src/platform/packages/shared/kbn-data-view-utils/index.ts

@@ -9,5 +9,6 @@
 
 export * from './src/constants';
 export { convertDatatableColumnToDataViewFieldSpec } from './src/utils/convert_to_data_view_field_spec';
+export { getDataViewFieldOrCreateFromColumnMeta } from './src/utils/get_data_view_field_or_create';
 export { createRegExpPatternFrom } from './src/utils/create_regexp_pattern_from';
 export { testPatternAgainstAllowedList } from './src/utils/test_pattern_against_allowed_list';

src/platform/packages/shared/kbn-data-view-utils/src/utils/get_data_view_field_or_create.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { isEqual } from 'lodash';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import type { DatatableColumnMeta } from '@kbn/expressions-plugin/common';
+import { convertDatatableColumnToDataViewFieldSpec } from './convert_to_data_view_field_spec';
+
+export const getDataViewFieldOrCreateFromColumnMeta = ({
+  dataView,
+  fieldName,
+  columnMeta,
+}: {
+  dataView: DataView;
+  fieldName: string;
+  columnMeta?: DatatableColumnMeta; // based on ES|QL query
+}) => {
+  const dataViewField = dataView.fields.getByName(fieldName);
+
+  if (!columnMeta) {
+    return dataViewField;
+  }
+
+  const fieldSpecFromColumnMeta = convertDatatableColumnToDataViewFieldSpec({
+    name: fieldName,
+    id: fieldName,
+    meta: columnMeta,
+  });
+
+  if (
+    !dataViewField ||
+    dataViewField.type !== fieldSpecFromColumnMeta.type ||
+    !isEqual(dataViewField.esTypes, fieldSpecFromColumnMeta.esTypes)
+  ) {
+    return dataView.fields.create(fieldSpecFromColumnMeta);
+  }
+
+  return dataViewField;
+};

src/platform/packages/shared/kbn-discover-utils/src/__mocks__/data_view.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { DataView, DataViewField } from '@kbn/data-views-plugin/public';
+import { DataView, DataViewField, FieldSpec } from '@kbn/data-views-plugin/public';
 
 export const shallowMockedFields = [
   {
@@ -101,6 +101,10 @@ export const buildDataViewMock = ({
     return dataViewFields;
   };
 
+  dataViewFields.create = (spec: FieldSpec) => {
+    return new DataViewField(spec);
+  };
+
   const dataView = {
     id: `${name}-id`,
     title: `${name}-title`,

src/platform/packages/shared/kbn-unified-data-table/__mocks__/table_context.ts

@@ -36,6 +36,7 @@ const buildTableContext = (dataView: DataView, rows: DataTableRecord[]): DataTab
         fieldFormats: servicesMock.fieldFormats,
         rows,
         dataView,
+        columnsMeta: undefined,
         options,
       }),
   };

src/platform/packages/shared/kbn-unified-data-table/src/components/data_table.tsx

@@ -606,10 +606,11 @@ export const UnifiedDataTable = ({
         dataView,
         columnId,
         fieldFormats,
+        columnsMeta,
         options,
       });
     },
-    [displayedRows, dataView, fieldFormats]
+    [displayedRows, dataView, fieldFormats, columnsMeta]
   );
 
   /**
@@ -732,6 +733,7 @@ export const UnifiedDataTable = ({
         externalCustomRenderers,
         isPlainRecord,
         isCompressed: dataGridDensity === DataGridDensity.COMPACT,
+        columnsMeta,
       }),
     [
       dataView,
@@ -742,6 +744,7 @@ export const UnifiedDataTable = ({
       externalCustomRenderers,
       isPlainRecord,
       dataGridDensity,
+      columnsMeta,
     ]
   );
 

src/platform/packages/shared/kbn-unified-data-table/src/components/data_table_columns.tsx

@@ -15,7 +15,8 @@ import {
   EuiScreenReaderOnly,
   EuiListGroupItemProps,
 } from '@elastic/eui';
-import { type DataView, DataViewField } from '@kbn/data-views-plugin/public';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import { getDataViewFieldOrCreateFromColumnMeta } from '@kbn/data-view-utils';
 import { ToastsStart, IUiSettingsClient } from '@kbn/core/public';
 import { DocViewFilterFn } from '@kbn/unified-doc-viewer/types';
 import type { DataTableRecord } from '@kbn/discover-utils';
@@ -141,17 +142,11 @@ function buildEuiGridColumn({
   columnDisplay?: string;
   onResize: UnifiedDataTableProps['onResize'];
 }) {
-  const dataViewField = !isPlainRecord
-    ? dataView.getFieldByName(columnName)
-    : new DataViewField({
-        name: columnName,
-        type: columnsMeta?.[columnName]?.type ?? 'unknown',
-        esTypes: columnsMeta?.[columnName]?.esType
-          ? ([columnsMeta[columnName].esType] as string[])
-          : undefined,
-        searchable: true,
-        aggregatable: false,
-      });
+  const dataViewField = getDataViewFieldOrCreateFromColumnMeta({
+    dataView,
+    fieldName: columnName,
+    columnMeta: columnsMeta?.[columnName],
+  });
   const editFieldButton =
     editField &&
     dataViewField &&
@@ -203,7 +198,7 @@ function buildEuiGridColumn({
     }
   }
 
-  const columnType = columnsMeta?.[columnName]?.type ?? dataViewField?.type;
+  const columnType = dataViewField?.type;
 
   const column: EuiDataGridColumn = {
     id: columnName,

src/platform/packages/shared/kbn-unified-data-table/src/hooks/use_sorting.ts

@@ -10,6 +10,7 @@
 import type { DataView } from '@kbn/data-views-plugin/public';
 import type { DataTableRecord } from '@kbn/discover-utils';
 import { getSortingCriteria } from '@kbn/sort-predicates';
+import { getDataViewFieldOrCreateFromColumnMeta } from '@kbn/data-view-utils';
 import { useMemo } from 'react';
 import type { EuiDataGridColumnSortingConfig, EuiDataGridProps } from '@elastic/eui';
 import type { SortOrder } from '../components/data_table';
@@ -49,17 +50,17 @@ export const useSorting = ({
 
     return sortingColumns.reduce<Array<(a: DataTableRecord, b: DataTableRecord) => number>>(
       (acc, { id, direction }) => {
-        const field = dataView.fields.getByName(id);
+        const field = getDataViewFieldOrCreateFromColumnMeta({
+          dataView,
+          fieldName: id,
+          columnMeta: columnsMeta?.[id],
+        });
 
         if (!field) {
           return acc;
         }
 
-        const sortField = getSortingCriteria(
-          columnsMeta?.[id]?.type ?? field.type,
-          id,
-          dataView.getFormatterForField(field)
-        );
+        const sortField = getSortingCriteria(field.type, id, dataView.getFormatterForField(field));
 
         acc.push((a, b) => sortField(a.flattened, b.flattened, direction as 'asc' | 'desc'));
 

src/platform/packages/shared/kbn-unified-data-table/src/utils/convert_value_to_string.test.tsx

@@ -17,6 +17,8 @@ import { servicesMock } from '../../__mocks__/services';
 import { convertValueToString, convertNameToString } from './convert_value_to_string';
 
 describe('convertValueToString', () => {
+  jest.spyOn(dataTableContextComplexMock.dataView.fields, 'create');
+
   it('should convert a keyword value to text', () => {
     const result = convertValueToString({
       rows: dataTableContextComplexRowsMock,
@@ -24,6 +26,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'keyword_key',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -39,6 +42,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'text_message',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -47,21 +51,6 @@ describe('convertValueToString', () => {
     expect(result.formattedString).toBe('"Hi there! I am a sample string."');
   });
 
-  it('should convert a text value to text (not for CSV)', () => {
-    const result = convertValueToString({
-      rows: dataTableContextComplexRowsMock,
-      dataView: dataTableContextComplexMock.dataView,
-      fieldFormats: servicesMock.fieldFormats,
-      columnId: 'text_message',
-      rowIndex: 0,
-      options: {
-        compatibleWithCSV: false,
-      },
-    });
-
-    expect(result.formattedString).toBe('Hi there! I am a sample string.');
-  });
-
   it('should convert a multiline text value to text', () => {
     const result = convertValueToString({
       rows: dataTableContextComplexRowsMock,
@@ -69,6 +58,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'text_message',
       rowIndex: 1,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -85,12 +75,36 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'number_price',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
     });
 
     expect(result.formattedString).toBe('10.99');
+    expect(dataTableContextComplexMock.dataView.fields.create).toHaveBeenCalledTimes(0);
+  });
+
+  it('should convert a number value as keyword override to text', () => {
+    const result = convertValueToString({
+      rows: dataTableContextComplexRowsMock,
+      dataView: dataTableContextComplexMock.dataView,
+      fieldFormats: servicesMock.fieldFormats,
+      columnId: 'number_price',
+      rowIndex: 0,
+      columnsMeta: {
+        number_price: {
+          type: 'string',
+          esType: 'keyword',
+        },
+      },
+      options: {
+        compatibleWithCSV: true,
+      },
+    });
+
+    expect(result.formattedString).toBe('10.99');
+    expect(dataTableContextComplexMock.dataView.fields.create).toHaveBeenCalledTimes(1);
   });
 
   it('should convert a date value to text', () => {
@@ -100,6 +114,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'date',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -115,6 +130,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'date_nanos',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -130,6 +146,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'date_nanos',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: false,
       },
@@ -145,6 +162,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'bool_enabled',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -160,6 +178,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'binary_blob',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -175,6 +194,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'binary_blob',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: false,
       },
@@ -190,6 +210,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'object_user.first',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -205,6 +226,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'nested_user',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -222,6 +244,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'flattened_labels',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -237,6 +260,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'range_time_frame',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -254,6 +278,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'rank_features',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -269,6 +294,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'histogram',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -284,6 +310,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'ip_addr',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -299,6 +326,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'ip_addr',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: false,
       },
@@ -314,6 +342,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'version',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -329,6 +358,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'version',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: false,
       },
@@ -344,6 +374,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'vector',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -359,6 +390,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'geo_point',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -374,6 +406,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'geo_point',
       rowIndex: 1,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -389,6 +422,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'array_tags',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -404,6 +438,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'geometry',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -421,6 +456,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'runtime_number',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -436,6 +472,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'scripted_string',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -451,6 +488,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'scripted_string',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: false,
       },
@@ -466,6 +504,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'unknown',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -481,6 +520,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'unknown',
       rowIndex: -1,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -496,6 +536,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: '_source',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: false,
       },
@@ -519,6 +560,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: '_source',
       rowIndex: 0,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -536,6 +578,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'array_tags',
       rowIndex: 1,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -550,6 +593,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'scripted_string',
       rowIndex: 1,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: true,
       },
@@ -566,6 +610,7 @@ describe('convertValueToString', () => {
       fieldFormats: servicesMock.fieldFormats,
       columnId: 'array_tags',
       rowIndex: 1,
+      columnsMeta: undefined,
       options: {
         compatibleWithCSV: false,
       },

src/platform/packages/shared/kbn-unified-data-table/src/utils/convert_value_to_string.ts

@@ -9,8 +9,9 @@
 
 import type { DataView } from '@kbn/data-views-plugin/public';
 import { cellHasFormulas, createEscapeValue } from '@kbn/data-plugin/common';
+import { getDataViewFieldOrCreateFromColumnMeta } from '@kbn/data-view-utils';
 import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
-import type { DataTableRecord } from '@kbn/discover-utils/types';
+import type { DataTableRecord, DataTableColumnsMeta } from '@kbn/discover-utils/types';
 import { formatFieldValue } from '@kbn/discover-utils';
 
 interface ConvertedResult {
@@ -26,6 +27,7 @@ export const convertValueToString = ({
   columnId,
   dataView,
   fieldFormats,
+  columnsMeta,
   options,
 }: {
   rowIndex: number;
@@ -33,6 +35,7 @@ export const convertValueToString = ({
   columnId: string;
   dataView: DataView;
   fieldFormats: FieldFormatsStart;
+  columnsMeta: DataTableColumnsMeta | undefined;
   options?: {
     compatibleWithCSV?: boolean; // values as one-liner + escaping formulas + adding wrapping quotes
   };
@@ -45,7 +48,11 @@ export const convertValueToString = ({
   }
   const rowFlattened = rows[rowIndex].flattened;
   const value = rowFlattened?.[columnId];
-  const field = dataView.fields.getByName(columnId);
+  const field = getDataViewFieldOrCreateFromColumnMeta({
+    fieldName: columnId,
+    dataView,
+    columnMeta: columnsMeta?.[columnId],
+  });
   const valuesArray = Array.isArray(value) ? value : [value];
   const disableMultiline = options?.compatibleWithCSV ?? false;
   const enableEscapingForValue = options?.compatibleWithCSV ?? false;

src/platform/packages/shared/kbn-unified-data-table/src/utils/copy_value_to_clipboard.test.tsx

@@ -33,6 +33,7 @@ describe('copyValueToClipboard', () => {
       fieldFormats: servicesMock.fieldFormats,
       rowIndex,
       columnId,
+      columnsMeta: undefined,
       options,
     });
 

src/platform/packages/shared/kbn-unified-data-table/src/utils/get_render_cell_value.test.tsx

@@ -117,6 +117,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -142,6 +143,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -168,6 +170,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: closePopoverMockFn,
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = mountWithIntl(
       <DataTableCellValue
@@ -197,6 +200,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -233,6 +237,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -300,6 +305,7 @@ describe('Unified data table cell rendering', function () {
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
       isPlainRecord: true,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -339,6 +345,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -378,6 +385,7 @@ describe('Unified data table cell rendering', function () {
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       // this is the number of rendered items
       maxEntries: 1,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -414,6 +422,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -490,6 +499,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -530,6 +540,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -567,6 +578,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: closePopoverMockFn,
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -641,6 +653,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: closePopoverMockFn,
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = mountWithIntl(
       <KibanaContextProvider services={mockServices}>
@@ -669,6 +682,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -703,6 +717,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -728,6 +743,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -766,6 +782,7 @@ describe('Unified data table cell rendering', function () {
       closePopover: jest.fn(),
       fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
       maxEntries: 100,
+      columnsMeta: undefined,
     });
     const component = shallow(
       <DataTableCellValue
@@ -837,4 +854,122 @@ describe('Unified data table cell rendering', function () {
       </EuiFlexGroup>
     `);
   });
+
+  it('renders custom ES|QL fields correctly', () => {
+    jest.spyOn(dataViewMock.fields, 'create');
+
+    const rows: EsHitRecord[] = [
+      {
+        _id: '1',
+        _index: 'test',
+        _score: 1,
+        _source: undefined,
+        fields: { bytes: 100, var0: 350, extension: 'gif' },
+      },
+    ];
+    const DataTableCellValue = getRenderCellValueFn({
+      dataView: dataViewMock,
+      rows: rows.map(build),
+      shouldShowFieldHandler: () => true,
+      closePopover: jest.fn(),
+      fieldFormats: mockServices.fieldFormats as unknown as FieldFormatsStart,
+      maxEntries: 100,
+      columnsMeta: {
+        // custom ES|QL var
+        var0: {
+          type: 'number',
+          esType: 'long',
+        },
+        // custom ES|QL override
+        bytes: {
+          type: 'string',
+          esType: 'keyword',
+        },
+      },
+    });
+    const componentWithDataViewField = shallow(
+      <DataTableCellValue
+        rowIndex={0}
+        colIndex={0}
+        columnId="extension"
+        isDetails={false}
+        isExpanded={false}
+        isExpandable={true}
+        setCellProps={jest.fn()}
+      />
+    );
+    expect(componentWithDataViewField).toMatchInlineSnapshot(`
+      <span
+        className="unifiedDataTable__cellValue"
+        dangerouslySetInnerHTML={
+          Object {
+            "__html": "gif",
+          }
+        }
+      />
+    `);
+    const componentWithCustomESQLField = shallow(
+      <DataTableCellValue
+        rowIndex={0}
+        colIndex={0}
+        columnId="var0"
+        isDetails={false}
+        isExpanded={false}
+        isExpandable={true}
+        setCellProps={jest.fn()}
+      />
+    );
+    expect(componentWithCustomESQLField).toMatchInlineSnapshot(`
+      <span
+        className="unifiedDataTable__cellValue"
+        dangerouslySetInnerHTML={
+          Object {
+            "__html": 350,
+          }
+        }
+      />
+    `);
+
+    expect(dataViewMock.fields.create).toHaveBeenCalledTimes(1);
+    expect(dataViewMock.fields.create).toHaveBeenCalledWith({
+      name: 'var0',
+      type: 'number',
+      esTypes: ['long'],
+      searchable: true,
+      aggregatable: false,
+      isNull: false,
+    });
+
+    const componentWithCustomESQLFieldOverride = shallow(
+      <DataTableCellValue
+        rowIndex={0}
+        colIndex={0}
+        columnId="bytes"
+        isDetails={false}
+        isExpanded={false}
+        isExpandable={true}
+        setCellProps={jest.fn()}
+      />
+    );
+    expect(componentWithCustomESQLFieldOverride).toMatchInlineSnapshot(`
+      <span
+        className="unifiedDataTable__cellValue"
+        dangerouslySetInnerHTML={
+          Object {
+            "__html": 100,
+          }
+        }
+      />
+    `);
+
+    expect(dataViewMock.fields.create).toHaveBeenCalledTimes(2);
+    expect(dataViewMock.fields.create).toHaveBeenLastCalledWith({
+      name: 'bytes',
+      type: 'string',
+      esTypes: ['keyword'],
+      searchable: true,
+      aggregatable: false,
+      isNull: false,
+    });
+  });
 });

src/platform/packages/shared/kbn-unified-data-table/src/utils/get_render_cell_value.tsx

@@ -18,7 +18,12 @@ import {
   useEuiTheme,
 } from '@elastic/eui';
 import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
-import type { DataTableRecord, ShouldShowFieldInTableHandler } from '@kbn/discover-utils/types';
+import { getDataViewFieldOrCreateFromColumnMeta } from '@kbn/data-view-utils';
+import {
+  DataTableColumnsMeta,
+  DataTableRecord,
+  ShouldShowFieldInTableHandler,
+} from '@kbn/discover-utils/types';
 import { formatFieldValue } from '@kbn/discover-utils';
 import { UnifiedDataTableContext } from '../table_context';
 import type { CustomCellRenderer } from '../types';
@@ -40,6 +45,7 @@ export const getRenderCellValueFn = ({
   externalCustomRenderers,
   isPlainRecord,
   isCompressed = true,
+  columnsMeta,
 }: {
   dataView: DataView;
   rows: DataTableRecord[] | undefined;
@@ -50,6 +56,7 @@ export const getRenderCellValueFn = ({
   externalCustomRenderers?: CustomCellRenderer;
   isPlainRecord?: boolean;
   isCompressed?: boolean;
+  columnsMeta: DataTableColumnsMeta | undefined;
 }) => {
   const UnifiedDataTableRenderCellValue = ({
     rowIndex,
@@ -61,7 +68,11 @@ export const getRenderCellValueFn = ({
     isExpanded,
   }: EuiDataGridCellValueElementProps) => {
     const row = rows ? rows[rowIndex] : undefined;
-    const field = dataView.fields.getByName(columnId);
+    const field = getDataViewFieldOrCreateFromColumnMeta({
+      dataView,
+      fieldName: columnId,
+      columnMeta: columnsMeta?.[columnId],
+    });
     const ctx = useContext(UnifiedDataTableContext);
     const { euiTheme } = useEuiTheme();
     const { backgroundBasePrimary: anchorColor } = euiTheme.colors;

src/platform/packages/shared/kbn-unified-data-table/tsconfig.json

@@ -44,5 +44,6 @@
     "@kbn/core-capabilities-browser-mocks",
     "@kbn/sort-predicates",
     "@kbn/data-grid-in-table-search",
+    "@kbn/data-view-utils",
   ]
 }

src/platform/plugins/shared/data_views/common/fields/field_list.ts

@@ -22,6 +22,12 @@ interface ToSpecOptions {
  * Interface for data view field list which _extends_ the array class.
  */
 export interface IIndexPatternFieldList extends Array<DataViewField> {
+  /**
+   * Creates a DataViewField instance. Does not add it to the data view.
+   * @param field field spec to create field instance
+   * @returns a new data view field instance
+   */
+  create(field: FieldSpec): DataViewField;
   /**
    * Add field to field list.
    * @param field field spec to add field to list
@@ -101,8 +107,13 @@ export const fieldList = (
     public readonly getByType = (type: DataViewField['type']) => [
       ...(this.groups.get(type) || new Map()).values(),
     ];
+
+    public readonly create = (field: FieldSpec): DataViewField => {
+      return new DataViewField({ ...field, shortDotsEnable });
+    };
+
     public readonly add = (field: FieldSpec): DataViewField => {
-      const newField = new DataViewField({ ...field, shortDotsEnable });
+      const newField = this.create(field);
       this.push(newField);
       this.setByName(newField);
       this.setByGroup(newField);

src/platform/plugins/shared/unified_doc_viewer/public/components/doc_viewer_table/field_row.ts

@@ -17,6 +17,7 @@ import {
 } from '@kbn/discover-utils';
 import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
 import { getFieldIconType, getTextBasedColumnIconType } from '@kbn/field-utils';
+import { getDataViewFieldOrCreateFromColumnMeta } from '@kbn/data-view-utils';
 
 export class FieldRow {
   readonly name: string;
@@ -62,7 +63,11 @@ export class FieldRow {
 
     this.name = name;
     this.flattenedValue = flattenedValue;
-    this.dataViewField = dataView.getFieldByName(name);
+    this.dataViewField = getDataViewFieldOrCreateFromColumnMeta({
+      dataView,
+      fieldName: name,
+      columnMeta: columnsMeta?.[name],
+    });
     this.isPinned = isPinned;
     this.columnsMeta = columnsMeta;
   }

src/platform/plugins/shared/unified_doc_viewer/tsconfig.json

@@ -38,7 +38,8 @@
     "@kbn/core-lifecycle-browser",
     "@kbn/management-settings-ids",
     "@kbn/apm-types",
-    "@kbn/event-stacktrace"
+    "@kbn/event-stacktrace",
+    "@kbn/data-view-utils"
 
   ],
   "exclude": [

test/functional/apps/discover/esql/_esql_view.ts

@@ -474,6 +474,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     });
 
     describe('sorting', () => {
+      beforeEach(async () => {
+        await common.navigateToApp('discover');
+        await timePicker.setDefaultAbsoluteRange();
+      });
+
       it('should sort correctly', async () => {
         const savedSearchName = 'testSorting';
 
@@ -640,6 +645,100 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
           'Sort fields\n2'
         );
       });
+
+      it('should sort on custom vars too', async () => {
+        const savedSearchName = 'testSortingForCustomVars';
+
+        await discover.selectTextBaseLang();
+        await header.waitUntilLoadingHasFinished();
+        await discover.waitUntilSearchingHasFinished();
+
+        const testQuery =
+          'from logstash-* | sort @timestamp | limit 100 | keep bytes | eval var0 = abs(bytes) + 1';
+        await monacoEditor.setCodeEditorValue(testQuery);
+        await testSubjects.click('querySubmitButton');
+        await header.waitUntilLoadingHasFinished();
+        await discover.waitUntilSearchingHasFinished();
+
+        await retry.waitFor('first cell contains an initial value', async () => {
+          const cell = await dataGrid.getCellElementExcludingControlColumns(0, 1);
+          const text = await cell.getVisibleText();
+          return text === '1,624';
+        });
+
+        expect(await testSubjects.getVisibleText('dataGridColumnSortingButton')).to.be(
+          'Sort fields'
+        );
+
+        await dataGrid.clickDocSortDesc('var0', 'Sort High-Low');
+
+        await discover.waitUntilSearchingHasFinished();
+
+        await retry.waitFor('first cell contains the highest value', async () => {
+          const cell = await dataGrid.getCellElementExcludingControlColumns(0, 1);
+          const text = await cell.getVisibleText();
+          return text === '17,967';
+        });
+
+        expect(await testSubjects.getVisibleText('dataGridColumnSortingButton')).to.be(
+          'Sort fields\n1'
+        );
+
+        await discover.saveSearch(savedSearchName);
+
+        await header.waitUntilLoadingHasFinished();
+        await discover.waitUntilSearchingHasFinished();
+
+        await retry.waitFor('first cell contains the same highest value', async () => {
+          const cell = await dataGrid.getCellElementExcludingControlColumns(0, 1);
+          const text = await cell.getVisibleText();
+          return text === '17,967';
+        });
+
+        await browser.refresh();
+
+        await header.waitUntilLoadingHasFinished();
+        await discover.waitUntilSearchingHasFinished();
+
+        await retry.waitFor('first cell contains the same highest value after reload', async () => {
+          const cell = await dataGrid.getCellElementExcludingControlColumns(0, 1);
+          const text = await cell.getVisibleText();
+          return text === '17,967';
+        });
+
+        await discover.clickNewSearchButton();
+
+        await header.waitUntilLoadingHasFinished();
+        await discover.waitUntilSearchingHasFinished();
+
+        await discover.loadSavedSearch(savedSearchName);
+
+        await header.waitUntilLoadingHasFinished();
+        await discover.waitUntilSearchingHasFinished();
+
+        await retry.waitFor(
+          'first cell contains the same highest value after reopening',
+          async () => {
+            const cell = await dataGrid.getCellElementExcludingControlColumns(0, 1);
+            const text = await cell.getVisibleText();
+            return text === '17,967';
+          }
+        );
+
+        await dataGrid.clickDocSortDesc('var0', 'Sort Low-High');
+
+        await discover.waitUntilSearchingHasFinished();
+
+        await retry.waitFor('first cell contains the lowest value', async () => {
+          const cell = await dataGrid.getCellElementExcludingControlColumns(0, 1);
+          const text = await cell.getVisibleText();
+          return text === '1';
+        });
+
+        expect(await testSubjects.getVisibleText('dataGridColumnSortingButton')).to.be(
+          'Sort fields\n1'
+        );
+      });
     });
 
     describe('filtering by clicking on the table', () => {