[Lens] Datatable improvements (#174994)

## Summary

Fixes #160719 and #164413

This PR contains some work about Lens datatable, here's a short list:

* moved out the sorting logic of the datatable into an independent
package: `@kbn/sort-predicates`
* leverage the EUI Datagrid `schemaDetectors` for the table rows sorting
* apply datatable columns sorting also to the CSV exporter

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -770,6 +770,7 @@ x-pack/packages/kbn-slo-schema @elastic/obs-ux-management-team
 x-pack/plugins/snapshot_restore @elastic/platform-deployment-management
 packages/kbn-some-dev-log @elastic/kibana-operations
 packages/kbn-sort-package-json @elastic/kibana-operations
+packages/kbn-sort-predicates @elastic/kibana-visualizations
 x-pack/plugins/spaces @elastic/kibana-security
 x-pack/test/spaces_api_integration/common/plugins/spaces_test_plugin @elastic/kibana-security
 packages/kbn-spec-to-console @elastic/platform-deployment-management

package.json

@@ -766,6 +766,7 @@
     "@kbn/shared-ux-utility": "link:packages/kbn-shared-ux-utility",
     "@kbn/slo-schema": "link:x-pack/packages/kbn-slo-schema",
     "@kbn/snapshot-restore-plugin": "link:x-pack/plugins/snapshot_restore",
+    "@kbn/sort-predicates": "link:packages/kbn-sort-predicates",
     "@kbn/spaces-plugin": "link:x-pack/plugins/spaces",
     "@kbn/spaces-test-plugin": "link:x-pack/test/spaces_api_integration/common/plugins/spaces_test_plugin",
     "@kbn/stack-alerts-plugin": "link:x-pack/plugins/stack_alerts",

packages/kbn-sort-predicates/README.md

@@ -0,0 +1,74 @@
+# @kbn/sort-predicates
+
+This package contains a flexible sorting function who supports the following types:
+
+* string
+* number
+* version
+* ip addresses (both IPv4 and IPv6) - handles `Others`/strings correcly in this case
+* dates
+* ranges open and closed (number type only for now)
+* null and undefined (always sorted as last entries, no matter the direction)
+* any multi-value version of the types above (version excluded)
+
+The function is intended to use with objects and to simplify the usage with sorting by a specific column/field.
+The functions has been extracted from Lens datatable where it was originally used.
+
+### How to use it
+
+Basic usage with an array of objects:
+
+```js
+import { getSortingCriteria } from '@kbn/sorting-predicates';
+
+...
+const predicate = getSortingCriteria( typeHint, columnId, formatterFn );
+
+const orderedRows = [{a: 1, b: 2}, {a: 3, b: 4}]
+    .sort( (rowA, rowB) => predicate(rowA, rowB, 'asc' /* or 'desc' */));
+```
+
+Basic usage with EUI DataGrid schemaDetector:
+
+```tsx
+const [data, setData] = useState(table);
+const dataGridColumns: EuiDataGridColumn[] = data.columns.map( (column) => ({
+    ...
+    schema: getColumnType(column)
+}));
+const [sortingColumns, setSortingColumns] = useState([
+  { id: 'custom', direction: 'asc' },
+]);
+
+const schemaDetectors = dataGridColumns.map((column) => {
+    const sortingHint = getColumnType(column);
+    const sortingCriteria = getSortingCriteria(
+        sortingHint,
+        column.id,
+        (val: unknwon) => String(val)
+    );
+    return {
+        sortTextAsc: 'asc'
+        sortTextDesc: 'desc',
+        icon: 'starFilled',
+        type: sortingHint || '',
+        detector: () => 1,
+        // This is the actual logic that is used to sort the table
+        comparator: (_a, _b, direction, { aIndex, bIndex }) =>
+        sortingCriteria(data.rows[aIndex], data.rows[bIndex], direction) as 0 | 1 | -1
+    };
+});
+
+return <EuiDataGrid
+    ...
+    inMemory={{ level: 'sorting' }}
+    columns={dataGridColumns}
+    schemaDetectors={schemaDetectors}
+    sorting={{
+        columns: sortingColumns,
+        // this is called only for those columns not covered by the schema detector
+        // and can use the sorting predica as well, manually applied to the data rows
+        onSort: () => { ... }
+    }}
+/>;
+```

packages/kbn-sort-predicates/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { getSortingCriteria } from './src/sorting';

packages/kbn-sort-predicates/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-sort-predicates'],
+};

packages/kbn-sort-predicates/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/sort-predicates",
+  "owner": "@elastic/kibana-visualizations"
+}

packages/kbn-sort-predicates/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/sort-predicates",
+  "private": true,
+  "version": "1.0.0",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "sideEffects": false
+}

packages/kbn-sort-predicates/src/sorting.test.ts

@@ -1,8 +1,9 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
 import { getSortingCriteria } from './sorting';
@@ -40,8 +41,8 @@ function testSorting({
       sorted.push(firstEl);
     }
   }
-  const criteria = getSortingCriteria(type, 'a', getMockFormatter(), direction);
-  expect(datatable.sort(criteria).map((row) => row.a)).toEqual(sorted);
+  const criteria = getSortingCriteria(type, 'a', getMockFormatter());
+  expect(datatable.sort((a, b) => criteria(a, b, direction)).map((row) => row.a)).toEqual(sorted);
 }
 
 describe('Data sorting criteria', () => {

packages/kbn-sort-predicates/src/sorting.ts

@@ -1,8 +1,9 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
 import versionCompare from 'compare-versions';
@@ -130,9 +131,15 @@ const rangeComparison: CompareFn<Omit<Range, 'type'>> = (v1, v2) => {
   return fromComparison || toComparison || 0;
 };
 
-function createArrayValuesHandler(sortBy: string, directionFactor: number, formatter: FieldFormat) {
+function createArrayValuesHandler(sortBy: string, formatter: FieldFormat) {
   return function <T>(criteriaFn: CompareFn<T>) {
-    return (rowA: Record<string, unknown>, rowB: Record<string, unknown>) => {
+    return (
+      rowA: Record<string, unknown>,
+      rowB: Record<string, unknown>,
+      direction: 'asc' | 'desc'
+    ) => {
+      // handle the direction with a multiply factor.
+      const directionFactor = direction === 'asc' ? 1 : -1;
       // if either side of the comparison is an array, make it also the other one become one
       // then perform an array comparison
       if (Array.isArray(rowA[sortBy]) || Array.isArray(rowB[sortBy])) {
@@ -157,13 +164,21 @@ function createArrayValuesHandler(sortBy: string, directionFactor: number, forma
 
 function getUndefinedHandler(
   sortBy: string,
-  sortingCriteria: (rowA: Record<string, unknown>, rowB: Record<string, unknown>) => number
+  sortingCriteria: (
+    rowA: Record<string, unknown>,
+    rowB: Record<string, unknown>,
+    directionFactor: 'asc' | 'desc'
+  ) => number
 ) {
-  return (rowA: Record<string, unknown>, rowB: Record<string, unknown>) => {
+  return (
+    rowA: Record<string, unknown>,
+    rowB: Record<string, unknown>,
+    direction: 'asc' | 'desc'
+  ) => {
     const valueA = rowA[sortBy];
     const valueB = rowB[sortBy];
     if (valueA != null && valueB != null && !Number.isNaN(valueA) && !Number.isNaN(valueB)) {
-      return sortingCriteria(rowA, rowB);
+      return sortingCriteria(rowA, rowB, direction);
     }
     if (valueA == null || Number.isNaN(valueA)) {
       return 1;
@@ -179,13 +194,9 @@ function getUndefinedHandler(
 export function getSortingCriteria(
   type: string | undefined,
   sortBy: string,
-  formatter: FieldFormat,
-  direction: string
+  formatter: FieldFormat
 ) {
-  // handle the direction with a multiply factor.
-  const directionFactor = direction === 'asc' ? 1 : -1;
-
-  const arrayValueHandler = createArrayValuesHandler(sortBy, directionFactor, formatter);
+  const arrayValueHandler = createArrayValuesHandler(sortBy, formatter);
 
   if (['number', 'date'].includes(type || '')) {
     return getUndefinedHandler(sortBy, arrayValueHandler(numberCompare));

packages/kbn-sort-predicates/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+    ]
+  },
+  "include": [
+    "**/*.ts"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/field-formats-plugin",
+    "@kbn/expressions-plugin"
+  ]
+}

src/plugins/data/common/exports/export_csv.test.ts

@@ -96,4 +96,14 @@ describe('CSV exporter', () => {
       })
     ).toMatch('columnOne\r\n"a,b"\r\n');
   });
+
+  test('should respect the sorted columns order when passed', () => {
+    const datatable = getDataTable({ multipleColumns: true });
+    expect(
+      datatableToCSV(datatable, {
+        ...getDefaultOptions(),
+        columnsSorting: ['col2', 'col1'],
+      })
+    ).toMatch('columnTwo,columnOne\r\n"Formatted_5","Formatted_value"\r\n');
+  });
 });

src/plugins/data/common/exports/export_csv.tsx

@@ -21,39 +21,51 @@ interface CSVOptions {
   escapeFormulaValues: boolean;
   formatFactory: FormatFactory;
   raw?: boolean;
+  columnsSorting?: string[];
 }
 
 export function datatableToCSV(
   { columns, rows }: Datatable,
-  { csvSeparator, quoteValues, formatFactory, raw, escapeFormulaValues }: CSVOptions
+  { csvSeparator, quoteValues, formatFactory, raw, escapeFormulaValues, columnsSorting }: CSVOptions
 ) {
   const escapeValues = createEscapeValue({
     separator: csvSeparator,
     quoteValues,
     escapeFormulaValues,
   });
+
+  const sortedIds = columnsSorting || columns.map((col) => col.id);
+
+  // Build an index lookup table
+  const columnIndexLookup = sortedIds.reduce((memo, id, index) => {
+    memo[id] = index;
+    return memo;
+  }, {} as Record<string, number>);
+
   // Build the header row by its names
-  const header = columns.map((col) => escapeValues(col.name));
+  const header: string[] = [];
+  const sortedColumnIds: string[] = [];
+  const formatters: Record<string, ReturnType<FormatFactory>> = {};
 
-  const formatters = columns.reduce<Record<string, ReturnType<FormatFactory>>>(
-    (memo, { id, meta }) => {
-      memo[id] = formatFactory(meta?.params);
-      return memo;
-    },
-    {}
-  );
+  for (const column of columns) {
+    const columnIndex = columnIndexLookup[column.id];
 
-  // Convert the array of row objects to an array of row arrays
-  const csvRows = rows.map((row) => {
-    return columns.map((column) =>
-      escapeValues(raw ? row[column.id] : formatters[column.id].convert(row[column.id]))
-    );
-  });
+    header[columnIndex] = escapeValues(column.name);
+    sortedColumnIds[columnIndex] = column.id;
+    formatters[column.id] = formatFactory(column.meta?.params);
+  }
 
   if (header.length === 0) {
     return '';
   }
 
+  // Convert the array of row objects to an array of row arrays
+  const csvRows = rows.map((row) => {
+    return sortedColumnIds.map((id) =>
+      escapeValues(raw ? row[id] : formatters[id].convert(row[id]))
+    );
+  });
+
   return (
     [header, ...csvRows].map((row) => row.join(csvSeparator)).join(LINE_FEED_CHARACTER) +
     LINE_FEED_CHARACTER

tsconfig.base.json

@@ -1534,6 +1534,8 @@
       "@kbn/some-dev-log/*": ["packages/kbn-some-dev-log/*"],
       "@kbn/sort-package-json": ["packages/kbn-sort-package-json"],
       "@kbn/sort-package-json/*": ["packages/kbn-sort-package-json/*"],
+      "@kbn/sort-predicates": ["packages/kbn-sort-predicates"],
+      "@kbn/sort-predicates/*": ["packages/kbn-sort-predicates/*"],
       "@kbn/spaces-plugin": ["x-pack/plugins/spaces"],
       "@kbn/spaces-plugin/*": ["x-pack/plugins/spaces/*"],
       "@kbn/spaces-test-plugin": ["x-pack/test/spaces_api_integration/common/plugins/spaces_test_plugin"],

x-pack/plugins/lens/common/expressions/datatable/datatable_fn.ts

@@ -8,21 +8,12 @@
 import { cloneDeep } from 'lodash';
 import { i18n } from '@kbn/i18n';
 import { prepareLogTable } from '@kbn/visualizations-plugin/common/utils';
-import type {
-  Datatable,
-  DatatableColumnMeta,
-  ExecutionContext,
-} from '@kbn/expressions-plugin/common';
+import type { Datatable, ExecutionContext } from '@kbn/expressions-plugin/common';
 import { FormatFactory } from '../../types';
 import { transposeTable } from './transpose_helpers';
 import { computeSummaryRowForColumn } from './summary';
-import { getSortingCriteria } from './sorting';
 import type { DatatableExpressionFunction } from './types';
 
-function isRange(meta: { params?: { id?: string } } | undefined) {
-  return meta?.params?.id === 'range';
-}
-
 export const datatableFn =
   (
     getFormatFactory: (context: ExecutionContext) => FormatFactory | Promise<FormatFactory>
@@ -49,8 +40,6 @@ export const datatableFn =
     }
 
     let untransposedData: Datatable | undefined;
-    // do the sorting at this level to propagate it also at CSV download
-    const [layerId] = Object.keys(context.inspectorAdapters.tables || {});
 
     const formatters: Record<string, ReturnType<FormatFactory>> = {};
     const formatFactory = await getFormatFactory(context);
@@ -67,15 +56,6 @@ export const datatableFn =
       transposeTable(args, table, formatters);
     }
 
-    const { sortingColumnId: sortBy, sortingDirection: sortDirection } = args;
-
-    const columnsReverseLookup = table.columns.reduce<
-      Record<string, { name: string; index: number; meta?: DatatableColumnMeta }>
-    >((memo, { id, name, meta }, i) => {
-      memo[id] = { name, index: i, meta };
-      return memo;
-    }, {});
-
     const columnsWithSummary = args.columns.filter((c) => c.summaryRow);
     for (const column of columnsWithSummary) {
       column.summaryRowValue = computeSummaryRowForColumn(
@@ -86,29 +66,6 @@ export const datatableFn =
       );
     }
 
-    if (sortBy && columnsReverseLookup[sortBy] && sortDirection !== 'none') {
-      const sortingHint = args.columns.find((col) => col.columnId === sortBy)?.sortingHint;
-      // Sort on raw values for these types, while use the formatted value for the rest
-      const sortingCriteria = getSortingCriteria(
-        sortingHint ??
-          (isRange(columnsReverseLookup[sortBy]?.meta)
-            ? 'range'
-            : columnsReverseLookup[sortBy]?.meta?.type),
-        sortBy,
-        formatters[sortBy],
-        sortDirection
-      );
-      // replace the table here
-      context.inspectorAdapters.tables[layerId].rows = (table.rows || [])
-        .slice()
-        .sort(sortingCriteria);
-      // replace also the local copy
-      table.rows = context.inspectorAdapters.tables[layerId].rows;
-    } else {
-      args.sortingColumnId = undefined;
-      args.sortingDirection = 'none';
-    }
-
     return {
       type: 'render',
       as: 'lens_datatable_renderer',

x-pack/plugins/lens/public/app_plugin/csv_download_provider/csv_download_provider.tsx

@@ -30,11 +30,13 @@ async function downloadCSVs({
   title,
   formatFactory,
   uiSettings,
+  columnsSorting,
 }: {
   title: string;
   activeData: TableInspectorAdapter;
   formatFactory: FormatFactory;
   uiSettings: IUiSettingsClient;
+  columnsSorting?: string[];
 }) {
   if (!activeData) {
     if (window.ELASTIC_LENS_CSV_DOWNLOAD_DEBUG) {
@@ -55,6 +57,7 @@ async function downloadCSVs({
             quoteValues: uiSettings.get('csv:quoteValues', true),
             formatFactory,
             escapeFormulaValues: false,
+            columnsSorting,
           }),
           type: exporters.CSV_MIME_TYPE,
         };
@@ -104,10 +107,11 @@ export const downloadCsvShareProvider = ({
       return [];
     }
 
-    const { title, activeData, csvEnabled } = sharingData as {
+    const { title, activeData, csvEnabled, columnsSorting } = sharingData as {
       title: string;
       activeData: TableInspectorAdapter;
       csvEnabled: boolean;
+      columnsSorting?: string[];
     };
 
     const panelTitle = i18n.translate(
@@ -138,6 +142,7 @@ export const downloadCsvShareProvider = ({
                   formatFactory: formatFactoryFn(),
                   activeData,
                   uiSettings,
+                  columnsSorting,
                 });
                 onClose?.();
               }}

x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx

@@ -37,6 +37,7 @@ import { combineQueryAndFilters, getLayerMetaInfo } from './show_underlying_data
 import { changeIndexPattern } from '../state_management/lens_slice';
 import { LensByReferenceInput } from '../embeddable';
 import { DEFAULT_LENS_LAYOUT_DIMENSIONS, getShareURL } from './share_action';
+import { getDatasourceLayers } from '../state_management/utils';
 
 function getSaveButtonMeta({
   contextFromEmbeddable,
@@ -602,8 +603,13 @@ export const LensTopNavMenu = ({
               shareUrlEnabled,
               isCurrentStateDirty
             );
+
             const sharingData = {
               activeData,
+              columnsSorting: visualizationMap[visualization.activeId].getSortedColumns?.(
+                visualization.state,
+                getDatasourceLayers(datasourceStates, datasourceMap, dataViews.indexPatterns)
+              ),
               csvEnabled,
               reportingDisabled: !csvEnabled,
               title: title || defaultLensTitle,

x-pack/plugins/lens/public/types.ts

@@ -1320,6 +1320,10 @@ export interface Visualization<T = unknown, P = T, ExtraAppendLayerArg = unknown
    * A visualization can return custom dimensions for the reporting tool
    */
   getReportingLayout?: (state: T) => { height: number; width: number };
+  /**
+   * A visualization can share how columns are visually sorted
+   */
+  getSortedColumns?: (state: T, datasourceLayers?: DatasourceLayers) => string[];
   /**
    * returns array of telemetry events for the visualization on save
    */

x-pack/plugins/lens/public/visualizations/datatable/components/__snapshots__/table_basic.test.tsx.snap

@@ -141,6 +141,7 @@ exports[`DatatableComponent it renders actions column when there are row actions
             </div>,
             "displayAsText": "a",
             "id": "a",
+            "schema": "a",
             "visibleCellActions": 5,
           },
           Object {
@@ -191,6 +192,7 @@ exports[`DatatableComponent it renders actions column when there are row actions
             </div>,
             "displayAsText": "b",
             "id": "b",
+            "schema": "b",
             "visibleCellActions": 5,
           },
           Object {
@@ -241,6 +243,7 @@ exports[`DatatableComponent it renders actions column when there are row actions
             </div>,
             "displayAsText": "c",
             "id": "c",
+            "schema": "c",
             "visibleCellActions": 5,
           },
         ]
@@ -252,6 +255,11 @@ exports[`DatatableComponent it renders actions column when there are row actions
           "header": "underline",
         }
       }
+      inMemory={
+        Object {
+          "level": "sorting",
+        }
+      }
       onColumnResize={[Function]}
       renderCellValue={[Function]}
       rowCount={1}
@@ -260,6 +268,37 @@ exports[`DatatableComponent it renders actions column when there are row actions
           "defaultHeight": undefined,
         }
       }
+      schemaDetectors={
+        Array [
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "a",
+          },
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "b",
+          },
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "c",
+          },
+        ]
+      }
       sorting={
         Object {
           "columns": Array [],
@@ -419,6 +458,7 @@ exports[`DatatableComponent it renders custom row height if set to another value
             </div>,
             "displayAsText": "a",
             "id": "a",
+            "schema": "a",
             "visibleCellActions": 5,
           },
           Object {
@@ -469,6 +509,7 @@ exports[`DatatableComponent it renders custom row height if set to another value
             </div>,
             "displayAsText": "b",
             "id": "b",
+            "schema": "b",
             "visibleCellActions": 5,
           },
           Object {
@@ -519,6 +560,7 @@ exports[`DatatableComponent it renders custom row height if set to another value
             </div>,
             "displayAsText": "c",
             "id": "c",
+            "schema": "c",
             "visibleCellActions": 5,
           },
         ]
@@ -530,6 +572,11 @@ exports[`DatatableComponent it renders custom row height if set to another value
           "header": "underline",
         }
       }
+      inMemory={
+        Object {
+          "level": "sorting",
+        }
+      }
       onColumnResize={[Function]}
       renderCellValue={[Function]}
       rowCount={1}
@@ -540,6 +587,37 @@ exports[`DatatableComponent it renders custom row height if set to another value
           },
         }
       }
+      schemaDetectors={
+        Array [
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "a",
+          },
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "b",
+          },
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "c",
+          },
+        ]
+      }
       sorting={
         Object {
           "columns": Array [],
@@ -690,6 +768,7 @@ exports[`DatatableComponent it renders the title and value 1`] = `
             </div>,
             "displayAsText": "a",
             "id": "a",
+            "schema": "a",
             "visibleCellActions": 5,
           },
           Object {
@@ -740,6 +819,7 @@ exports[`DatatableComponent it renders the title and value 1`] = `
             </div>,
             "displayAsText": "b",
             "id": "b",
+            "schema": "b",
             "visibleCellActions": 5,
           },
           Object {
@@ -790,6 +870,7 @@ exports[`DatatableComponent it renders the title and value 1`] = `
             </div>,
             "displayAsText": "c",
             "id": "c",
+            "schema": "c",
             "visibleCellActions": 5,
           },
         ]
@@ -801,6 +882,11 @@ exports[`DatatableComponent it renders the title and value 1`] = `
           "header": "underline",
         }
       }
+      inMemory={
+        Object {
+          "level": "sorting",
+        }
+      }
       onColumnResize={[Function]}
       renderCellValue={[Function]}
       rowCount={1}
@@ -809,6 +895,37 @@ exports[`DatatableComponent it renders the title and value 1`] = `
           "defaultHeight": undefined,
         }
       }
+      schemaDetectors={
+        Array [
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "a",
+          },
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "b",
+          },
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "c",
+          },
+        ]
+      }
       sorting={
         Object {
           "columns": Array [],
@@ -963,6 +1080,7 @@ exports[`DatatableComponent it should render hide, reset, and sort actions on he
             </div>,
             "displayAsText": "a",
             "id": "a",
+            "schema": "a",
             "visibleCellActions": 5,
           },
           Object {
@@ -1013,6 +1131,7 @@ exports[`DatatableComponent it should render hide, reset, and sort actions on he
             </div>,
             "displayAsText": "b",
             "id": "b",
+            "schema": "b",
             "visibleCellActions": 5,
           },
           Object {
@@ -1063,6 +1182,7 @@ exports[`DatatableComponent it should render hide, reset, and sort actions on he
             </div>,
             "displayAsText": "c",
             "id": "c",
+            "schema": "c",
             "visibleCellActions": 5,
           },
         ]
@@ -1074,6 +1194,11 @@ exports[`DatatableComponent it should render hide, reset, and sort actions on he
           "header": "underline",
         }
       }
+      inMemory={
+        Object {
+          "level": "sorting",
+        }
+      }
       onColumnResize={[Function]}
       renderCellValue={[Function]}
       rowCount={1}
@@ -1082,6 +1207,37 @@ exports[`DatatableComponent it should render hide, reset, and sort actions on he
           "defaultHeight": undefined,
         }
       }
+      schemaDetectors={
+        Array [
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "a",
+          },
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "b",
+          },
+          Object {
+            "comparator": [Function],
+            "defaultSortDirection": undefined,
+            "detector": [Function],
+            "icon": "",
+            "sortTextAsc": "Sort Ascending",
+            "sortTextDesc": "Sort Descending",
+            "type": "c",
+          },
+        ]
+      }
       sorting={
         Object {
           "columns": Array [],

x-pack/plugins/lens/public/visualizations/datatable/components/columns.tsx

@@ -13,16 +13,13 @@ import {
   EuiDataGridColumnCellActionProps,
   EuiListGroupItemProps,
 } from '@elastic/eui';
-import type {
-  Datatable,
-  DatatableColumn,
-  DatatableColumnMeta,
-} from '@kbn/expressions-plugin/common';
+import type { Datatable, DatatableColumn } from '@kbn/expressions-plugin/common';
 import { EuiDataGridColumnCellAction } from '@elastic/eui/src/components/datagrid/data_grid_types';
 import { FILTER_CELL_ACTION_TYPE } from '@kbn/cell-actions/constants';
 import type { FormatFactory } from '../../../../common/types';
 import type { ColumnConfig } from '../../../../common/expressions';
 import { LensCellValueAction } from '../../../types';
+import { buildColumnsMetaLookup } from './helpers';
 
 const hasFilterCellAction = (actions: LensCellValueAction[]) => {
   return actions.some(({ type }) => type === FILTER_CELL_ACTION_TYPE);
@@ -59,12 +56,7 @@ export const createGridColumns = (
   closeCellPopover?: Function,
   columnFilterable?: boolean[]
 ) => {
-  const columnsReverseLookup = table.columns.reduce<
-    Record<string, { name: string; index: number; meta?: DatatableColumnMeta }>
-  >((memo, { id, name, meta }, i) => {
-    memo[id] = { name, index: i, meta };
-    return memo;
-  }, {});
+  const columnsReverseLookup = buildColumnsMetaLookup(table);
 
   const getContentData = ({
     rowIndex,
@@ -288,6 +280,7 @@ export const createGridColumns = (
       visibleCellActions: 5,
       display: <div css={columnStyle}>{name}</div>,
       displayAsText: name,
+      schema: field,
       actions: {
         showHide: false,
         showMoveLeft: false,

x-pack/plugins/lens/public/visualizations/datatable/components/helpers.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Datatable, DatatableColumnMeta } from '@kbn/expressions-plugin/common';
+import memoizeOne from 'memoize-one';
+
+function buildColumnsMetaLookupInner(table: Datatable) {
+  return table.columns.reduce<
+    Record<string, { name: string; index: number; meta?: DatatableColumnMeta }>
+  >((memo, { id, name, meta }, i) => {
+    memo[id] = { name, index: i, meta };
+    return memo;
+  }, {});
+}
+
+export const buildColumnsMetaLookup = memoizeOne(buildColumnsMetaLookupInner);

x-pack/plugins/lens/public/visualizations/datatable/components/table_actions.ts

@@ -5,12 +5,28 @@
  * 2.0.
  */
 
-import type { EuiDataGridSorting } from '@elastic/eui';
-import type { Datatable, DatatableColumn } from '@kbn/expressions-plugin/common';
+import type {
+  EuiDataGridColumn,
+  EuiDataGridSchemaDetector,
+  EuiDataGridSorting,
+} from '@elastic/eui';
+import type {
+  Datatable,
+  DatatableColumn,
+  DatatableColumnMeta,
+} from '@kbn/expressions-plugin/common';
 import { ClickTriggerEvent } from '@kbn/charts-plugin/public';
+import { getSortingCriteria } from '@kbn/sort-predicates';
+import { i18n } from '@kbn/i18n';
 import type { LensResizeAction, LensSortAction, LensToggleAction } from './types';
-import type { ColumnConfig, LensGridDirection } from '../../../../common/expressions';
+import type {
+  ColumnConfig,
+  ColumnConfigArg,
+  LensGridDirection,
+} from '../../../../common/expressions';
 import { getOriginalId } from '../../../../common/expressions/datatable/transpose_helpers';
+import type { FormatFactory } from '../../../../common/types';
+import { buildColumnsMetaLookup } from './helpers';
 
 export const createGridResizeHandler =
   (
@@ -73,7 +89,7 @@ export const createGridFilterHandler =
     tableRef: React.MutableRefObject<Datatable>,
     onClickValue: (data: ClickTriggerEvent['data']) => void
   ) =>
-  (field: string, value: unknown, colIndex: number, rowIndex: number, negate: boolean = false) => {
+  (_field: string, value: unknown, colIndex: number, rowIndex: number, negate: boolean = false) => {
     const data: ClickTriggerEvent['data'] = {
       negate,
       data: [
@@ -151,3 +167,68 @@ export const createGridSortingConfig = (
     });
   },
 });
+
+function isRange(meta: { params?: { id?: string } } | undefined) {
+  return meta?.params?.id === 'range';
+}
+
+function getColumnType({
+  columnConfig,
+  columnId,
+  lookup,
+}: {
+  columnConfig: ColumnConfig;
+  columnId: string;
+  lookup: Record<
+    string,
+    {
+      name: string;
+      index: number;
+      meta?: DatatableColumnMeta | undefined;
+    }
+  >;
+}) {
+  const sortingHint = columnConfig.columns.find((col) => col.columnId === columnId)?.sortingHint;
+  return sortingHint ?? (isRange(lookup[columnId]?.meta) ? 'range' : lookup[columnId]?.meta?.type);
+}
+
+export const buildSchemaDetectors = (
+  columns: EuiDataGridColumn[],
+  columnConfig: {
+    columns: ColumnConfigArg[];
+    sortingColumnId: string | undefined;
+    sortingDirection: 'none' | 'asc' | 'desc';
+  },
+  table: Datatable,
+  formatters: Record<string, ReturnType<FormatFactory>>
+): EuiDataGridSchemaDetector[] => {
+  const columnsReverseLookup = buildColumnsMetaLookup(table);
+
+  return columns.map((column) => {
+    const schemaType = getColumnType({
+      columnConfig,
+      columnId: column.id,
+      lookup: columnsReverseLookup,
+    });
+    const sortingCriteria = getSortingCriteria(schemaType, column.id, formatters?.[column.id]);
+    return {
+      sortTextAsc: i18n.translate('xpack.lens.datatable.sortTextAsc', {
+        defaultMessage: 'Sort Ascending',
+      }),
+      sortTextDesc: i18n.translate('xpack.lens.datatable.sortTextDesc', {
+        defaultMessage: 'Sort Descending',
+      }),
+      icon: '',
+      type: column.id,
+      detector: () => 1,
+      // This is the actual logic that is used to sort the table
+      comparator: (_a, _b, direction, { aIndex, bIndex }) =>
+        sortingCriteria(table.rows[aIndex], table.rows[bIndex], direction) as 0 | 1 | -1,
+      // When the SO is updated, then this property will trigger a re-sort of the table
+      defaultSortDirection:
+        columnConfig.sortingColumnId === column.id && columnConfig.sortingDirection !== 'none'
+          ? columnConfig.sortingDirection
+          : undefined,
+    };
+  });
+};

x-pack/plugins/lens/public/visualizations/datatable/components/table_basic.tsx

@@ -46,6 +46,7 @@ import type {
 import { createGridColumns } from './columns';
 import { createGridCell } from './cell_value';
 import {
+  buildSchemaDetectors,
   createGridFilterHandler,
   createGridHideHandler,
   createGridResizeHandler,
@@ -244,8 +245,6 @@ export const DatatableComponent = (props: DatatableRenderProps) => {
     [columnConfig]
   );
 
-  const { sortingColumnId: sortBy, sortingDirection: sortDirection } = props.args;
-
   const isReadOnlySorted = renderMode !== 'edit';
 
   const onColumnResize = useMemo(
@@ -337,6 +336,11 @@ export const DatatableComponent = (props: DatatableRenderProps) => {
     ]
   );
 
+  const schemaDetectors = useMemo(
+    () => buildSchemaDetectors(columns, columnConfig, firstLocalTable, formatters),
+    [columns, firstLocalTable, columnConfig, formatters]
+  );
+
   const trailingControlColumns: EuiDataGridControlColumn[] = useMemo(() => {
     if (!hasAtLeastOneRowClickAction || !onRowContextMenuClick || !isInteractive) {
       return [];
@@ -400,8 +404,13 @@ export const DatatableComponent = (props: DatatableRenderProps) => {
   );
 
   const sorting = useMemo<EuiDataGridSorting | undefined>(
-    () => createGridSortingConfig(sortBy, sortDirection as LensGridDirection, onEditAction),
-    [onEditAction, sortBy, sortDirection]
+    () =>
+      createGridSortingConfig(
+        columnConfig.sortingColumnId,
+        columnConfig.sortingDirection as LensGridDirection,
+        onEditAction
+      ),
+    [onEditAction, columnConfig]
   );
 
   const renderSummaryRow = useMemo(() => {
@@ -476,12 +485,14 @@ export const DatatableComponent = (props: DatatableRenderProps) => {
                 }
               : undefined,
           }}
+          inMemory={{ level: 'sorting' }}
           columns={columns}
           columnVisibility={columnVisibility}
           trailingControlColumns={trailingControlColumns}
           rowCount={firstLocalTable.rows.length}
           renderCellValue={renderCellValue}
           gridStyle={gridStyle}
+          schemaDetectors={schemaDetectors}
           sorting={sorting}
           pagination={
             pagination && {

x-pack/plugins/lens/public/visualizations/datatable/visualization.tsx

@@ -631,6 +631,12 @@ export const getDatatableVisualization = ({
     return suggestion;
   },
 
+  getSortedColumns(state, datasourceLayers) {
+    const { sortedColumns } =
+      getDataSourceAndSortedColumns(state, datasourceLayers || {}, state.layerId) || {};
+    return sortedColumns;
+  },
+
   getVisualizationInfo(state) {
     const visibleMetricColumns = state.columns.filter(
       (c) => !c.hidden && c.colorMode && c.colorMode !== 'none'

x-pack/plugins/lens/tsconfig.json

@@ -105,7 +105,8 @@
     "@kbn/visualization-utils",
     "@kbn/test-eui-helpers",
     "@kbn/shared-ux-utility",
-    "@kbn/text-based-editor"
+    "@kbn/text-based-editor",
+    "@kbn/sort-predicates"
   ],
   "exclude": ["target/**/*"]
 }

x-pack/test/functional/apps/lens/group2/table.ts

@@ -124,6 +124,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         from: 'lnsDatatable_rows > lns-dimensionTrigger',
         to: 'lnsDatatable_columns > lns-empty-dimension',
       });
+      // await PageObjects.common.sleep(100000);
       expect(await PageObjects.lens.getDatatableHeaderText(0)).to.equal('@timestamp per 3 hours');
       expect(await PageObjects.lens.getDatatableHeaderText(1)).to.equal(
         '169.228.188.120 › Average of bytes'

x-pack/test/functional/page_objects/lens_page.ts

@@ -1173,7 +1173,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont
 
     async getDatatableCell(rowIndex = 0, colIndex = 0) {
       return await find.byCssSelector(
-        `[data-test-subj="lnsDataTable"] [data-test-subj="dataGridRowCell"][data-gridcell-column-index="${colIndex}"][data-gridcell-row-index="${rowIndex}"]`
+        `[data-test-subj="lnsDataTable"] [data-test-subj="dataGridRowCell"][data-gridcell-column-index="${colIndex}"][data-gridcell-visible-row-index="${rowIndex}"]`
       );
     },
 

yarn.lock

@@ -6128,6 +6128,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/sort-predicates@link:packages/kbn-sort-predicates":
+  version "0.0.0"
+  uid ""
+
 "@kbn/spaces-plugin@link:x-pack/plugins/spaces":
   version "0.0.0"
   uid ""