Discover SQL mode: Allow sorting (#158708)

## Summary

Closes https://github.com/elastic/kibana/issues/154331

Enables sorting the table in Discover when in SQL (text based) mode. 
- sorting is performed on the client (inmemory)
- inmemory sorting configuration is not preserved in the appstate (to
prevent unnecesarry data fetches)
- all fields can be sorted, even the ones that can't be sorted on the
elasticsearch side.

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>

src/plugins/discover/public/application/main/components/layout/discover_documents.tsx

@@ -251,7 +251,7 @@ function DiscoverDocumentsComponent({
               useNewFieldsApi={useNewFieldsApi}
               rowHeightState={rowHeight}
               onUpdateRowHeight={onUpdateRowHeight}
-              isSortEnabled={!isPlainRecord}
+              isSortEnabled={true}
               isPlainRecord={isPlainRecord}
               query={query}
               rowsPerPageState={rowsPerPage}

src/plugins/discover/public/components/discover_grid/discover_grid.test.tsx

@@ -209,4 +209,27 @@ describe('DiscoverGrid', () => {
       expect(findTestSubject(component, 'gridEditFieldButton').exists()).toBe(false);
     });
   });
+
+  describe('sorting', () => {
+    it('should enable in memory sorting with plain records', () => {
+      const component = getComponent({
+        ...getProps(),
+        columns: ['message'],
+        isPlainRecord: true,
+      });
+
+      expect(
+        (
+          findTestSubject(component, 'docTable')
+            .find('EuiDataGridInMemoryRenderer')
+            .first()
+            .props() as Record<string, string>
+        ).inMemory
+      ).toMatchInlineSnapshot(`
+        Object {
+          "level": "sorting",
+        }
+      `);
+    });
+  });
 });

src/plugins/discover/public/components/discover_grid/discover_grid.tsx

@@ -362,13 +362,18 @@ export const DiscoverGrid = ({
    */
   const sortingColumns = useMemo(() => sort.map(([id, direction]) => ({ id, direction })), [sort]);
 
+  const [inmemorySortingColumns, setInmemorySortingColumns] = useState([]);
   const onTableSort = useCallback(
     (sortingColumnsData) => {
-      if (isSortEnabled && onSort) {
-        onSort(sortingColumnsData.map(({ id, direction }: SortObj) => [id, direction]));
+      if (isSortEnabled) {
+        if (isPlainRecord) {
+          setInmemorySortingColumns(sortingColumnsData);
+        } else if (onSort) {
+          onSort(sortingColumnsData.map(({ id, direction }: SortObj) => [id, direction]));
+        }
       }
     },
-    [onSort, isSortEnabled]
+    [onSort, isSortEnabled, isPlainRecord, setInmemorySortingColumns]
   );
 
   const showMultiFields = services.uiSettings.get(SHOW_MULTIFIELDS);
@@ -437,6 +442,7 @@ export const DiscoverGrid = ({
         showTimeCol,
         defaultColumns,
         isSortEnabled,
+        isPlainRecord,
         services: {
           uiSettings,
           toastNotifications,
@@ -455,6 +461,7 @@ export const DiscoverGrid = ({
       settings,
       defaultColumns,
       isSortEnabled,
+      isPlainRecord,
       uiSettings,
       toastNotifications,
       dataViewFieldEditor,
@@ -479,10 +486,13 @@ export const DiscoverGrid = ({
   );
   const sorting = useMemo(() => {
     if (isSortEnabled) {
-      return { columns: sortingColumns, onSort: onTableSort };
+      return {
+        columns: isPlainRecord ? inmemorySortingColumns : sortingColumns,
+        onSort: onTableSort,
+      };
     }
     return { columns: sortingColumns, onSort: () => {} };
-  }, [sortingColumns, onTableSort, isSortEnabled]);
+  }, [isSortEnabled, sortingColumns, isPlainRecord, inmemorySortingColumns, onTableSort]);
 
   const canSetExpandedDoc = Boolean(setExpandedDoc && DocumentView);
 
@@ -619,6 +629,7 @@ export const DiscoverGrid = ({
             sorting={sorting as EuiDataGridSorting}
             toolbarVisibility={toolbarVisibility}
             rowHeightsOptions={rowHeightsOptions}
+            inMemory={isPlainRecord ? { level: 'sorting' } : undefined}
             gridStyle={GRID_STYLE}
           />
         </div>

src/plugins/discover/public/components/discover_grid/discover_grid_columns.test.tsx

@@ -21,6 +21,7 @@ describe('Discover grid columns', function () {
       showTimeCol: false,
       defaultColumns: false,
       isSortEnabled: true,
+      isPlainRecord: false,
       valueToStringConverter: discoverGridContextMock.valueToStringConverter,
       rowsCount: 100,
       services: {
@@ -140,6 +141,7 @@ describe('Discover grid columns', function () {
       showTimeCol: false,
       defaultColumns: true,
       isSortEnabled: true,
+      isPlainRecord: false,
       valueToStringConverter: discoverGridContextMock.valueToStringConverter,
       rowsCount: 100,
       services: {
@@ -253,6 +255,7 @@ describe('Discover grid columns', function () {
       showTimeCol: true,
       defaultColumns: false,
       isSortEnabled: true,
+      isPlainRecord: false,
       valueToStringConverter: discoverGridContextMock.valueToStringConverter,
       rowsCount: 100,
       services: {
@@ -429,4 +432,190 @@ describe('Discover grid columns', function () {
       ]
     `);
   });
+
+  it('returns eui grid with inmemory sorting', async () => {
+    const actual = getEuiGridColumns({
+      columns: ['extension', 'message'],
+      settings: {},
+      dataView: dataViewWithTimefieldMock,
+      showTimeCol: true,
+      defaultColumns: false,
+      isSortEnabled: true,
+      isPlainRecord: true,
+      valueToStringConverter: discoverGridContextMock.valueToStringConverter,
+      rowsCount: 100,
+      services: {
+        uiSettings: discoverServiceMock.uiSettings,
+        toastNotifications: discoverServiceMock.toastNotifications,
+      },
+      hasEditDataViewPermission: () =>
+        discoverServiceMock.dataViewFieldEditor.userPermissions.editIndexPattern(),
+      onFilter: () => {},
+    });
+    expect(actual).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "actions": Object {
+            "additional": Array [
+              Object {
+                "data-test-subj": "gridCopyColumnNameToClipBoardButton",
+                "iconProps": Object {
+                  "size": "m",
+                },
+                "iconType": "copyClipboard",
+                "label": <FormattedMessage
+                  defaultMessage="Copy name"
+                  id="discover.grid.copyColumnNameToClipBoardButton"
+                  values={Object {}}
+                />,
+                "onClick": [Function],
+                "size": "xs",
+              },
+              Object {
+                "data-test-subj": "gridCopyColumnValuesToClipBoardButton",
+                "iconProps": Object {
+                  "size": "m",
+                },
+                "iconType": "copyClipboard",
+                "label": <FormattedMessage
+                  defaultMessage="Copy column"
+                  id="discover.grid.copyColumnValuesToClipBoardButton"
+                  values={Object {}}
+                />,
+                "onClick": [Function],
+                "size": "xs",
+              },
+            ],
+            "showHide": false,
+            "showMoveLeft": true,
+            "showMoveRight": true,
+          },
+          "cellActions": Array [
+            [Function],
+            [Function],
+            [Function],
+          ],
+          "display": <div
+            aria-label="timestamp - this field represents the time that events occurred."
+          >
+            <EuiToolTip
+              content="This field represents the time that events occurred."
+              delay="regular"
+              display="inlineBlock"
+              position="top"
+            >
+              <React.Fragment>
+                timestamp
+                 
+                <EuiIcon
+                  type="clock"
+                />
+              </React.Fragment>
+            </EuiToolTip>
+          </div>,
+          "displayAsText": "timestamp",
+          "id": "timestamp",
+          "initialWidth": 210,
+          "isSortable": true,
+          "schema": "datetime",
+        },
+        Object {
+          "actions": Object {
+            "additional": Array [
+              Object {
+                "data-test-subj": "gridCopyColumnNameToClipBoardButton",
+                "iconProps": Object {
+                  "size": "m",
+                },
+                "iconType": "copyClipboard",
+                "label": <FormattedMessage
+                  defaultMessage="Copy name"
+                  id="discover.grid.copyColumnNameToClipBoardButton"
+                  values={Object {}}
+                />,
+                "onClick": [Function],
+                "size": "xs",
+              },
+              Object {
+                "data-test-subj": "gridCopyColumnValuesToClipBoardButton",
+                "iconProps": Object {
+                  "size": "m",
+                },
+                "iconType": "copyClipboard",
+                "label": <FormattedMessage
+                  defaultMessage="Copy column"
+                  id="discover.grid.copyColumnValuesToClipBoardButton"
+                  values={Object {}}
+                />,
+                "onClick": [Function],
+                "size": "xs",
+              },
+            ],
+            "showHide": Object {
+              "iconType": "cross",
+              "label": "Remove column",
+            },
+            "showMoveLeft": true,
+            "showMoveRight": true,
+          },
+          "cellActions": Array [
+            [Function],
+            [Function],
+            [Function],
+          ],
+          "displayAsText": "extension",
+          "id": "extension",
+          "isSortable": true,
+          "schema": "string",
+        },
+        Object {
+          "actions": Object {
+            "additional": Array [
+              Object {
+                "data-test-subj": "gridCopyColumnNameToClipBoardButton",
+                "iconProps": Object {
+                  "size": "m",
+                },
+                "iconType": "copyClipboard",
+                "label": <FormattedMessage
+                  defaultMessage="Copy name"
+                  id="discover.grid.copyColumnNameToClipBoardButton"
+                  values={Object {}}
+                />,
+                "onClick": [Function],
+                "size": "xs",
+              },
+              Object {
+                "data-test-subj": "gridCopyColumnValuesToClipBoardButton",
+                "iconProps": Object {
+                  "size": "m",
+                },
+                "iconType": "copyClipboard",
+                "label": <FormattedMessage
+                  defaultMessage="Copy column"
+                  id="discover.grid.copyColumnValuesToClipBoardButton"
+                  values={Object {}}
+                />,
+                "onClick": [Function],
+                "size": "xs",
+              },
+            ],
+            "showHide": Object {
+              "iconType": "cross",
+              "label": "Remove column",
+            },
+            "showMoveLeft": true,
+            "showMoveRight": true,
+          },
+          "cellActions": Array [
+            [Function],
+          ],
+          "displayAsText": "message",
+          "id": "message",
+          "isSortable": true,
+          "schema": "string",
+        },
+      ]
+    `);
+  });
 });

src/plugins/discover/public/components/discover_grid/discover_grid_columns.tsx

@@ -65,6 +65,7 @@ function buildEuiGridColumn({
   dataView,
   defaultColumns,
   isSortEnabled,
+  isPlainRecord,
   toastNotifications,
   hasEditDataViewPermission,
   valueToStringConverter,
@@ -77,6 +78,7 @@ function buildEuiGridColumn({
   dataView: DataView;
   defaultColumns: boolean;
   isSortEnabled: boolean;
+  isPlainRecord?: boolean;
   toastNotifications: ToastsStart;
   hasEditDataViewPermission: () => boolean;
   valueToStringConverter: ValueToStringConverter;
@@ -99,7 +101,7 @@ function buildEuiGridColumn({
   const column: EuiDataGridColumn = {
     id: columnName,
     schema: getSchemaByKbnType(dataViewField?.type),
-    isSortable: isSortEnabled && dataViewField?.sortable === true,
+    isSortable: isSortEnabled && (isPlainRecord || dataViewField?.sortable === true),
     displayAsText: columnDisplayName,
     actions: {
       showHide:
@@ -176,6 +178,7 @@ export function getEuiGridColumns({
   showTimeCol,
   defaultColumns,
   isSortEnabled,
+  isPlainRecord,
   services,
   hasEditDataViewPermission,
   valueToStringConverter,
@@ -189,6 +192,7 @@ export function getEuiGridColumns({
   showTimeCol: boolean;
   defaultColumns: boolean;
   isSortEnabled: boolean;
+  isPlainRecord?: boolean;
   services: {
     uiSettings: IUiSettingsClient;
     toastNotifications: ToastsStart;
@@ -213,6 +217,7 @@ export function getEuiGridColumns({
       dataView,
       defaultColumns,
       isSortEnabled,
+      isPlainRecord,
       toastNotifications: services.toastNotifications,
       hasEditDataViewPermission,
       valueToStringConverter,

src/plugins/discover/public/embeddable/saved_search_embeddable.tsx

@@ -258,7 +258,7 @@ export class SavedSearchEmbeddable
         this.searchProps!.isLoading = false;
         this.searchProps!.isPlainRecord = true;
         this.searchProps!.showTimeCol = false;
-        this.searchProps!.isSortEnabled = false;
+        this.searchProps!.isSortEnabled = true;
         return;
       }
 

test/functional/apps/discover/group2/_sql_view.ts

@@ -77,8 +77,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         expect(await testSubjects.exists('unifiedHistogramQueryHits')).to.be(true);
         expect(await testSubjects.exists('discoverAlertsButton')).to.be(false);
         expect(await testSubjects.exists('shareTopNavButton')).to.be(true);
+        expect(await testSubjects.exists('dataGridColumnSortingButton')).to.be(true);
         expect(await testSubjects.exists('docTableExpandToggleColumn')).to.be(true);
-        expect(await testSubjects.exists('dataGridColumnSortingButton')).to.be(false);
         expect(await testSubjects.exists('fieldListFiltersFieldTypeFilterToggle')).to.be(true);
         await testSubjects.click('field-@message-showDetails');
         expect(await testSubjects.exists('discoverFieldListPanelEditItem')).to.be(false);