[Lens][Datatable] Fix share export and inspect data (#193780)

The exported table data table provided in the inspector and the share export now match what was visible in the UI.

.github/CODEOWNERS

@@ -947,6 +947,7 @@ packages/kbn-tinymath @elastic/kibana-visualizations
 packages/kbn-tooling-log @elastic/kibana-operations
 x-pack/plugins/transform @elastic/ml-ui
 x-pack/plugins/translations @elastic/kibana-localization
+packages/kbn-transpose-utils @elastic/kibana-visualizations
 x-pack/examples/triggers_actions_ui_example @elastic/response-ops
 x-pack/plugins/triggers_actions_ui @elastic/response-ops
 packages/kbn-triggers-actions-ui-types @elastic/response-ops

package.json

@@ -948,6 +948,7 @@
     "@kbn/tinymath": "link:packages/kbn-tinymath",
     "@kbn/transform-plugin": "link:x-pack/plugins/transform",
     "@kbn/translations-plugin": "link:x-pack/plugins/translations",
+    "@kbn/transpose-utils": "link:packages/kbn-transpose-utils",
     "@kbn/triggers-actions-ui-example-plugin": "link:x-pack/examples/triggers_actions_ui_example",
     "@kbn/triggers-actions-ui-plugin": "link:x-pack/plugins/triggers_actions_ui",
     "@kbn/triggers-actions-ui-types": "link:packages/kbn-triggers-actions-ui-types",

packages/kbn-transpose-utils/README.md

@@ -0,0 +1,3 @@
+# @kbn/transpose-utils
+
+Utility functions used to identify and convert transposed column ids.

packages/kbn-transpose-utils/index.test.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { getOriginalId, getTransposeId, isTransposeId } from '.';
+
+describe('transpose utils', () => {
+  it('should covert value and id to transposed id', () => {
+    expect(getTransposeId('test', 'column-1')).toBe('test---column-1');
+  });
+
+  it('should know if id is transposed', () => {
+    const testId = getTransposeId('test', 'column-1');
+    expect(isTransposeId(testId)).toBe(true);
+  });
+
+  it('should know if id is not transposed', () => {
+    expect(isTransposeId('test')).toBe(false);
+  });
+
+  it('should return id for transposed id', () => {
+    const testId = getTransposeId('test', 'column-1');
+
+    expect(getOriginalId(testId)).toBe('column-1');
+  });
+
+  it('should return id for non-transposed id', () => {
+    expect(getOriginalId('test')).toBe('test');
+  });
+});

packages/kbn-transpose-utils/index.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+/**
+ * Used to delimitate felids of a transposed column id
+ */
+export const TRANSPOSE_SEPARATOR = '---';
+
+/**
+ * Visual deliminator between felids of a transposed column id
+ *
+ * Meant to align with the `MULTI_FIELD_KEY_SEPARATOR` from the data plugin
+ */
+export const TRANSPOSE_VISUAL_SEPARATOR = '›';
+
+export function getTransposeId(value: string, columnId: string) {
+  return `${value}${TRANSPOSE_SEPARATOR}${columnId}`;
+}
+
+export function isTransposeId(id: string): boolean {
+  return id.split(TRANSPOSE_SEPARATOR).length > 1;
+}
+
+export function getOriginalId(id: string) {
+  if (id.includes(TRANSPOSE_SEPARATOR)) {
+    const idParts = id.split(TRANSPOSE_SEPARATOR);
+    return idParts[idParts.length - 1];
+  }
+  return id;
+}

packages/kbn-transpose-utils/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-transpose-utils'],
+};

packages/kbn-transpose-utils/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/transpose-utils",
+  "owner": "@elastic/kibana-visualizations"
+}

packages/kbn-transpose-utils/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/transpose-utils",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
+}

packages/kbn-transpose-utils/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+      "react"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.tsx",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": []
+}

src/plugins/chart_expressions/expression_gauge/common/types/expression_functions.ts

@@ -11,6 +11,8 @@ import { $Values } from '@kbn/utility-types';
 import type { PaletteOutput, CustomPaletteParams } from '@kbn/coloring';
 import {
   Datatable,
+  DefaultInspectorAdapters,
+  ExecutionContext,
   ExpressionFunctionDefinition,
   ExpressionValueRender,
 } from '@kbn/expressions-plugin/common';
@@ -86,7 +88,8 @@ export type GaugeExpressionFunctionDefinition = ExpressionFunctionDefinition<
   typeof EXPRESSION_GAUGE_NAME,
   GaugeInput,
   GaugeArguments,
-  ExpressionValueRender<GaugeExpressionProps>
+  ExpressionValueRender<GaugeExpressionProps>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;
 
 export interface Accessors {

src/plugins/chart_expressions/expression_heatmap/common/types/expression_functions.ts

@@ -11,6 +11,8 @@ import { Position } from '@elastic/charts';
 import type { PaletteOutput } from '@kbn/coloring';
 import {
   Datatable,
+  DefaultInspectorAdapters,
+  ExecutionContext,
   ExpressionFunctionDefinition,
   ExpressionValueRender,
 } from '@kbn/expressions-plugin/common';
@@ -114,7 +116,8 @@ export type HeatmapExpressionFunctionDefinition = ExpressionFunctionDefinition<
   typeof EXPRESSION_HEATMAP_NAME,
   HeatmapInput,
   HeatmapArguments,
-  ExpressionValueRender<HeatmapExpressionProps>
+  ExpressionValueRender<HeatmapExpressionProps>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;
 
 export type HeatmapLegendExpressionFunctionDefinition = ExpressionFunctionDefinition<

src/plugins/chart_expressions/expression_heatmap/public/components/helpers.ts

@@ -18,6 +18,7 @@ import {
   DEFAULT_MAX_STOP,
   DEFAULT_MIN_STOP,
 } from '@kbn/coloring';
+import { getOriginalId } from '@kbn/transpose-utils';
 
 import type { Datatable, DatatableColumn } from '@kbn/expressions-plugin/public';
 import { FormatFactory, IFieldFormat } from '@kbn/field-formats-plugin/common';
@@ -83,10 +84,6 @@ export function applyPaletteParams<T extends PaletteOutput<CustomPaletteParams>>
   return displayStops;
 }
 
-function getId(id: string) {
-  return id;
-}
-
 export function getNumericValue(rowValue: number | number[] | undefined) {
   if (rowValue == null || Array.isArray(rowValue)) {
     return;
@@ -94,11 +91,7 @@ export function getNumericValue(rowValue: number | number[] | undefined) {
   return rowValue;
 }
 
-export const findMinMaxByColumnId = (
-  columnIds: string[],
-  table: Datatable | undefined,
-  getOriginalId: (id: string) => string = getId
-) => {
+export const findMinMaxByColumnId = (columnIds: string[], table: Datatable | undefined) => {
   const minMax: Record<string, { min: number; max: number; fallback?: boolean }> = {};
 
   if (table != null) {

src/plugins/chart_expressions/expression_heatmap/tsconfig.json

@@ -28,6 +28,7 @@
     "@kbn/chart-expressions-common",
     "@kbn/visualization-utils",
     "@kbn/react-kibana-context-render",
+    "@kbn/transpose-utils",
   ],
   "exclude": [
     "target/**/*",

src/plugins/chart_expressions/expression_legacy_metric/common/types/expression_functions.ts

@@ -10,6 +10,8 @@
 import type { PaletteOutput } from '@kbn/coloring';
 import {
   Datatable,
+  DefaultInspectorAdapters,
+  ExecutionContext,
   ExpressionFunctionDefinition,
   ExpressionValueRender,
   Style,
@@ -47,5 +49,6 @@ export type MetricVisExpressionFunctionDefinition = ExpressionFunctionDefinition
   typeof EXPRESSION_METRIC_NAME,
   MetricInput,
   MetricArguments,
-  ExpressionValueRender<MetricVisRenderConfig>
+  ExpressionValueRender<MetricVisRenderConfig>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;

src/plugins/chart_expressions/expression_metric/common/types/expression_functions.ts

@@ -12,6 +12,8 @@ import { LayoutDirection, MetricStyle, MetricWTrend } from '@elastic/charts';
 import { $Values } from '@kbn/utility-types';
 import {
   Datatable,
+  DefaultInspectorAdapters,
+  ExecutionContext,
   ExpressionFunctionDefinition,
   ExpressionValueRender,
 } from '@kbn/expressions-plugin/common';
@@ -64,7 +66,8 @@ export type MetricVisExpressionFunctionDefinition = ExpressionFunctionDefinition
   typeof EXPRESSION_METRIC_NAME,
   MetricInput,
   MetricArguments,
-  ExpressionValueRender<MetricVisRenderConfig>
+  ExpressionValueRender<MetricVisRenderConfig>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;
 
 export interface TrendlineArguments {

src/plugins/chart_expressions/expression_partition_vis/common/types/expression_functions.ts

@@ -14,6 +14,8 @@ import {
   Datatable,
   ExpressionValueRender,
   ExpressionValueBoxed,
+  DefaultInspectorAdapters,
+  ExecutionContext,
 } from '@kbn/expressions-plugin/common';
 import {
   PARTITION_LABELS_VALUE,
@@ -66,28 +68,32 @@ export type PieVisExpressionFunctionDefinition = ExpressionFunctionDefinition<
   typeof PIE_VIS_EXPRESSION_NAME,
   Datatable,
   PieVisConfig,
-  ExpressionValueRender<PartitionChartProps>
+  ExpressionValueRender<PartitionChartProps>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;
 
 export type TreemapVisExpressionFunctionDefinition = ExpressionFunctionDefinition<
   typeof TREEMAP_VIS_EXPRESSION_NAME,
   Datatable,
   TreemapVisConfig,
-  ExpressionValueRender<PartitionChartProps>
+  ExpressionValueRender<PartitionChartProps>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;
 
 export type MosaicVisExpressionFunctionDefinition = ExpressionFunctionDefinition<
   typeof MOSAIC_VIS_EXPRESSION_NAME,
   Datatable,
   MosaicVisConfig,
-  ExpressionValueRender<PartitionChartProps>
+  ExpressionValueRender<PartitionChartProps>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;
 
 export type WaffleVisExpressionFunctionDefinition = ExpressionFunctionDefinition<
   typeof WAFFLE_VIS_EXPRESSION_NAME,
   Datatable,
   WaffleVisConfig,
-  ExpressionValueRender<PartitionChartProps>
+  ExpressionValueRender<PartitionChartProps>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;
 
 export enum ChartTypes {

src/plugins/chart_expressions/expression_xy/common/expression_functions/xy_vis.test.ts

@@ -11,8 +11,12 @@ import { xyVisFunction } from '.';
 import { createMockExecutionContext } from '@kbn/expressions-plugin/common/mocks';
 import { sampleArgs, sampleLayer } from '../__mocks__';
 import { XY_VIS } from '../constants';
+import { createDefaultInspectorAdapters } from '@kbn/expressions-plugin/common';
 
 describe('xyVis', () => {
+  const getExecutionContext = () =>
+    createMockExecutionContext({}, createDefaultInspectorAdapters());
+
   test('it renders with the specified data and args', async () => {
     const { data, args } = sampleArgs();
     const { layers, ...rest } = args;
@@ -20,7 +24,7 @@ describe('xyVis', () => {
     const result = await xyVisFunction.fn(
       data,
       { ...rest, ...restLayerArgs, referenceLines: [] },
-      createMockExecutionContext()
+      getExecutionContext()
     );
 
     expect(result).toEqual({
@@ -59,7 +63,7 @@ describe('xyVis', () => {
           markSizeRatio: 0,
           referenceLines: [],
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
 
@@ -72,7 +76,7 @@ describe('xyVis', () => {
           markSizeRatio: 101,
           referenceLines: [],
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -90,7 +94,7 @@ describe('xyVis', () => {
           minTimeBarInterval: '1q',
           referenceLines: [],
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -108,7 +112,7 @@ describe('xyVis', () => {
           minTimeBarInterval: '1h',
           referenceLines: [],
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -126,7 +130,7 @@ describe('xyVis', () => {
           addTimeMarker: true,
           referenceLines: [],
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -147,7 +151,7 @@ describe('xyVis', () => {
 
           splitRowAccessor,
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -168,7 +172,7 @@ describe('xyVis', () => {
 
           splitColumnAccessor,
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -188,7 +192,7 @@ describe('xyVis', () => {
 
           markSizeRatio: 5,
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -211,7 +215,7 @@ describe('xyVis', () => {
           seriesType: 'bar',
           showLines: true,
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -238,7 +242,7 @@ describe('xyVis', () => {
             extent: { type: 'axisExtentConfig', mode: 'dataBounds' },
           },
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -268,7 +272,7 @@ describe('xyVis', () => {
             },
           },
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -293,7 +297,7 @@ describe('xyVis', () => {
             extent: { type: 'axisExtentConfig', mode: 'dataBounds' },
           },
         },
-        createMockExecutionContext()
+        getExecutionContext()
       )
     ).rejects.toThrowErrorMatchingSnapshot();
   });
@@ -320,7 +324,7 @@ describe('xyVis', () => {
           },
         },
       },
-      createMockExecutionContext()
+      getExecutionContext()
     );
 
     expect(result).toEqual({
@@ -370,7 +374,7 @@ describe('xyVis', () => {
       },
     };
     const context = {
-      ...createMockExecutionContext(),
+      ...getExecutionContext(),
       variables: {
         overrides,
       },

src/plugins/chart_expressions/expression_xy/common/types/expression_functions.ts

@@ -13,6 +13,8 @@ import type { PaletteOutput } from '@kbn/coloring';
 import type {
   Datatable,
   DatatableColumnMeta,
+  DefaultInspectorAdapters,
+  ExecutionContext,
   ExpressionFunctionDefinition,
 } from '@kbn/expressions-plugin/common';
 import {
@@ -449,13 +451,15 @@ export type XyVisFn = ExpressionFunctionDefinition<
   typeof XY_VIS,
   Datatable,
   XYArgs,
-  Promise<XYRender>
+  Promise<XYRender>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;
 export type LayeredXyVisFn = ExpressionFunctionDefinition<
   typeof LAYERED_XY_VIS,
   Datatable,
   LayeredXYArgs,
-  Promise<XYRender>
+  Promise<XYRender>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;
 
 export type ExtendedDataLayerFn = ExpressionFunctionDefinition<

src/plugins/chart_expressions/expression_xy/common/utils/log_datatables.ts

@@ -8,7 +8,11 @@
  */
 
 import { QueryPointEventAnnotationOutput } from '@kbn/event-annotation-plugin/common';
-import { Datatable, ExecutionContext } from '@kbn/expressions-plugin/common';
+import {
+  Datatable,
+  DefaultInspectorAdapters,
+  ExecutionContext,
+} from '@kbn/expressions-plugin/common';
 import { ExpressionValueVisDimension } from '@kbn/visualizations-plugin/common';
 import { Dimension, prepareLogTable } from '@kbn/visualizations-plugin/common/utils';
 import { LayerTypes, REFERENCE_LINE } from '../constants';
@@ -23,7 +27,7 @@ import {
 
 export const logDatatables = (
   layers: CommonXYLayerConfig[],
-  handlers: ExecutionContext,
+  handlers: ExecutionContext<DefaultInspectorAdapters>,
   splitColumnAccessor?: string | ExpressionValueVisDimension,
   splitRowAccessor?: string | ExpressionValueVisDimension,
   annotations?: ExpressionAnnotationResult
@@ -88,7 +92,7 @@ const getLogAnnotationTable = (data: Datatable, layer: AnnotationLayerConfigResu
 export const logDatatable = (
   data: Datatable,
   layers: CommonXYLayerConfig[],
-  handlers: ExecutionContext,
+  handlers: ExecutionContext<DefaultInspectorAdapters>,
   splitColumnAccessor?: string | ExpressionValueVisDimension,
   splitRowAccessor?: string | ExpressionValueVisDimension
 ) => {

src/plugins/data/common/exports/export_csv.test.ts

@@ -97,14 +97,4 @@ describe('CSV exporter', () => {
       })
     ).toMatch('columnOne\r\n"a,b"\r\n');
   });
-
-  test('should respect the sorted columns order when passed', () => {
-    const datatable = getDataTable({ multipleColumns: true });
-    expect(
-      datatableToCSV(datatable, {
-        ...getDefaultOptions(),
-        columnsSorting: ['col2', 'col1'],
-      })
-    ).toMatch('columnTwo,columnOne\r\n"Formatted_5","Formatted_value"\r\n');
-  });
 });

src/plugins/data/common/exports/export_csv.tsx

@@ -7,8 +7,6 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-// Inspired by the inspector CSV exporter
-
 import { Datatable } from '@kbn/expressions-plugin/common';
 import { FormatFactory } from '@kbn/field-formats-plugin/common';
 import { createEscapeValue } from './escape_value';
@@ -22,12 +20,11 @@ interface CSVOptions {
   escapeFormulaValues: boolean;
   formatFactory: FormatFactory;
   raw?: boolean;
-  columnsSorting?: string[];
 }
 
 export function datatableToCSV(
   { columns, rows }: Datatable,
-  { csvSeparator, quoteValues, formatFactory, raw, escapeFormulaValues, columnsSorting }: CSVOptions
+  { csvSeparator, quoteValues, formatFactory, raw, escapeFormulaValues }: CSVOptions
 ) {
   const escapeValues = createEscapeValue({
     separator: csvSeparator,
@@ -35,26 +32,15 @@ export function datatableToCSV(
     escapeFormulaValues,
   });
 
-  const sortedIds = columnsSorting || columns.map((col) => col.id);
-
-  // Build an index lookup table
-  const columnIndexLookup = sortedIds.reduce((memo, id, index) => {
-    memo[id] = index;
-    return memo;
-  }, {} as Record<string, number>);
-
-  // Build the header row by its names
   const header: string[] = [];
   const sortedColumnIds: string[] = [];
   const formatters: Record<string, ReturnType<FormatFactory>> = {};
 
-  for (const column of columns) {
-    const columnIndex = columnIndexLookup[column.id];
-
-    header[columnIndex] = escapeValues(column.name);
-    sortedColumnIds[columnIndex] = column.id;
+  columns.forEach((column, i) => {
+    header[i] = escapeValues(column.name);
+    sortedColumnIds[i] = column.id;
     formatters[column.id] = formatFactory(column.meta?.params);
-  }
+  });
 
   if (header.length === 0) {
     return '';
@@ -69,6 +55,6 @@ export function datatableToCSV(
 
   return (
     [header, ...csvRows].map((row) => row.join(csvSeparator)).join(LINE_FEED_CHARACTER) +
-    LINE_FEED_CHARACTER
-  ); // Add \r\n after last line
+    LINE_FEED_CHARACTER // Add \r\n after last line
+  );
 }

src/plugins/data/public/utils/table_inspector_view/components/__snapshots__/data_view.test.tsx.snap

@@ -156,7 +156,8 @@ exports[`Inspector Data View component should render loading state 1`] = `
           "_events": Object {},
           "_eventsCount": 0,
           "_maxListeners": undefined,
-          "_tables": Object {},
+          "allowCsvExport": false,
+          "initialSelectedTable": undefined,
           Symbol(shapeMode): false,
           Symbol(kCapture): false,
         },
@@ -430,29 +431,31 @@ Array [
         <div
           class="euiFlexItem emotion-euiFlexItem-grow-1"
         >
-          <div
-            class="euiPopover emotion-euiPopover-inline-block"
-            id="inspectorTableChooser"
-          >
-            <button
-              class="euiButtonEmpty emotion-euiButtonDisplay-euiButtonEmpty-s-empty-primary"
-              data-test-subj="inspectorTableChooser"
-              type="button"
+          <div>
+            <div
+              class="euiPopover emotion-euiPopover-inline-block"
+              id="inspectorTableChooser"
             >
-              <span
-                class="euiButtonEmpty__content emotion-euiButtonDisplayContent"
+              <button
+                class="euiButtonEmpty emotion-euiButtonDisplay-euiButtonEmpty-s-empty-primary"
+                data-test-subj="inspectorTableChooser"
+                type="button"
               >
                 <span
-                  class="eui-textTruncate euiButtonEmpty__text"
+                  class="euiButtonEmpty__content emotion-euiButtonDisplayContent"
                 >
-                  Table 1
+                  <span
+                    class="eui-textTruncate euiButtonEmpty__text"
+                  >
+                    Table 1
+                  </span>
+                  <span
+                    color="inherit"
+                    data-euiicon-type="arrowDown"
+                  />
                 </span>
-                <span
-                  color="inherit"
-                  data-euiicon-type="arrowDown"
-                />
-              </span>
-            </button>
+              </button>
+            </div>
           </div>
         </div>
       </div>

src/plugins/data/public/utils/table_inspector_view/components/data_table_selector.tsx

@@ -9,7 +9,6 @@
 
 import React, { Component } from 'react';
 import { FormattedMessage } from '@kbn/i18n-react';
-import PropTypes from 'prop-types';
 import {
   EuiButtonEmpty,
   EuiContextMenuPanel,
@@ -27,16 +26,10 @@ interface TableSelectorState {
 interface TableSelectorProps {
   tables: Datatable[];
   selectedTable: Datatable;
-  onTableChanged: Function;
+  onTableChanged: (table: Datatable) => void;
 }
 
 export class TableSelector extends Component<TableSelectorProps, TableSelectorState> {
-  static propTypes = {
-    tables: PropTypes.array.isRequired,
-    selectedTable: PropTypes.object.isRequired,
-    onTableChanged: PropTypes.func,
-  };
-
   state = {
     isPopoverOpen: false,
   };
@@ -85,35 +78,37 @@ export class TableSelector extends Component<TableSelectorProps, TableSelectorSt
             />
           </strong>
         </EuiFlexItem>
-        <EuiFlexItem grow={true}>
-          <EuiPopover
-            id="inspectorTableChooser"
-            button={
-              <EuiButtonEmpty
-                iconType="arrowDown"
-                iconSide="right"
-                size="s"
-                onClick={this.togglePopover}
-                data-test-subj="inspectorTableChooser"
-              >
-                <FormattedMessage
-                  id="data.inspector.table.inspectorTableChooserButton"
-                  defaultMessage="Table {index}"
-                  values={{ index: currentIndex + 1 }}
-                />
-              </EuiButtonEmpty>
-            }
-            isOpen={this.state.isPopoverOpen}
-            closePopover={this.closePopover}
-            panelPaddingSize="none"
-            anchorPosition="downLeft"
-            repositionOnScroll
-          >
-            <EuiContextMenuPanel
-              items={this.props.tables.map(this.renderTableDropdownItem)}
-              data-test-subj="inspectorTableChooserMenuPanel"
-            />
-          </EuiPopover>
+        <EuiFlexItem>
+          <div>
+            <EuiPopover
+              id="inspectorTableChooser"
+              button={
+                <EuiButtonEmpty
+                  iconType="arrowDown"
+                  iconSide="right"
+                  size="s"
+                  onClick={this.togglePopover}
+                  data-test-subj="inspectorTableChooser"
+                >
+                  <FormattedMessage
+                    id="data.inspector.table.inspectorTableChooserButton"
+                    defaultMessage="Table {index}"
+                    values={{ index: currentIndex + 1 }}
+                  />
+                </EuiButtonEmpty>
+              }
+              isOpen={this.state.isPopoverOpen}
+              closePopover={this.closePopover}
+              panelPaddingSize="none"
+              anchorPosition="downLeft"
+              repositionOnScroll
+            >
+              <EuiContextMenuPanel
+                items={this.props.tables.map(this.renderTableDropdownItem)}
+                data-test-subj="inspectorTableChooserMenuPanel"
+              />
+            </EuiPopover>
+          </div>
         </EuiFlexItem>
       </EuiFlexGroup>
     );

src/plugins/data/public/utils/table_inspector_view/components/data_view.tsx

@@ -8,7 +8,6 @@
  */
 
 import React, { Component } from 'react';
-import PropTypes from 'prop-types';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { EuiEmptyPrompt, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiText } from '@elastic/eui';
 
@@ -35,15 +34,6 @@ interface DataViewComponentProps extends InspectorViewProps {
 }
 
 class DataViewComponent extends Component<DataViewComponentProps, DataViewComponentState> {
-  static propTypes = {
-    adapters: PropTypes.object.isRequired,
-    title: PropTypes.string.isRequired,
-    uiSettings: PropTypes.object,
-    uiActions: PropTypes.object.isRequired,
-    fieldFormats: PropTypes.object.isRequired,
-    isFilterable: PropTypes.func.isRequired,
-  };
-
   state = {} as DataViewComponentState;
 
   static getDerivedStateFromProps(
@@ -54,9 +44,10 @@ class DataViewComponent extends Component<DataViewComponentProps, DataViewCompon
       return null;
     }
 
-    const { tables } = nextProps.adapters.tables;
+    const { tables, initialSelectedTable } = nextProps.adapters.tables ?? {};
     const keys = Object.keys(tables);
-    const datatable = keys.length ? tables[keys[0]] : undefined;
+    const intialTableKey = keys.includes(initialSelectedTable) ? initialSelectedTable : keys[0];
+    const datatable = keys.length ? tables[intialTableKey] : undefined;
 
     return {
       adapters: nextProps.adapters,

src/plugins/expressions/common/mocks.ts

@@ -7,11 +7,16 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { Adapters } from '@kbn/inspector-plugin/common';
 import { ExecutionContext } from './execution/types';
 
-export const createMockExecutionContext = <ExtraContext extends object = object>(
-  extraContext: ExtraContext = {} as ExtraContext
-): ExecutionContext & ExtraContext => {
+export const createMockExecutionContext = <
+  ExtraContext extends object = object,
+  ExtraAdapters extends Adapters = Adapters
+>(
+  extraContext: ExtraContext = {} as ExtraContext,
+  extraAdapters: ExtraAdapters = {} as ExtraAdapters
+): ExecutionContext<ExtraAdapters> & ExtraContext => {
   const executionContext = {
     getSearchContext: jest.fn(),
     getSearchSessionId: jest.fn(),
@@ -28,9 +33,10 @@ export const createMockExecutionContext = <ExtraContext extends object = object>
     inspectorAdapters: {
       requests: {},
       data: {},
+      ...extraAdapters,
     },
     allowCache: false,
-  } as unknown as ExecutionContext;
+  } as unknown as ExecutionContext<ExtraAdapters>;
 
   return {
     ...executionContext,

src/plugins/expressions/common/util/tables_adapter.ts

@@ -11,19 +11,23 @@ import { EventEmitter } from 'events';
 import type { Datatable } from '../expression_types/specs';
 
 export class TablesAdapter extends EventEmitter {
-  private _tables: { [key: string]: Datatable } = {};
+  #tables: { [key: string]: Datatable } = {};
 
-  public logDatatable(name: string, datatable: Datatable): void {
-    this._tables[name] = datatable;
+  public allowCsvExport: boolean = false;
+  /** Key of table to set as initial selection */
+  public initialSelectedTable?: string;
+
+  public logDatatable(key: string, datatable: Datatable): void {
+    this.#tables[key] = datatable;
     this.emit('change', this.tables);
   }
 
   public reset() {
-    this._tables = {};
+    this.#tables = {};
     this.emit('change', this.tables);
   }
 
   public get tables() {
-    return this._tables;
+    return this.#tables;
   }
 }

tsconfig.base.json

@@ -1888,6 +1888,8 @@
       "@kbn/transform-plugin/*": ["x-pack/plugins/transform/*"],
       "@kbn/translations-plugin": ["x-pack/plugins/translations"],
       "@kbn/translations-plugin/*": ["x-pack/plugins/translations/*"],
+      "@kbn/transpose-utils": ["packages/kbn-transpose-utils"],
+      "@kbn/transpose-utils/*": ["packages/kbn-transpose-utils/*"],
       "@kbn/triggers-actions-ui-example-plugin": ["x-pack/examples/triggers_actions_ui_example"],
       "@kbn/triggers-actions-ui-example-plugin/*": ["x-pack/examples/triggers_actions_ui_example/*"],
       "@kbn/triggers-actions-ui-plugin": ["x-pack/plugins/triggers_actions_ui"],

x-pack/plugins/lens/common/expressions/datatable/datatable_fn.ts

@@ -10,9 +10,17 @@ import { i18n } from '@kbn/i18n';
 import { prepareLogTable } from '@kbn/visualizations-plugin/common/utils';
 import type { Datatable, ExecutionContext } from '@kbn/expressions-plugin/common';
 import { FormatFactory } from '../../types';
-import { transposeTable } from './transpose_helpers';
 import { computeSummaryRowForColumn } from './summary';
 import type { DatatableExpressionFunction } from './types';
+import { transposeTable } from './transpose_helpers';
+
+/**
+ * Available datatables logged to inspector
+ */
+export const DatatableInspectorTables = {
+  Default: 'default',
+  Transpose: 'transpose',
+};
 
 export const datatableFn =
   (
@@ -36,7 +44,7 @@ export const datatableFn =
         true
       );
 
-      context.inspectorAdapters.tables.logDatatable('default', logTable);
+      context.inspectorAdapters.tables.logDatatable(DatatableInspectorTables.Default, logTable);
     }
 
     let untransposedData: Datatable | undefined;
@@ -52,8 +60,29 @@ export const datatableFn =
     if (hasTransposedColumns) {
       // store original shape of data separately
       untransposedData = cloneDeep(table);
-      // transposes table and args inplace
+      // transposes table and args in-place
       transposeTable(args, table, formatters);
+
+      if (context?.inspectorAdapters?.tables) {
+        const logTransposedTable = prepareLogTable(
+          table,
+          [
+            [
+              args.columns.map((column) => column.columnId),
+              i18n.translate('xpack.lens.datatable.column.help', {
+                defaultMessage: 'Datatable column',
+              }),
+            ],
+          ],
+          true
+        );
+
+        context.inspectorAdapters.tables.logDatatable(
+          DatatableInspectorTables.Transpose,
+          logTransposedTable
+        );
+        context.inspectorAdapters.tables.initialSelectedTable = DatatableInspectorTables.Transpose;
+      }
     }
 
     const columnsWithSummary = args.columns.filter((c) => c.summaryRow);

x-pack/plugins/lens/common/expressions/datatable/index.ts

@@ -7,6 +7,5 @@
 
 export * from './datatable_column';
 export * from './datatable';
-export { isTransposeId, getOriginalId } from './transpose_helpers';
 
 export type { DatatableProps, DatatableExpressionFunction } from './types';

x-pack/plugins/lens/common/expressions/datatable/summary.ts

@@ -8,8 +8,8 @@
 import { i18n } from '@kbn/i18n';
 import type { FieldFormat } from '@kbn/field-formats-plugin/common';
 import type { Datatable } from '@kbn/expressions-plugin/common';
+import { getOriginalId } from '@kbn/transpose-utils';
 import { DatatableColumnArgs } from './datatable_column';
-import { getOriginalId } from './transpose_helpers';
 import { isNumericFieldForDatatable } from './utils';
 
 type SummaryRowType = Extract<DatatableColumnArgs['summaryRow'], string>;

x-pack/plugins/lens/common/expressions/datatable/transpose_helpers.test.ts

@@ -7,11 +7,10 @@
 
 import type { FieldFormat } from '@kbn/field-formats-plugin/common';
 import type { Datatable } from '@kbn/expressions-plugin/common';
-import { DatatableArgs } from './datatable';
-
+import { DatatableArgs } from '..';
 import { transposeTable } from './transpose_helpers';
 
-describe('transpose_helpers', () => {
+describe('transpose helpers', () => {
   function buildTable(): Datatable {
     // 3 buckets, 2 metrics
     // first bucket goes A/B/C
@@ -120,10 +119,10 @@ describe('transpose_helpers', () => {
       'bucket2',
       'bucket3',
       'A---metric1',
-      'B---metric1',
-      'C---metric1',
       'A---metric2',
+      'B---metric1',
       'B---metric2',
+      'C---metric1',
       'C---metric2',
     ]);
 
@@ -179,22 +178,22 @@ describe('transpose_helpers', () => {
     expect(table.columns.map((c) => c.id)).toEqual([
       'bucket3',
       'A---D---metric1',
-      'B---D---metric1',
-      'C---D---metric1',
-      'A---E---metric1',
-      'B---E---metric1',
-      'C---E---metric1',
-      'A---F---metric1',
-      'B---F---metric1',
-      'C---F---metric1',
       'A---D---metric2',
-      'B---D---metric2',
-      'C---D---metric2',
+      'A---E---metric1',
       'A---E---metric2',
-      'B---E---metric2',
-      'C---E---metric2',
+      'A---F---metric1',
       'A---F---metric2',
+      'B---D---metric1',
+      'B---D---metric2',
+      'B---E---metric1',
+      'B---E---metric2',
+      'B---F---metric1',
       'B---F---metric2',
+      'C---D---metric1',
+      'C---D---metric2',
+      'C---E---metric1',
+      'C---E---metric2',
+      'C---F---metric1',
       'C---F---metric2',
     ]);
 

x-pack/plugins/lens/common/expressions/datatable/transpose_helpers.ts

@@ -7,43 +7,20 @@
 
 import type { Datatable, DatatableColumn, DatatableRow } from '@kbn/expressions-plugin/common';
 import type { FieldFormat } from '@kbn/field-formats-plugin/common';
-import type { DatatableArgs } from './datatable';
+import { TRANSPOSE_VISUAL_SEPARATOR, getTransposeId } from '@kbn/transpose-utils';
+import { DatatableArgs } from './datatable';
 import type { DatatableColumnConfig, DatatableColumnArgs } from './datatable_column';
 
-const TRANSPOSE_SEPARATOR = '---';
-
-const TRANSPOSE_VISUAL_SEPARATOR = '›';
-
-export function getTransposeId(value: string, columnId: string) {
-  return `${value}${TRANSPOSE_SEPARATOR}${columnId}`;
-}
-
-export function isTransposeId(id: string): boolean {
-  return id.split(TRANSPOSE_SEPARATOR).length > 1;
-}
-
-export function getOriginalId(id: string) {
-  if (id.includes(TRANSPOSE_SEPARATOR)) {
-    const idParts = id.split(TRANSPOSE_SEPARATOR);
-    return idParts[idParts.length - 1];
-  }
-  return id;
-}
-
 /**
  * Transposes the columns of the given table as defined in the arguments.
  * This function modifies the passed in args and firstTable objects.
  * This process consists out of three parts:
+ *
  * * Calculating the new column arguments
  * * Calculating the new datatable columns
  * * Calculating the new rows
  *
- * If the table is tranposed by multiple columns, this process is repeated on top of the previous transformation.
- *
- * @internal
- * @param args Arguments for the table visualization
- * @param firstTable datatable object containing the actual data
- * @param formatters Formatters for all columns to transpose columns by actual display values
+ * If the table is transposed by multiple columns, this process is repeated on top of the previous transformation.
  */
 export function transposeTable(
   args: DatatableArgs,
@@ -52,8 +29,7 @@ export function transposeTable(
 ) {
   args.columns
     .filter((columnArgs) => columnArgs.isTransposed)
-    // start with the inner nested transposed column and work up to preserve column grouping
-    .reverse()
+    .reverse() // start with the inner nested transposed column and work up to preserve column grouping
     .forEach(({ columnId: transposedColumnId }) => {
       const datatableColumnIndex = firstTable.columns.findIndex((c) => c.id === transposedColumnId);
       const datatableColumn = firstTable.columns[datatableColumnIndex];
@@ -86,6 +62,11 @@ export function transposeTable(
         transposedColumnId,
         metricsColumnArgs
       );
+
+      const colOrderMap = new Map(args.columns.map((c, i) => [c.columnId, i]));
+      firstTable.columns.sort((a, b) => {
+        return (colOrderMap.get(a.id) ?? 0) - (colOrderMap.get(b.id) ?? 0);
+      });
     });
 }
 
@@ -131,9 +112,6 @@ function updateColumnArgs(
 
 /**
  * Finds all unique values in a column in order of first occurence
- * @param table Table to search through
- * @param formatter formatter for the column
- * @param columnId column
  */
 function getUniqueValues(table: Datatable, formatter: FieldFormat, columnId: string) {
   const values = new Map<string, unknown>();
@@ -149,9 +127,6 @@ function getUniqueValues(table: Datatable, formatter: FieldFormat, columnId: str
 /**
  * Calculate transposed column objects of the datatable object and puts them into the datatable.
  * Returns args for additional columns grouped by metric
- * @param metricColumns
- * @param firstTable
- * @param uniqueValues
  */
 function transposeColumns(
   args: DatatableArgs,

x-pack/plugins/lens/common/expressions/datatable/types.ts

@@ -5,7 +5,12 @@
  * 2.0.
  */
 
-import type { Datatable, ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common';
+import type {
+  Datatable,
+  DefaultInspectorAdapters,
+  ExecutionContext,
+  ExpressionFunctionDefinition,
+} from '@kbn/expressions-plugin/common';
 import type { DatatableArgs } from './datatable';
 
 export interface DatatableProps {
@@ -25,5 +30,6 @@ export type DatatableExpressionFunction = ExpressionFunctionDefinition<
   'lens_datatable',
   Datatable,
   DatatableArgs,
-  Promise<DatatableRender>
+  Promise<DatatableRender>,
+  ExecutionContext<DefaultInspectorAdapters>
 >;

x-pack/plugins/lens/common/expressions/datatable/utils.ts

@@ -6,7 +6,7 @@
  */
 
 import { type Datatable, type DatatableColumnMeta } from '@kbn/expressions-plugin/common';
-import { getOriginalId } from './transpose_helpers';
+import { getOriginalId } from '@kbn/transpose-utils';
 
 /**
  * Returns true for numerical fields

x-pack/plugins/lens/public/app_plugin/csv_download_provider/csv_download_provider.tsx

@@ -12,9 +12,15 @@ import { downloadMultipleAs } from '@kbn/share-plugin/public';
 import { exporters } from '@kbn/data-plugin/public';
 import { IUiSettingsClient } from '@kbn/core-ui-settings-browser';
 import { FormattedMessage } from '@kbn/i18n-react';
+import type { Datatable } from '@kbn/expressions-plugin/common';
 import { ShareMenuItemV2, ShareMenuProviderV2 } from '@kbn/share-plugin/public/types';
 import { FormatFactory } from '../../../common/types';
-import { TableInspectorAdapter } from '../../editor_frame_service/types';
+
+export interface CSVSharingData {
+  title: string;
+  datatables: Datatable[];
+  csvEnabled: boolean;
+}
 
 declare global {
   interface Window {
@@ -27,25 +33,21 @@ declare global {
 }
 
 async function downloadCSVs({
-  activeData,
   title,
+  datatables,
   formatFactory,
   uiSettings,
-  columnsSorting,
 }: {
-  title: string;
-  activeData: TableInspectorAdapter;
   formatFactory: FormatFactory;
   uiSettings: IUiSettingsClient;
-  columnsSorting?: string[];
-}) {
-  if (!activeData) {
+} & Pick<CSVSharingData, 'title' | 'datatables'>) {
+  if (datatables.length === 0) {
     if (window.ELASTIC_LENS_CSV_DOWNLOAD_DEBUG) {
       window.ELASTIC_LENS_CSV_CONTENT = undefined;
     }
     return;
   }
-  const datatables = Object.values(activeData);
+
   const content = datatables.reduce<Record<string, { content: string; type: string }>>(
     (memo, datatable, i) => {
       // skip empty datatables
@@ -58,7 +60,6 @@ async function downloadCSVs({
             quoteValues: uiSettings.get('csv:quoteValues', true),
             formatFactory,
             escapeFormulaValues: false,
-            columnsSorting,
           }),
           type: exporters.CSV_MIME_TYPE,
         };
@@ -67,33 +68,34 @@ async function downloadCSVs({
     },
     {}
   );
+
   if (window.ELASTIC_LENS_CSV_DOWNLOAD_DEBUG) {
     window.ELASTIC_LENS_CSV_CONTENT = content;
   }
+
   if (content) {
     downloadMultipleAs(content);
   }
 }
 
-function getWarnings(activeData: TableInspectorAdapter) {
+function getWarnings(datatables: Datatable[]) {
   const warnings: Array<{ title: string; message: string }> = [];
-  if (activeData) {
-    const datatables = Object.values(activeData);
-    const formulaDetected = datatables.some((datatable) => {
-      return tableHasFormulas(datatable.columns, datatable.rows);
+
+  const formulaDetected = datatables.some((datatable) => {
+    return tableHasFormulas(datatable.columns, datatable.rows);
+  });
+  if (formulaDetected) {
+    warnings.push({
+      title: i18n.translate('xpack.lens.app.downloadButtonFormulasWarningTitle', {
+        defaultMessage: 'Formulas detected',
+      }),
+      message: i18n.translate('xpack.lens.app.downloadButtonFormulasWarningMessage', {
+        defaultMessage:
+          'Your CSV contains characters that spreadsheet applications might interpret as formulas.',
+      }),
     });
-    if (formulaDetected) {
-      warnings.push({
-        title: i18n.translate('xpack.lens.app.downloadButtonFormulasWarningTitle', {
-          defaultMessage: 'Formulas detected',
-        }),
-        message: i18n.translate('xpack.lens.app.downloadButtonFormulasWarningMessage', {
-          defaultMessage:
-            'Your CSV contains characters that spreadsheet applications might interpret as formulas.',
-        }),
-      });
-    }
   }
+
   return warnings;
 }
 
@@ -116,12 +118,8 @@ export const downloadCsvShareProvider = ({
       return [];
     }
 
-    const { title, activeData, csvEnabled, columnsSorting } = sharingData as {
-      title: string;
-      activeData: TableInspectorAdapter;
-      csvEnabled: boolean;
-      columnsSorting?: string[];
-    };
+    // TODO fix sharingData types
+    const { title, datatables, csvEnabled } = sharingData as unknown as CSVSharingData;
 
     const panelTitle = i18n.translate(
       'xpack.lens.reporting.shareContextMenu.csvReportsButtonLabel',
@@ -134,9 +132,8 @@ export const downloadCsvShareProvider = ({
       downloadCSVs({
         title,
         formatFactory: formatFactoryFn(),
-        activeData,
+        datatables,
         uiSettings,
-        columnsSorting,
       });
 
     return [
@@ -150,7 +147,7 @@ export const downloadCsvShareProvider = ({
         label: 'CSV' as const,
         reportType: 'lens_csv' as const,
         generateExport: downloadCSVHandler,
-        warnings: getWarnings(activeData),
+        warnings: getWarnings(datatables),
         ...(atLeastGold()
           ? {
               disabled: !csvEnabled,

x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx

@@ -576,6 +576,8 @@ export const LensTopNavMenu = ({
               return;
             }
 
+            const activeVisualization = visualizationMap[visualization.activeId];
+
             const {
               shareableUrl,
               savedObjectURL,
@@ -598,12 +600,22 @@ export const LensTopNavMenu = ({
               isCurrentStateDirty
             );
 
-            const sharingData = {
-              activeData,
-              columnsSorting: visualizationMap[visualization.activeId].getSortedColumns?.(
+            const datasourceLayers = getDatasourceLayers(
+              datasourceStates,
+              datasourceMap,
+              dataViews.indexPatterns
+            );
+
+            const exportDatatables =
+              activeVisualization.getExportDatatables?.(
                 visualization.state,
-                getDatasourceLayers(datasourceStates, datasourceMap, dataViews.indexPatterns)
-              ),
+                datasourceLayers,
+                activeData
+              ) ?? [];
+            const datatables =
+              exportDatatables.length > 0 ? exportDatatables : Object.values(activeData ?? {});
+            const sharingData = {
+              datatables,
               csvEnabled,
               reportingDisabled: !csvEnabled,
               title: title || defaultLensTitle,
@@ -613,9 +625,8 @@ export const LensTopNavMenu = ({
               },
               layout: {
                 dimensions:
-                  visualizationMap[visualization.activeId].getReportingLayout?.(
-                    visualization.state
-                  ) ?? DEFAULT_LENS_LAYOUT_DIMENSIONS,
+                  activeVisualization.getReportingLayout?.(visualization.state) ??
+                  DEFAULT_LENS_LAYOUT_DIMENSIONS,
               },
             };
 

x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/lens_configuration_flyout.tsx

@@ -117,7 +117,7 @@ export function LensEditConfigurationFlyout({
   useEffect(() => {
     const s = output$?.subscribe(() => {
       const activeData: Record<string, Datatable> = {};
-      const adaptersTables = previousAdapters.current?.tables?.tables as Record<string, Datatable>;
+      const adaptersTables = previousAdapters.current?.tables?.tables;
       const [table] = Object.values(adaptersTables || {});
       if (table) {
         // there are cases where a query can return a big amount of columns

x-pack/plugins/lens/public/datasources/form_based/utils.tsx

@@ -25,6 +25,7 @@ import {
 
 import { estypes } from '@elastic/elasticsearch';
 import { isQueryValid } from '@kbn/visualization-ui-components';
+import { getOriginalId } from '@kbn/transpose-utils';
 import type { DateRange } from '../../../common/types';
 import type {
   FramePublicAPI,
@@ -60,7 +61,6 @@ import { hasField } from './pure_utils';
 import { mergeLayer } from './state_helpers';
 import { supportsRarityRanking } from './operations/definitions/terms';
 import { DEFAULT_MAX_DOC_COUNT } from './operations/definitions/terms/constants';
-import { getOriginalId } from '../../../common/expressions/datatable/transpose_helpers';
 import { ReducedSamplingSectionEntries } from './info_badges';
 import { IgnoredGlobalFiltersEntries } from '../../shared_components/ignore_global_filter';
 import {

x-pack/plugins/lens/public/shared_components/coloring/utils.ts

@@ -21,6 +21,7 @@ import {
   getColorsFromMapping,
   DEFAULT_FALLBACK_PALETTE,
 } from '@kbn/coloring';
+import { getOriginalId } from '@kbn/transpose-utils';
 import { Datatable, DatatableColumnType } from '@kbn/expressions-plugin/common';
 import { DataType } from '../../types';
 
@@ -90,11 +91,7 @@ export function applyPaletteParams<T extends PaletteOutput<CustomPaletteParams>>
   return displayStops;
 }
 
-export const findMinMaxByColumnId = (
-  columnIds: string[],
-  table: Datatable | undefined,
-  getOriginalId: (id: string) => string = (id: string) => id
-) => {
+export const findMinMaxByColumnId = (columnIds: string[], table: Datatable | undefined) => {
   const minMaxMap = new Map<string, DataBounds>();
 
   if (table != null) {

x-pack/plugins/lens/public/types.ts

@@ -64,6 +64,7 @@ import type { LensInspector } from './lens_inspector_service';
 import type { DataViewsState } from './state_management/types';
 import type { IndexPatternServiceAPI } from './data_views_service/service';
 import type { Document } from './persistence/saved_object_store';
+import { TableInspectorAdapter } from './editor_frame_service/types';
 
 export type StartServices = Pick<
   CoreStart,
@@ -1351,9 +1352,13 @@ export interface Visualization<T = unknown, P = T, ExtraAppendLayerArg = unknown
    */
   getReportingLayout?: (state: T) => { height: number; width: number };
   /**
-   * A visualization can share how columns are visually sorted
+   * Get all datatables to be exported as csv
    */
-  getSortedColumns?: (state: T, datasourceLayers?: DatasourceLayers) => string[];
+  getExportDatatables?: (
+    state: T,
+    datasourceLayers?: DatasourceLayers,
+    activeData?: TableInspectorAdapter
+  ) => Datatable[];
   /**
    * returns array of telemetry events for the visualization on save
    */

x-pack/plugins/lens/public/visualizations/datatable/components/cell_value.test.tsx

@@ -8,12 +8,12 @@
 import React from 'react';
 import { DataContext } from './table_basic';
 import { createGridCell } from './cell_value';
+import { getTransposeId } from '@kbn/transpose-utils';
 import type { FieldFormat } from '@kbn/field-formats-plugin/common';
 import { Datatable } from '@kbn/expressions-plugin/public';
 import { DatatableArgs } from '../../../../common/expressions';
 import { DataContextType } from './types';
 import { render, screen } from '@testing-library/react';
-import { getTransposeId } from '../../../../common/expressions/datatable/transpose_helpers';
 
 describe('datatable cell renderer', () => {
   const innerCellColorFnMock = jest.fn().mockReturnValue('blue');

x-pack/plugins/lens/public/visualizations/datatable/components/dimension_editor.tsx

@@ -11,6 +11,7 @@ import { EuiFormRow, EuiSwitch, EuiButtonGroup, htmlIdGenerator } from '@elastic
 import { PaletteRegistry, getFallbackDataBounds } from '@kbn/coloring';
 import { getColorCategories } from '@kbn/chart-expressions-common';
 import { useDebouncedValue } from '@kbn/visualization-utils';
+import { getOriginalId } from '@kbn/transpose-utils';
 import type { VisualizationDimensionEditorProps } from '../../../types';
 import type { DatatableVisualizationState } from '../visualization';
 
@@ -20,7 +21,6 @@ import {
   findMinMaxByColumnId,
   shouldColorByTerms,
 } from '../../../shared_components';
-import { getOriginalId } from '../../../../common/expressions/datatable/transpose_helpers';
 
 import './dimension_editor.scss';
 import { CollapseSetting } from '../../../shared_components/collapse_setting';
@@ -31,6 +31,7 @@ import {
   getFieldMetaFromDatatable,
   isNumericField,
 } from '../../../../common/expressions/datatable/utils';
+import { DatatableInspectorTables } from '../../../../common/expressions/datatable/datatable_fn';
 
 const idPrefix = htmlIdGenerator()();
 
@@ -78,7 +79,8 @@ export function TableDimensionEditor(props: TableDimensionEditorProps) {
   if (!column) return null;
   if (column.isTransposed) return null;
 
-  const currentData = frame.activeData?.[localState.layerId];
+  const currentData =
+    frame.activeData?.[localState.layerId] ?? frame.activeData?.[DatatableInspectorTables.Default];
   const datasource = frame.datasourceLayers?.[localState.layerId];
   const { isBucketed } = datasource?.getOperationForColumnId(accessor) ?? {};
   const meta = getFieldMetaFromDatatable(currentData, accessor);
@@ -94,7 +96,7 @@ export function TableDimensionEditor(props: TableDimensionEditorProps) {
     ? currentData?.columns.filter(({ id }) => getOriginalId(id) === accessor).map(({ id }) => id) ||
       []
     : [accessor];
-  const minMaxByColumnId = findMinMaxByColumnId(columnsToCheck, currentData, getOriginalId);
+  const minMaxByColumnId = findMinMaxByColumnId(columnsToCheck, currentData);
   const currentMinMax = minMaxByColumnId.get(accessor) ?? getFallbackDataBounds();
 
   const activePalette = column?.palette ?? {

x-pack/plugins/lens/public/visualizations/datatable/components/dimension_editor_addtional_section.tsx

@@ -22,6 +22,7 @@ import {
   getSummaryRowOptions,
 } from '../../../../common/expressions/datatable/summary';
 import { isNumericFieldForDatatable } from '../../../../common/expressions/datatable/utils';
+import { DatatableInspectorTables } from '../../../../common/expressions/datatable/datatable_fn';
 
 import './dimension_editor.scss';
 
@@ -73,7 +74,8 @@ export function TableDimensionEditorAdditionalSection(
   if (!column) return null;
   if (column.isTransposed) return null;
 
-  const currentData = frame.activeData?.[state.layerId];
+  const currentData =
+    frame.activeData?.[state.layerId] ?? frame.activeData?.[DatatableInspectorTables.Default];
 
   const isNumeric = isNumericFieldForDatatable(currentData, accessor);
   // when switching from one operation to another, make sure to keep the configuration consistent

x-pack/plugins/lens/public/visualizations/datatable/components/table_actions.ts

@@ -18,9 +18,9 @@ import type {
 import { ClickTriggerEvent } from '@kbn/charts-plugin/public';
 import { getSortingCriteria } from '@kbn/sort-predicates';
 import { i18n } from '@kbn/i18n';
+import { getOriginalId } from '@kbn/transpose-utils';
 import type { LensResizeAction, LensSortAction, LensToggleAction } from './types';
 import type { DatatableColumnConfig, LensGridDirection } from '../../../../common/expressions';
-import { getOriginalId } from '../../../../common/expressions/datatable/transpose_helpers';
 import type { FormatFactory } from '../../../../common/types';
 import { buildColumnsMetaLookup } from './helpers';
 
@@ -168,6 +168,10 @@ function isRange(meta: { params?: { id?: string } } | undefined) {
   return meta?.params?.id === 'range';
 }
 
+export function getSimpleColumnType(meta?: DatatableColumnMeta) {
+  return isRange(meta) ? 'range' : meta?.type;
+}
+
 function getColumnType({
   columnConfig,
   columnId,
@@ -185,7 +189,7 @@ function getColumnType({
   >;
 }) {
   const sortingHint = columnConfig.columns.find((col) => col.columnId === columnId)?.sortingHint;
-  return sortingHint ?? (isRange(lookup[columnId]?.meta) ? 'range' : lookup[columnId]?.meta?.type);
+  return sortingHint ?? getSimpleColumnType(lookup[columnId]?.meta);
 }
 
 export const buildSchemaDetectors = (

x-pack/plugins/lens/public/visualizations/datatable/components/table_basic.test.tsx

@@ -20,9 +20,9 @@ import type { DatatableProps } from '../../../../common/expressions';
 import { LENS_EDIT_PAGESIZE_ACTION } from './constants';
 import { DatatableRenderProps } from './types';
 import { PaletteOutput } from '@kbn/coloring';
+import { getTransposeId } from '@kbn/transpose-utils';
 import { CustomPaletteState } from '@kbn/charts-plugin/common';
 import { getCellColorFn } from '../../../shared_components/coloring/get_cell_color_fn';
-import { getTransposeId } from '../../../../common/expressions/datatable/transpose_helpers';
 
 jest.mock('../../../shared_components/coloring/get_cell_color_fn', () => {
   const mod = jest.requireActual('../../../shared_components/coloring/get_cell_color_fn');

x-pack/plugins/lens/public/visualizations/datatable/components/table_basic.tsx

@@ -32,10 +32,11 @@ import { ClickTriggerEvent } from '@kbn/charts-plugin/public';
 import { IconChartDatatable } from '@kbn/chart-icons';
 import useObservable from 'react-use/lib/useObservable';
 import { getColorCategories } from '@kbn/chart-expressions-common';
+import { getOriginalId, isTransposeId } from '@kbn/transpose-utils';
 import type { LensTableRowContextMenuEvent } from '../../../types';
 import type { FormatFactory } from '../../../../common/types';
 import { RowHeightMode } from '../../../../common/types';
-import { getOriginalId, isTransposeId, LensGridDirection } from '../../../../common/expressions';
+import { LensGridDirection } from '../../../../common/expressions';
 import { VisualizationContainer } from '../../../visualization_container';
 import { findMinMaxByColumnId, shouldColorByTerms } from '../../../shared_components';
 import type {
@@ -288,8 +289,7 @@ export const DatatableComponent = (props: DatatableRenderProps) => {
       columnConfig.columns
         .filter(({ columnId }) => isNumericMap.get(columnId))
         .map(({ columnId }) => columnId),
-      props.data,
-      getOriginalId
+      props.data
     );
   }, [props.data, isNumericMap, columnConfig]);
 

x-pack/plugins/lens/public/visualizations/datatable/index.ts

@@ -48,6 +48,7 @@ export class DatatableVisualization {
       return getDatatableVisualization({
         paletteService: palettes,
         kibanaTheme: core.theme,
+        formatFactory,
       });
     });
   }

x-pack/plugins/lens/public/visualizations/datatable/visualization.test.tsx

@@ -28,6 +28,7 @@ import {
   DatatableExpressionFunction,
 } from '../../../common/expressions';
 import { getColorStops } from '../../shared_components/coloring';
+import { fieldFormatsServiceMock } from '@kbn/field-formats-plugin/public/mocks';
 
 jest.mock('../../shared_components/coloring', () => {
   return {
@@ -46,6 +47,7 @@ function mockFrame(): FramePublicAPI {
 const mockServices = {
   paletteService: chartPluginMock.createPaletteRegistry(),
   kibanaTheme: themeServiceMock.createStartContract(),
+  formatFactory: fieldFormatsServiceMock.createStartContract().deserialize,
 };
 
 const datatableVisualization = getDatatableVisualization(mockServices);

x-pack/plugins/lens/public/visualizations/datatable/visualization.tsx

@@ -12,9 +12,11 @@ import { PaletteRegistry, CUSTOM_PALETTE, PaletteOutput, CustomPaletteParams } f
 import { ThemeServiceStart } from '@kbn/core/public';
 import { VIS_EVENT_TO_TRIGGER } from '@kbn/visualizations-plugin/public';
 import { IconChartDatatable } from '@kbn/chart-icons';
+import { getOriginalId } from '@kbn/transpose-utils';
 import { LayerTypes } from '@kbn/expression-xy-plugin/public';
 import { buildExpression, buildExpressionFunction } from '@kbn/expressions-plugin/common';
 import useObservable from 'react-use/lib/useObservable';
+import { getSortingCriteria } from '@kbn/sort-predicates';
 import type { FormBasedPersistedState } from '../../datasources/form_based/types';
 import type {
   SuggestionRequest,
@@ -25,7 +27,7 @@ import type {
 } from '../../types';
 import { TableDimensionDataExtraEditor, TableDimensionEditor } from './components/dimension_editor';
 import { TableDimensionEditorAdditionalSection } from './components/dimension_editor_addtional_section';
-import type { LayerType } from '../../../common/types';
+import type { FormatFactory, LayerType } from '../../../common/types';
 import { RowHeightMode } from '../../../common/types';
 import { getDefaultSummaryLabel } from '../../../common/expressions/datatable/summary';
 import {
@@ -35,7 +37,6 @@ import {
   type CollapseExpressionFunction,
   type DatatableColumnFn,
   type DatatableExpressionFunction,
-  getOriginalId,
 } from '../../../common/expressions';
 import { DataTableToolbar } from './components/toolbar';
 import {
@@ -51,6 +52,8 @@ import {
   shouldColorByTerms,
 } from '../../shared_components';
 import { getColorMappingTelemetryEvents } from '../../lens_ui_telemetry/color_telemetry_helpers';
+import { DatatableInspectorTables } from '../../../common/expressions/datatable/datatable_fn';
+import { getSimpleColumnType } from './components/table_actions';
 export interface DatatableVisualizationState {
   columns: ColumnState[];
   layerId: string;
@@ -70,9 +73,11 @@ const visualizationLabel = i18n.translate('xpack.lens.datatable.label', {
 export const getDatatableVisualization = ({
   paletteService,
   kibanaTheme,
+  formatFactory,
 }: {
   paletteService: PaletteRegistry;
   kibanaTheme: ThemeServiceStart;
+  formatFactory: FormatFactory;
 }): Visualization<DatatableVisualizationState> => ({
   id: 'lnsDatatable',
 
@@ -146,7 +151,7 @@ export const getDatatableVisualization = ({
               .filter(({ id }) => getOriginalId(id) === accessor)
               .map(({ id }) => id) || []
           : [accessor];
-        const minMaxByColumnId = findMinMaxByColumnId(columnsToCheck, currentData, getOriginalId);
+        const minMaxByColumnId = findMinMaxByColumnId(columnsToCheck, currentData);
         const dataBounds = minMaxByColumnId.get(accessor);
         if (palette && !showColorByTerms && !palette?.canDynamicColoring && dataBounds) {
           const newPalette: PaletteOutput<CustomPaletteParams> = {
@@ -264,8 +269,10 @@ export const getDatatableVisualization = ({
   **/
   getConfiguration({ state, frame }) {
     const isDarkMode = kibanaTheme.getTheme().darkMode;
-    const { sortedColumns, datasource } =
-      getDataSourceAndSortedColumns(state, frame.datasourceLayers) || {};
+    const { sortedColumns, datasource } = getDatasourceAndSortedColumns(
+      state,
+      frame.datasourceLayers
+    );
 
     const columnMap: Record<string, ColumnState> = {};
     state.columns.forEach((column) => {
@@ -496,8 +503,7 @@ export const getDatatableVisualization = ({
     { title, description } = {},
     datasourceExpressionsByLayers = {}
   ): Ast | null {
-    const { sortedColumns, datasource } =
-      getDataSourceAndSortedColumns(state, datasourceLayers) || {};
+    const { sortedColumns, datasource } = getDatasourceAndSortedColumns(state, datasourceLayers);
     const isTextBasedLanguage = datasource?.isTextBasedLanguage();
 
     if (
@@ -730,9 +736,40 @@ export const getDatatableVisualization = ({
     return suggestion;
   },
 
-  getSortedColumns(state, datasourceLayers) {
-    const { sortedColumns } = getDataSourceAndSortedColumns(state, datasourceLayers || {}) || {};
-    return sortedColumns;
+  getExportDatatables(state, datasourceLayers = {}, activeData) {
+    const columnMap = new Map(state.columns.map((c) => [c.columnId, c]));
+    const datatable =
+      activeData?.[DatatableInspectorTables.Transpose] ??
+      activeData?.[DatatableInspectorTables.Default];
+    if (!datatable) return [];
+
+    const columns = datatable.columns.filter(({ id }) => !columnMap.get(getOriginalId(id))?.hidden);
+    let rows = datatable.rows;
+
+    const sortColumn =
+      state.sorting?.columnId && columns.find(({ id }) => id === state.sorting?.columnId);
+    const sortDirection = state.sorting?.direction;
+
+    if (sortColumn && sortDirection && sortDirection !== 'none') {
+      const datasource = datasourceLayers[state.layerId];
+      const schemaType =
+        datasource?.getOperationForColumnId?.(sortColumn.id)?.sortingHint ??
+        getSimpleColumnType(sortColumn.meta);
+      const sortingCriteria = getSortingCriteria(
+        schemaType,
+        sortColumn.id,
+        formatFactory(sortColumn.meta?.params)
+      );
+      rows = [...rows].sort((rA, rB) => sortingCriteria(rA, rB, sortDirection));
+    }
+
+    return [
+      {
+        ...datatable,
+        columns,
+        rows,
+      },
+    ];
   },
 
   getVisualizationInfo(state) {
@@ -782,7 +819,7 @@ export const getDatatableVisualization = ({
   },
 });
 
-function getDataSourceAndSortedColumns(
+function getDatasourceAndSortedColumns(
   state: DatatableVisualizationState,
   datasourceLayers: DatasourceLayers
 ) {
@@ -792,5 +829,6 @@ function getDataSourceAndSortedColumns(
   const sortedColumns = Array.from(
     new Set(originalOrder?.concat(state.columns.map(({ columnId }) => columnId)))
   );
+
   return { datasource, sortedColumns };
 }

x-pack/plugins/lens/tsconfig.json

@@ -113,6 +113,7 @@
     "@kbn/react-kibana-mount",
     "@kbn/es-types",
     "@kbn/esql-datagrid",
+    "@kbn/transpose-utils",
   ],
   "exclude": ["target/**/*"]
 }

x-pack/plugins/maps/public/lens/choropleth_chart/expression_function.ts

@@ -5,7 +5,11 @@
  * 2.0.
  */
 
-import type { ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common';
+import type {
+  DefaultInspectorAdapters,
+  ExecutionContext,
+  ExpressionFunctionDefinition,
+} from '@kbn/expressions-plugin/common';
 import { Datatable } from '@kbn/expressions-plugin/common';
 import { i18n } from '@kbn/i18n';
 import { prepareLogTable } from '@kbn/visualizations-plugin/common/utils';
@@ -22,7 +26,8 @@ export const getExpressionFunction = (): ExpressionFunctionDefinition<
   'lens_choropleth_chart',
   Datatable,
   Omit<ChoroplethChartConfig, 'layerType'>,
-  ChoroplethChartRender
+  ChoroplethChartRender,
+  ExecutionContext<DefaultInspectorAdapters>
 > => ({
   name: 'lens_choropleth_chart',
   type: 'render',

yarn.lock

@@ -7041,6 +7041,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/transpose-utils@link:packages/kbn-transpose-utils":
+  version "0.0.0"
+  uid ""
+
 "@kbn/triggers-actions-ui-example-plugin@link:x-pack/examples/triggers_actions_ui_example":
   version "0.0.0"
   uid ""