[Expressions] Introduce createTable expression function, and use in Lens (#103788)

* [Expressions] Introduce createTable expression function, and use in Lens

* Fix test

* Fix code style

* Fix typo

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

docs/canvas/canvas-function-reference.asciidoc

@@ -406,6 +406,60 @@ This prints the `datatable` objects in the browser console before and after the
 
 *Returns:* `any`
 
+[float]
+[[createTable_fn]]
+=== `createTable`
+
+Creates a datatable with a list of columns, and 1 or more empty rows.
+To populate the rows, use <<mapColumn_fn>> or <<mathColumn_fn>>.
+
+[cols="3*^<"]
+|===
+|Argument |Type |Description
+
+|ids *** †
+
+|`string`
+|Column ids to generate in positional order. ID represents the key in the row.
+
+|`names` †
+|`string`
+|Column names to generate in positional order. Names are not required to be unique, and default to the ID if not provided.
+
+|`rowCount`
+
+Default: 1
+|`number`
+|The number of empty rows to add to the table, to be assigned a value later.
+|===
+
+*Expression syntax*
+[source,js]
+----
+createTable id="a" id="b"
+createTable id="a" name="A" id="b" name="B" rowCount=5
+----
+
+*Code example*
+[source,text]
+----
+var_set
+  name="logs" value={essql "select count(*) as a from kibana_sample_data_logs"}
+  name="commerce" value={essql "select count(*) as b from kibana_sample_data_ecommerce"}
+| createTable ids="totalA" ids="totalB"
+| staticColumn name="totalA" value={var "logs" | getCell "a"}
+| alterColumn column="totalA" type="number"
+| staticColumn name="totalB" value={var "commerce" | getCell "b"}
+| alterColumn column="totalB" type="number"
+| mathColumn id="percent" name="percent" expression="totalA / totalB"
+| render
+----
+
+This creates a table based on the results of two `essql` queries, joined
+into one table.
+
+*Accepts:* `null`
+
 
 [float]
 [[columns_fn]]

src/plugins/expressions/common/expression_functions/specs/create_table.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { ExpressionFunctionDefinition } from '../types';
+import { Datatable, DatatableColumn } from '../../expression_types';
+
+export interface CreateTableArguments {
+  ids: string[];
+  names: string[] | null;
+  rowCount: number;
+}
+
+export const createTable: ExpressionFunctionDefinition<
+  'createTable',
+  null,
+  CreateTableArguments,
+  Datatable
+> = {
+  name: 'createTable',
+  type: 'datatable',
+  inputTypes: ['null'],
+  help: i18n.translate('expressions.functions.createTableHelpText', {
+    defaultMessage:
+      'Creates a datatable with a list of columns, and 1 or more empty rows. ' +
+      'To populate the rows, use {mapColumnFn} or {mathColumnFn}.',
+    values: {
+      mathColumnFn: '`mathColumn`',
+      mapColumnFn: '`mapColumn`',
+    },
+  }),
+  args: {
+    ids: {
+      types: ['string'],
+      help: i18n.translate('expressions.functions.createTable.args.idsHelpText', {
+        defaultMessage:
+          'Column ids to generate in positional order. ID represents the key in the row.',
+      }),
+      required: false,
+      multi: true,
+    },
+    names: {
+      types: ['string'],
+      help: i18n.translate('expressions.functions.createTable.args.nameHelpText', {
+        defaultMessage:
+          'Column names to generate in positional order. Names are not required to be unique, and default to the ID if not provided.',
+      }),
+      required: false,
+      multi: true,
+    },
+    rowCount: {
+      types: ['number'],
+      help: i18n.translate('expressions.functions.createTable.args.rowCountText', {
+        defaultMessage:
+          'The number of empty rows to add to the table, to be assigned a value later',
+      }),
+      default: 1,
+      required: false,
+    },
+  },
+  fn(input, args) {
+    const columns: DatatableColumn[] = [];
+
+    (args.ids ?? []).map((id, index) => {
+      columns.push({
+        id,
+        name: args.names?.[index] ?? id,
+        meta: { type: 'null' },
+      });
+    });
+
+    return {
+      columns,
+      // Each row gets a unique object
+      rows: [...Array(args.rowCount)].map(() => ({})),
+      type: 'datatable',
+    };
+  },
+};

src/plugins/expressions/common/expression_functions/specs/index.ts

@@ -7,6 +7,7 @@
  */
 
 export * from './clog';
+export * from './create_table';
 export * from './font';
 export * from './var_set';
 export * from './var';

src/plugins/expressions/common/expression_functions/specs/tests/create_table.test.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { functionWrapper } from './utils';
+import { createTable } from '../create_table';
+
+describe('clear', () => {
+  const fn = functionWrapper(createTable);
+
+  it('returns a blank table', () => {
+    expect(fn(null, {})).toEqual({
+      type: 'datatable',
+      columns: [],
+      rows: [{}],
+    });
+  });
+
+  it('creates a table with default names', () => {
+    expect(
+      fn(null, {
+        ids: ['a', 'b'],
+        rowCount: 3,
+      })
+    ).toEqual({
+      type: 'datatable',
+      columns: [
+        { id: 'a', name: 'a', meta: { type: 'null' } },
+        { id: 'b', name: 'b', meta: { type: 'null' } },
+      ],
+      rows: [{}, {}, {}],
+    });
+  });
+
+  it('create a table with names that match by position', () => {
+    expect(
+      fn(null, {
+        ids: ['a', 'b'],
+        names: ['name'],
+      })
+    ).toEqual({
+      type: 'datatable',
+      columns: [
+        { id: 'a', name: 'name', meta: { type: 'null' } },
+        { id: 'b', name: 'b', meta: { type: 'null' } },
+      ],
+      rows: [{}],
+    });
+  });
+
+  it('does provides unique objects for each row', () => {
+    const table = fn(null, {
+      ids: ['a', 'b'],
+      rowCount: 2,
+    });
+
+    table.rows[0].a = 'z';
+    table.rows[1].b = 5;
+
+    expect(table.rows).toEqual([{ a: 'z' }, { b: 5 }]);
+  });
+});

src/plugins/expressions/common/service/expressions_services.ts

@@ -21,6 +21,7 @@ import { PersistableStateService, SerializableState } from '../../../kibana_util
 import { Adapters } from '../../../inspector/common/adapters';
 import {
   clog,
+  createTable,
   font,
   variableSet,
   variable,
@@ -335,6 +336,7 @@ export class ExpressionsService implements PersistableStateService<ExpressionAst
   public setup(...args: unknown[]): ExpressionsServiceSetup {
     for (const fn of [
       clog,
+      createTable,
       font,
       variableSet,
       variable,

x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts

@@ -290,7 +290,7 @@ describe('IndexPattern Data Source', () => {
       expect(indexPatternDatasource.toExpression(state, 'first')).toEqual(null);
     });
 
-    it('should generate an empty expression when there is a formula without aggs', async () => {
+    it('should create a table when there is a formula without aggs', async () => {
       const queryBaseState: IndexPatternBaseState = {
         currentIndexPatternId: '1',
         layers: {
@@ -311,7 +311,21 @@ describe('IndexPattern Data Source', () => {
         },
       };
       const state = enrichBaseState(queryBaseState);
-      expect(indexPatternDatasource.toExpression(state, 'first')).toEqual(null);
+      expect(indexPatternDatasource.toExpression(state, 'first')).toEqual({
+        chain: [
+          {
+            function: 'createTable',
+            type: 'function',
+            arguments: { ids: [], names: [], rowCount: [1] },
+          },
+          {
+            arguments: { expression: [''], id: ['col1'], name: ['Formula'] },
+            function: 'mapColumn',
+            type: 'function',
+          },
+        ],
+        type: 'expression',
+      });
     });
 
     it('should generate an expression for an aggregated query', async () => {

x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx

@@ -440,6 +440,47 @@ describe('formula', () => {
       });
     });
 
+    it('should create a valid formula expression for numeric literals', () => {
+      expect(
+        regenerateLayerFromAst(
+          '0',
+          layer,
+          'col1',
+          currentColumn,
+          indexPattern,
+          operationDefinitionMap
+        ).newLayer
+      ).toEqual({
+        ...layer,
+        columnOrder: ['col1X0', 'col1'],
+        columns: {
+          ...layer.columns,
+          col1: {
+            ...currentColumn,
+            label: '0',
+            references: ['col1X0'],
+            params: {
+              ...currentColumn.params,
+              formula: '0',
+              isFormulaBroken: false,
+            },
+          },
+          col1X0: {
+            customLabel: true,
+            dataType: 'number',
+            isBucketed: false,
+            label: 'Part of 0',
+            operationType: 'math',
+            params: {
+              tinymathAst: 0,
+            },
+            references: [],
+            scale: 'ratio',
+          },
+        },
+      });
+    });
+
     it('returns no change but error if the formula cannot be parsed', () => {
       const formulas = [
         '+',

x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx

@@ -55,8 +55,8 @@ export const formulaOperation: OperationDefinition<
 
     const visibleOperationsMap = filterByVisibleOperation(operationDefinitionMap);
     const { root, error } = tryToParse(column.params.formula, visibleOperationsMap);
-    if (error || !root) {
-      return [error!.message];
+    if (error || root == null) {
+      return error?.message ? [error.message] : [];
     }
 
     const errors = runASTValidation(root, layer, indexPattern, visibleOperationsMap);

x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts

@@ -35,7 +35,7 @@ function parseAndExtract(
   label?: string
 ) {
   const { root, error } = tryToParse(text, operationDefinitionMap);
-  if (error || !root) {
+  if (error || root == null) {
     return { extracted: [], isValid: false };
   }
   // before extracting the data run the validation task and throw if invalid

x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts

@@ -135,10 +135,6 @@ function getExpressionForLayer(
       }
     });
 
-    if (esAggEntries.length === 0) {
-      // Return early if there are no aggs, for example if the user has an empty formula
-      return null;
-    }
     const idMap = esAggEntries.reduce((currentIdMap, [colId, column], index) => {
       const esAggsId = `col-${index}-${index}`;
       return {
@@ -234,6 +230,26 @@ function getExpressionForLayer(
       }
     );
 
+    if (esAggEntries.length === 0) {
+      return {
+        type: 'expression',
+        chain: [
+          {
+            type: 'function',
+            function: 'createTable',
+            arguments: {
+              ids: [],
+              names: [],
+              rowCount: [1],
+            },
+          },
+          ...expressions,
+          ...formatterOverrides,
+          ...timeScaleFunctions,
+        ],
+      };
+    }
+
     const allDateHistogramFields = Object.values(columns)
       .map((column) =>
         column.operationType === dateHistogramOperation.type ? column.sourceField : null

x-pack/plugins/lens/public/metric_visualization/expression.tsx

@@ -127,8 +127,11 @@ export function MetricChart({
     return <EmptyPlaceholder icon={LensIconChartMetric} />;
   }
 
-  const column = firstTable.columns.find(({ id }) => id === accessor)!;
+  const column = firstTable.columns.find(({ id }) => id === accessor);
   const row = firstTable.rows[0];
+  if (!column || !row) {
+    return <EmptyPlaceholder icon={LensIconChartMetric} />;
+  }
 
   // NOTE: Cardinality and Sum never receives "null" as value, but always 0, even for empty dataset.
   // Mind falsy values here as 0!

x-pack/test/functional/apps/lens/formula.ts

@@ -236,5 +236,25 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         'count()'
       );
     });
+
+    it('should allow numeric only formulas', async () => {
+      await PageObjects.visualize.navigateToNewVisualization();
+      await PageObjects.visualize.clickVisType('lens');
+      await PageObjects.lens.goToTimeRange();
+      await PageObjects.lens.switchToVisualization('lnsDatatable');
+
+      await PageObjects.lens.configureDimension({
+        dimension: 'lnsDatatable_metrics > lns-empty-dimension',
+        operation: 'formula',
+        formula: `0`,
+      });
+
+      await PageObjects.lens.dragDimensionToDimension(
+        'lnsDatatable_metrics > lns-dimensionTrigger',
+        'lnsDatatable_metrics > lns-empty-dimension'
+      );
+      expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('0');
+      expect(await PageObjects.lens.getDatatableCellText(0, 1)).to.eql('0');
+    });
   });
 }