github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

lens esql generation (#196049)

Browse source
This commit is contained in:
Peter Pisljar 2025-02-03 09:53:24 +01:00 • committed by GitHub
parent e0fa8468a7
commit 762fd8c4d0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 1276 additions and 195 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/platform/packages/shared/kbn-esql-utils/src/utils/run_query.test.ts
View file

@ -137,4 +137,14 @@ describe('run query helpers', () => {
});
});
});
it('should work correctly with datemath ranges', () => {
const time = { from: 'now/d', to: 'now/d' };
const query = 'FROM foo | where time < ?_tend amd time > ?_tstart';
const params = getStartEndParams(query, time);
expect(params).toHaveLength(2);
expect(params[0]).toHaveProperty('_tstart');
expect(params[1]).toHaveProperty('_tend');
expect(params[0]._tstart).not.toEqual(params[1]._tend);
});
});

2
src/platform/packages/shared/kbn-esql-utils/src/utils/run_query.ts
View file

@ -25,7 +25,7 @@ export const getStartEndParams = (query: string, time?: TimeRange) => {
if (time && (startNamedParams || endNamedParams)) {
const timeParams = {
start: startNamedParams ? dateMath.parse(time.from)?.toISOString() : undefined,
end: endNamedParams ? dateMath.parse(time.to)?.toISOString() : undefined,
end: endNamedParams ? dateMath.parse(time.to, { roundUp: true })?.toISOString() : undefined,
};
const namedParams = [];
if (timeParams?.start) {

2
src/platform/plugins/shared/data/common/search/aggs/buckets/index.ts
View file

@ -30,7 +30,7 @@ export * from './lib/cidr_mask';
export * from './lib/date_range';
export * from './lib/ip_range';
export * from './lib/time_buckets/calc_auto_interval';
export { TimeBuckets } from './lib/time_buckets';
export { TimeBuckets, convertDurationToNormalizedEsInterval } from './lib/time_buckets';
export * from './migrate_include_exclude_format';
export * from './range_fn';
export * from './range';

1
src/platform/plugins/shared/data/common/search/aggs/buckets/lib/time_buckets/index.ts
View file

@ -8,3 +8,4 @@
*/
export { TimeBuckets } from './time_buckets';
export { convertDurationToNormalizedEsInterval } from './calc_es_interval';

10
src/platform/plugins/shared/data/common/search/expressions/esql.ts
View file

@ -27,6 +27,7 @@ import { zipObject } from 'lodash';
import { catchError, defer, map, Observable, switchMap, tap, throwError } from 'rxjs';
import { buildEsQuery, type Filter } from '@kbn/es-query';
import type { ESQLSearchParams, ESQLSearchResponse } from '@kbn/es-types';
import DateMath from '@kbn/datemath';
import { getEsQueryConfig } from '../../es_query';
import { getTime } from '../../query';
import {
@ -333,6 +334,13 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
);
const indexPattern = getIndexPatternFromESQLQuery(query);
const appliedTimeRange = input?.timeRange
? {
from: DateMath.parse(input.timeRange.from),
to: DateMath.parse(input.timeRange.to, { roundUp: true }),
}
: undefined;
const allColumns =
(body.all_columns ?? body.columns)?.map(({ name, type }) => ({
id: name,
@ -343,7 +351,7 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
sourceParams:
type === 'date'
? {
appliedTimeRange: input?.timeRange,
appliedTimeRange,
params: {},
indexPattern,
}

16
src/platform/plugins/shared/data/public/actions/filters/create_filters_from_range_select.test.ts
View file

@ -97,6 +97,7 @@ describe('brushEvent', () => {
name: '1',
meta: {
type: 'date',
sourceParams: {},
},
},
],
@ -247,5 +248,20 @@ describe('brushEvent', () => {
expect(rangeFilter.query.range['1']).toHaveProperty('format', 'strict_date_optional_time');
}
});
test('for column with different name than source field', async () => {
const rangeBegin = JAN_01_2014;
const rangeEnd = rangeBegin + DAY_IN_MS;
esqlEventContext.range = [rangeBegin, rangeEnd];
esqlEventContext.table.columns[0].meta!.sourceParams!.sourceField = 'time';
esqlEventContext.table.columns[0].name = 'time over 12h';
const filter = await createFiltersFromRangeSelectAction(esqlEventContext);
expect(filter).toBeDefined();
expect(filter.length).toEqual(1);
expect(filter[0].query).toBeDefined();
expect(filter[0].query!.range.time).toBeDefined();
});
});
});

8
src/platform/plugins/shared/data/public/actions/filters/create_filters_from_range_select.ts
View file

@ -28,9 +28,9 @@ export interface RangeSelectDataContext {
const getParameters = async (event: RangeSelectDataContext) => {
const column: Record<string, any> = event.table.columns[event.column];
// Handling of the ES|QL datatable
if (isOfAggregateQueryType(event.query)) {
if (isOfAggregateQueryType(event.query) || event.table.meta?.type === 'es_ql') {
const field = new DataViewField({
name: column.name,
name: column.meta?.sourceParams?.sourceField || column.name,
type: column.meta?.type ?? 'unknown',
esTypes: column.meta?.esType ? ([column.meta.esType] as string[]) : undefined,
searchable: true,
@ -43,7 +43,9 @@ const getParameters = async (event: RangeSelectDataContext) => {
};
}
if (column.meta && 'sourceParams' in column.meta) {
const { indexPatternId, ...aggConfigs } = column.meta.sourceParams;
const { sourceField, ...aggConfigs } = column.meta.sourceParams;
const indexPatternId =
column.meta.sourceParams.indexPatternId || column.meta.sourceParams.indexPattern;
const indexPattern = await getIndexPatterns().get(indexPatternId);
const aggConfigsInstance = getSearchService().aggs.createAggConfigs(indexPattern, [
aggConfigs as AggConfigSerialized,

64
src/platform/plugins/shared/data/public/actions/filters/create_filters_from_value_click.test.ts
View file

@ -13,9 +13,11 @@ import { setIndexPatterns, setSearchService } from '../../services';
import {
createFiltersFromValueClickAction,
appendFilterToESQLQueryFromValueClickAction,
createFilterESQL,
} from './create_filters_from_value_click';
import { FieldFormatsGetConfigFn, BytesFormat } from '@kbn/field-formats-plugin/common';
import { RangeFilter } from '@kbn/es-query';
import { Datatable } from '@kbn/expressions-plugin/common';
const mockField = {
name: 'bytes',
@ -105,6 +107,68 @@ describe('createFiltersFromClickEvent', () => {
expect(filters.length).toEqual(1);
});
});
describe('createFilterESQL', () => {
let table: Datatable;
beforeEach(() => {
table = {
type: 'datatable',
columns: [
{
name: 'test',
id: '1-1',
meta: {
type: 'number',
sourceParams: {
sourceField: 'bytes',
},
},
},
],
rows: [
{
'1-1': '2048',
},
],
};
});
test('ignores event when sourceField is missing', async () => {
table.columns[0].meta.sourceParams = {};
const filter = await createFilterESQL(table, 0, 0);
expect(filter).toEqual([]);
});
test('ignores event when value for rows is not provided', async () => {
table.rows[0]['1-1'] = null;
const filter = await createFilterESQL(table, 0, 0);
expect(filter).toEqual([]);
});
test('handles an event when operation type is a date histogram', async () => {
(table.columns[0].meta.sourceParams as any).operationType = 'date_histogram';
const filter = await createFilterESQL(table, 0, 0);
expect(filter).toMatchInlineSnapshot(`Array []`);
});
test('handles an event when operation type is histogram', async () => {
(table.columns[0].meta.sourceParams as any).operationType = 'histogram';
const filter = await createFilterESQL(table, 0, 0);
expect(filter).toMatchInlineSnapshot(`Array []`);
});
test('handles an event when operation type is not date histogram', async () => {
const filter = await createFilterESQL(table, 0, 0);
expect(filter).toMatchInlineSnapshot(`Array []`);
});
});
describe('appendFilterToESQLQueryFromValueClickAction', () => {
let dataPoints: Parameters<typeof appendFilterToESQLQueryFromValueClickAction>[0]['data'];
beforeEach(() => {

58
src/platform/plugins/shared/data/public/actions/filters/create_filters_from_value_click.ts
View file

@ -8,7 +8,7 @@
*/
import _ from 'lodash';
import { Datatable } from '@kbn/expressions-plugin/public';
import { Datatable, isSourceParamsESQL } from '@kbn/expressions-plugin/public';
import {
compareFilters,
COMPARE_ALL_OPTIONS,
@ -17,13 +17,17 @@ import {
type AggregateQuery,
} from '@kbn/es-query';
import { appendWhereClauseToESQLQuery } from '@kbn/esql-utils';
import {
buildSimpleExistFilter,
buildSimpleNumberRangeFilter,
} from '@kbn/es-query/src/filters/build_filters';
import { getIndexPatterns, getSearchService } from '../../services';
import { AggConfigSerialized } from '../../../common/search/aggs';
import { mapAndFlattenFilters } from '../../query';
interface ValueClickDataContext {
data: Array<{
table: Pick<Datatable, 'rows' | 'columns'>;
table: Pick<Datatable, 'rows' | 'columns' | 'meta'>;
column: number;
row: number;
value: any;
@ -129,6 +133,51 @@ export const createFilter = async (
return filter;
};
export const createFilterESQL = async (
table: Pick<Datatable, 'rows' | 'columns'>,
columnIndex: number,
rowIndex: number
) => {
const column = table?.columns?.[columnIndex];
if (
!column?.meta?.sourceParams?.sourceField ||
column.meta.sourceParams?.sourceField === '___records___'
) {
return [];
}
const sourceParams = column.meta.sourceParams;
if (!isSourceParamsESQL(sourceParams)) {
return [];
}
const { indexPattern, sourceField, operationType, interval } = sourceParams;
const value = rowIndex > -1 ? table.rows[rowIndex][column.id] : null;
if (value == null) {
return [];
}
const filters: Filter[] = [];
if (['date_histogram', 'histogram'].includes(operationType)) {
filters.push(
buildSimpleNumberRangeFilter(
sourceField,
{
gte: value,
lt: value + (interval || 0),
...(operationType === 'date_hisotgram' ? { format: 'strict_date_optional_time' } : {}),
},
value,
indexPattern
)
);
} else {
filters.push(buildSimpleExistFilter(sourceField, indexPattern));
}
return filters;
};
/** @public */
export const createFiltersFromValueClickAction = async ({
data,
@ -141,7 +190,10 @@ export const createFiltersFromValueClickAction = async ({
.filter((point) => point)
.map(async (val) => {
const { table, column, row } = val;
const filter: Filter[] = (await createFilter(table, column, row)) || [];
const filter =
table.meta?.type === 'es_ql'
? await createFilterESQL(table, column, row)
: (await createFilter(table, column, row)) || [];
if (filter) {
filter.forEach((f) => {
if (negate) {

1
src/platform/plugins/shared/data/public/index.ts
View file

@ -22,6 +22,7 @@ export {
getIndexPatternFromFilter,
} from './query';
export { convertIntervalToEsInterval } from '../common/search/aggs/buckets/lib/time_buckets/calc_es_interval';
/**
* Exporters (CSV)
*/

17
src/platform/plugins/shared/expressions/common/expression_types/specs/datatable.ts
View file

@ -90,6 +90,23 @@ export interface DatatableColumnMeta {
sourceParams?: SerializableRecord;
}
interface SourceParamsESQL extends Record<string, unknown> {
indexPattern: string;
sourceField: string;
operationType: string;
interval?: number;
}
export function isSourceParamsESQL(obj: Record<string, unknown>): obj is SourceParamsESQL {
return (
obj &&
typeof obj.indexPattern === 'string' &&
typeof obj.sourceField === 'string' &&
typeof obj.operationType === 'string' &&
(typeof obj.interval === 'number' || !obj.interval)
);
}
/**
* This type represents the shape of a column in a `Datatable`.
*/

2
src/platform/plugins/shared/expressions/public/index.ts
View file

@ -117,3 +117,5 @@ export {
parseExpression,
createDefaultInspectorAdapters,
} from '../common';
export { isSourceParamsESQL } from '../common/expression_types';

6
x-pack/platform/plugins/shared/lens/common/expressions/map_to_columns/map_to_columns.test.ts
View file

@ -319,8 +319,8 @@ describe('map_to_columns', () => {
);
expect(result.columns).toStrictEqual([
{ id: 'a', name: 'A', meta: { type: 'number', field: undefined, params: undefined } },
{ id: 'b', name: 'B', meta: { type: 'number', field: undefined, params: undefined } },
{ id: 'a', name: 'A', meta: { type: 'number', sourceParams: {} } },
{ id: 'b', name: 'B', meta: { type: 'number', sourceParams: {} } },
]);
expect(result.rows).toStrictEqual([
@ -370,7 +370,7 @@ describe('map_to_columns', () => {
);
expect(result.columns).toStrictEqual([
{ id: 'a', name: 'A', meta: { type: 'number', field: undefined, params: undefined } },
{ id: 'a', name: 'A', meta: { type: 'number', sourceParams: {} } },
{ id: 'field', name: 'C', meta: { type: 'string' }, variable: 'field' },
]);

2
x-pack/platform/plugins/shared/lens/common/expressions/map_to_columns/map_to_columns_fn.ts
View file

@ -10,7 +10,7 @@ import type { OriginalColumn, MapToColumnsExpressionFunction } from './types';
function getColumnName(originalColumn: OriginalColumn, newColumn: DatatableColumn) {
if (originalColumn?.operationType === 'date_histogram') {
const fieldName = originalColumn.sourceField;
const fieldName = originalColumn.sourceField as string;
// HACK: This is a hack, and introduces some fragility into
// column naming. Eventually, this should be calculated and

20
x-pack/platform/plugins/shared/lens/common/expressions/map_to_columns/map_to_columns_fn_textbased.ts
View file

@ -75,8 +75,24 @@ export const mapToOriginalColumnsTextBased: MapToColumnsExpressionFunction['fn']
name: originalColumn.label,
meta: {
...column.meta,
field: originalColumn.sourceField,
params: originalColumn.format,
...('sourceField' in originalColumn ? { field: originalColumn.sourceField } : {}),
...('format' in originalColumn ? { params: originalColumn.format } : {}),
sourceParams: {
...(column.meta?.sourceParams ?? {}),
...('sourceField' in originalColumn ? { sourceField: originalColumn.sourceField } : {}),
...('operationType' in originalColumn
? { operationType: originalColumn.operationType }
: {}),
...('interval' in originalColumn ? { interval: originalColumn.interval } : {}),
...('params' in originalColumn
? {
params: {
...(originalColumn.params as object),
used_interval: `${originalColumn.interval}ms`,
},
}
: {}),
},
},
}));
}),

4
x-pack/platform/plugins/shared/lens/common/expressions/map_to_columns/types.ts
View file

@ -14,8 +14,8 @@ export type OriginalColumn = {
variable?: string;
format?: SerializedFieldFormat;
} & (
| { operationType: 'date_histogram'; sourceField: string }
| { operationType: string; sourceField: never }
| { operationType: 'date_histogram'; sourceField: string; interval: number }
| { operationType: string; sourceField?: string; interval: never }
);
export type MapToColumnsExpressionFunction = ExpressionFunctionDefinition<

3
x-pack/platform/plugins/shared/lens/public/datasources/form_based/form_based.test.ts
View file

@ -592,6 +592,9 @@ describe('IndexPattern Data Source', () => {
"idMap": Array [
"{\\"col-0-0\\":[{\\"label\\":\\"Count of records\\",\\"dataType\\":\\"number\\",\\"isBucketed\\":false,\\"sourceField\\":\\"___records___\\",\\"operationType\\":\\"count\\",\\"id\\":\\"col1\\"}],\\"col-1-1\\":[{\\"label\\":\\"timestampLabel\\",\\"dataType\\":\\"date\\",\\"isBucketed\\":true,\\"operationType\\":\\"date_histogram\\",\\"sourceField\\":\\"timestamp\\",\\"params\\":{\\"interval\\":\\"1d\\"},\\"id\\":\\"col2\\"}]}",
],
"isTextBased": Array [
false,
],
},
"function": "lens_map_to_columns",
"type": "function",

3
x-pack/platform/plugins/shared/lens/public/datasources/form_based/form_based.tsx
View file

@ -211,7 +211,7 @@ export function getFormBasedDatasource({
dataViewFieldEditor: IndexPatternFieldEditorStart;
uiActions: UiActionsStart;
}) {
const { uiSettings } = core;
const { uiSettings, featureFlags } = core;
const DATASOURCE_ID = 'formBased';
const ALIAS_IDS = ['indexpattern'];
@ -464,6 +464,7 @@ export function getFormBasedDatasource({
layerId,
indexPatterns,
uiSettings,
featureFlags,
dateRange,
nowInstant,
searchSessionId,

9
x-pack/platform/plugins/shared/lens/public/datasources/form_based/operations/definitions/cardinality.tsx
View file

@ -77,6 +77,11 @@ export const cardinalityOperation: OperationDefinition<
displayName: CARDINALITY_NAME,
allowAsReference: true,
input: 'field',
getSerializedFormat() {
return {
id: 'number',
};
},
getPossibleOperationForField: ({
aggregationRestrictions,
aggregatable,
@ -176,6 +181,10 @@ export const cardinalityOperation: OperationDefinition<
},
];
},
toESQL: (column, columnId) => {
if (column.params?.emptyAsNull || column.timeShift) return;
return `COUNT_DISTINCT(${column.sourceField})`;
},
toEsAggsFn: (column, columnId) => {
return buildExpressionFunction<AggFunctionsMapping['aggCardinality']>('aggCardinality', {
id: columnId,

53
x-pack/platform/plugins/shared/lens/public/datasources/form_based/operations/definitions/count.test.ts
View file

@ -7,6 +7,10 @@
import { buildExpression, parseExpression } from '@kbn/expressions-plugin/common';
import { operationDefinitionMap } from '.';
import { IndexPattern } from '../../../../types';
import { FormBasedLayer } from '../../../..';
import { IUiSettingsClient } from '@kbn/core-ui-settings-browser';
import { DateRange } from '../../../../../common/types';
describe('count operation', () => {
describe('getGroupByKey', () => {
@ -119,4 +123,53 @@ describe('count operation', () => {
).toBeUndefined();
});
});
describe('toESQL', () => {
const callToESQL = (column: unknown) =>
operationDefinitionMap.count.toESQL!(
column as unknown as FormBasedLayer['columns'][0],
'1',
{
getFieldByName: jest.fn().mockImplementation((field) => {
if (field) return { type: 'number', name: field };
}),
} as unknown as IndexPattern,
{} as unknown as FormBasedLayer,
{} as unknown as IUiSettingsClient,
{} as unknown as DateRange
);
test('doesnt support filters', () => {
const esql = callToESQL({
sourceField: 'bytes',
operationType: 'count',
filter: { language: 'kquery' },
});
expect(esql).toBeUndefined();
});
test('doesnt support timeShift', () => {
const esql = callToESQL({
sourceField: 'bytes',
operationType: 'count',
timeShift: '1m',
});
expect(esql).toBeUndefined();
});
test('returns COUNT(*) for document fields', () => {
const esql = callToESQL({
operationType: 'count',
});
expect(esql).toBe('COUNT(*)');
});
test('returns COUNT(field) for non-document fields', () => {
const esql = callToESQL({
sourceField: 'bytes',
operationType: 'count',
});
expect(esql).toBe('COUNT(`bytes`)');
});
});
});

18
x-pack/platform/plugins/shared/lens/public/datasources/form_based/operations/definitions/count.tsx
View file

@ -12,6 +12,7 @@ import { EuiSwitch, EuiText } from '@elastic/eui';
import { AggFunctionsMapping } from '@kbn/data-plugin/public';
import { buildExpressionFunction } from '@kbn/expressions-plugin/public';
import { COUNT_ID, COUNT_NAME } from '@kbn/lens-formula-docs';
import { sanitazeESQLInput } from '@kbn/esql-utils';
import { TimeScaleUnit } from '../../../../../common/expressions';
import { OperationDefinition, ParamEditorProps } from '.';
import { FieldBasedIndexPatternColumn, ValueFormatConfig } from './column_types';
@ -185,6 +186,23 @@ export const countOperation: OperationDefinition<CountIndexPatternColumn, 'field
},
];
},
getSerializedFormat: (column, columnId, indexPattern) => {
const field = indexPattern?.getFieldByName(column.sourceField);
return field?.format ?? { id: 'number' };
},
toESQL: (column, columnId, indexPattern) => {
if (column.params?.emptyAsNull === false || column.timeShift || column.filter) return;
const field = indexPattern.getFieldByName(column.sourceField);
let esql = '';
if (!field || field?.type === 'document') {
esql = `COUNT(*)`;
} else {
esql = `COUNT(${sanitazeESQLInput(field.name)})`;
}
return esql;
},
toEsAggsFn: (column, columnId, indexPattern) => {
const field = indexPattern.getFieldByName(column.sourceField);
if (field?.type === 'document') {

127
x-pack/platform/plugins/shared/lens/public/datasources/form_based/operations/definitions/date_histogram.tsx
View file

@ -6,6 +6,7 @@
*/
import React, { useCallback, useEffect, useState } from 'react';
import moment from 'moment';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
@ -27,9 +28,16 @@ import {
search,
UI_SETTINGS,
} from '@kbn/data-plugin/public';
import { extendedBoundsToAst, intervalOptions } from '@kbn/data-plugin/common';
import {
extendedBoundsToAst,
intervalOptions,
getCalculateAutoTimeExpression,
} from '@kbn/data-plugin/common';
import { buildExpressionFunction } from '@kbn/expressions-plugin/public';
import { TooltipWrapper } from '@kbn/visualization-utils';
import { sanitazeESQLInput } from '@kbn/esql-utils';
import { DateRange } from '../../../../../common/types';
import { IndexPattern } from '../../../../types';
import { updateColumnParam } from '../layer_helpers';
import { FieldBasedOperationErrorMessage, OperationDefinition, ParamEditorProps } from '.';
import { FieldBasedIndexPatternColumn } from './column_types';
@ -81,6 +89,52 @@ function getMultipleDateHistogramsErrorMessage(
];
}
function getTimeZoneAndInterval(
column: DateHistogramIndexPatternColumn,
indexPattern: IndexPattern
) {
const usedField = indexPattern.getFieldByName(column.sourceField);
if (
usedField &&
usedField.aggregationRestrictions &&
usedField.aggregationRestrictions.date_histogram
) {
return {
interval: restrictedInterval(usedField.aggregationRestrictions) ?? autoInterval,
timeZone: usedField.aggregationRestrictions.date_histogram.time_zone,
usedField,
};
}
return {
usedField: undefined,
timeZone: undefined,
interval: column.params?.interval ?? autoInterval,
};
}
export function mapToEsqlInterval(dateRange: DateRange, interval: string) {
if (interval !== 'm' && interval.endsWith('m')) {
return interval.replace('m', ' minutes');
}
switch (interval) {
case '1M':
return '1 month';
case 'd':
return '1d';
case 'h':
return '1h';
case 'm':
return '1 minute';
case 's':
return '1s';
case 'ms':
return '1ms';
default:
return interval;
}
}
export const dateHistogramOperation: OperationDefinition<
DateHistogramIndexPatternColumn,
'field',
@ -110,7 +164,24 @@ export const dateHistogramOperation: OperationDefinition<
};
}
},
getDefaultLabel: (column, columns, indexPattern) => getSafeName(column.sourceField, indexPattern),
getDefaultLabel: (column, columns, indexPattern, uiSettings, dateRange) => {
const field = getSafeName(column.sourceField, indexPattern);
let interval = column.params?.interval || autoInterval;
if (dateRange && uiSettings) {
const calcAutoInterval = getCalculateAutoTimeExpression((key) => uiSettings.get(key));
interval =
calcAutoInterval({ from: dateRange.fromDate, to: dateRange.toDate }, interval, false)
?.description || 'hour';
return i18n.translate('xpack.lens.indexPattern.dateHistogram.interval', {
defaultMessage: `{field} per {interval}`,
values: {
field: field || '',
interval,
},
});
}
return field;
},
buildColumn({ field }, columnParams) {
return {
label: field.displayName,
@ -143,23 +214,53 @@ export const dateHistogramOperation: OperationDefinition<
sourceField: field.name,
};
},
getSerializedFormat: (column, targetColumn, indexPattern, uiSettings, dateRange) => {
if (!indexPattern || !dateRange || !uiSettings)
return {
id: 'date',
};
const { interval } = getTimeZoneAndInterval(column, indexPattern);
const calcAutoInterval = getCalculateAutoTimeExpression((key) => uiSettings.get(key));
const usedInterval =
calcAutoInterval(
{ from: dateRange.fromDate, to: dateRange.toDate },
interval,
false
)?.asMilliseconds() || 3600000;
const rules = uiSettings?.get('dateFormat:scaled');
for (let i = rules.length - 1; i >= 0; i--) {
const rule = rules[i];
if (!Array.isArray(rule) || rule.length !== 2) continue;
if (!rule[0] || (usedInterval && usedInterval >= moment.duration(rule[0]).asMilliseconds())) {
return { id: 'date', params: { pattern: rule[1] } };
}
}
return { id: 'date', params: { pattern: uiSettings?.get('dateFormat') } };
},
toESQL: (column, columnId, indexPattern, layer, uiSettings, dateRange) => {
if (column.params?.includeEmptyRows) return;
const { interval } = getTimeZoneAndInterval(column, indexPattern);
const calcAutoInterval = getCalculateAutoTimeExpression((key) => uiSettings.get(key));
if (interval === 'auto') {
return `BUCKET(${sanitazeESQLInput(column.sourceField)}, ${mapToEsqlInterval(
dateRange,
calcAutoInterval({ from: dateRange.fromDate, to: dateRange.toDate }) || '1h'
)})`;
}
return `BUCKET(${sanitazeESQLInput(column.sourceField)}, ${mapToEsqlInterval(
dateRange,
interval
)})`;
},
toEsAggsFn: (column, columnId, indexPattern) => {
const usedField = indexPattern.getFieldByName(column.sourceField);
let timeZone: string | undefined;
let interval = column.params?.interval ?? autoInterval;
const { usedField, timeZone, interval } = getTimeZoneAndInterval(column, indexPattern);
const dropPartials = Boolean(
column.params?.dropPartials &&
// set to false when detached from time picker
(indexPattern.timeFieldName === usedField?.name || !column.params?.ignoreTimeRange)
);
if (
usedField &&
usedField.aggregationRestrictions &&
usedField.aggregationRestrictions.date_histogram
) {
interval = restrictedInterval(usedField.aggregationRestrictions) as string;
timeZone = usedField.aggregationRestrictions.date_histogram.time_zone;
}
return buildExpressionFunction<AggFunctionsMapping['aggDateHistogram']>('aggDateHistogram', {
id: columnId,
enabled: true,

20
x-pack/platform/plugins/shared/lens/public/datasources/form_based/operations/definitions/index.ts
View file

@ -268,8 +268,17 @@ interface BaseOperationDefinitionProps<
getDefaultLabel: (
column: C,
columns: Record<string, GenericIndexPatternColumn>,
indexPattern?: IndexPattern
indexPattern?: IndexPattern,
uiSettings?: IUiSettingsClient,
dateRange?: DateRange
) => string;
getSerializedFormat?: (
column: C,
targetColumn: C,
indexPattern?: IndexPattern,
uiSettings?: IUiSettingsClient,
dateRange?: DateRange
) => Record<string, unknown>;
/**
* This function is called if another column in the same layer changed or got added/removed.
* Can be used to update references to other columns (e.g. for sorting).
@ -444,6 +453,15 @@ interface BaseOperationDefinitionProps<
* When present returns a dictionary of unsupported layer settings
*/
getUnsupportedSettings?: () => LayerSettingsFeatures;
toESQL?: (
column: C,
columnId: string,
indexPattern: IndexPattern,
layer: FormBasedLayer,
uiSettings: IUiSettingsClient,
dateRange: DateRange
) => string | undefined;
}
interface BaseBuildColumnArgs {

15
x-pack/platform/plugins/shared/lens/public/datasources/form_based/operations/definitions/metrics.tsx
View file

@ -24,6 +24,7 @@ import {
SUM_ID,
SUM_NAME,
} from '@kbn/lens-formula-docs';
import { sanitazeESQLInput } from '@kbn/esql-utils';
import { LayerSettingsFeatures, OperationDefinition, ParamEditorProps } from '.';
import {
getFormatFromPreviousColumn,
@ -59,6 +60,15 @@ const typeToFn: Record<string, string> = {
standard_deviation: 'aggStdDeviation',
};
const typeToESQLFn: Record<string, string> = {
min: 'MIN',
max: 'MAX',
average: 'AVG',
sum: 'SUM',
median: 'MEDIAN',
standard_deviation: 'MEDIAN_ABSOLUTE_DEVIATION',
};
const supportedTypes = ['number', 'histogram'];
function isTimeSeriesCompatible(type: string, timeSeriesMetric?: string) {
@ -213,6 +223,11 @@ function buildMetricOperation<T extends MetricColumn<string>>({
},
];
},
toESQL: (column, columnId, _indexPattern, layer) => {
if (column.timeShift) return;
if (!typeToESQLFn[type]) return;
return `${typeToESQLFn[type]}(${sanitazeESQLInput(column.sourceField)})`;
},
toEsAggsFn: (column, columnId, _indexPattern) => {
return buildExpressionFunction(typeToFn[type], {
id: columnId,

5
x-pack/platform/plugins/shared/lens/public/datasources/form_based/operations/definitions/percentile.tsx
View file

@ -17,6 +17,7 @@ import {
} from '@kbn/expressions-plugin/public';
import { useDebouncedValue } from '@kbn/visualization-utils';
import { PERCENTILE_ID, PERCENTILE_NAME } from '@kbn/lens-formula-docs';
import { sanitazeESQLInput } from '@kbn/esql-utils';
import { OperationDefinition } from '.';
import {
getFormatFromPreviousColumn,
@ -196,6 +197,10 @@ export const percentileOperation: OperationDefinition<
sourceField: field.name,
};
},
toESQL: (column, columnId) => {
if (column.timeShift) return;
return `PERCENTILE(${sanitazeESQLInput(column.sourceField)}, ${column.params.percentile})`;
},
toEsAggsFn: (column, columnId, _indexPattern) => {
return buildExpressionFunction<AggFunctionsMapping['aggSinglePercentile']>(
'aggSinglePercentile',

3
x-pack/platform/plugins/shared/lens/public/datasources/form_based/operations/definitions/ranges/ranges.tsx
View file

@ -138,6 +138,9 @@ export const rangeOperation: OperationDefinition<
sourceField: field.name,
};
},
toESQL: (column, columnId, _indexPattern, layer, uiSettings) => {
return undefined;
},
toEsAggsFn: (column, columnId, indexPattern, layer, uiSettings) => {
const { sourceField, params } = column;
if (params.type === MODES.Range) {

329
x-pack/platform/plugins/shared/lens/public/datasources/form_based/to_esql.test.ts Normal file
View file

@ -0,0 +1,329 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IndexPattern } from '../../types';
import { getESQLForLayer } from './to_esql';
import { createCoreSetupMock } from '@kbn/core-lifecycle-browser-mocks/src/core_setup.mock';
import { DateHistogramIndexPatternColumn } from '../..';
const defaultUiSettingsGet = (key: string) => {
switch (key) {
case 'dateFormat':
return 'MMM D, YYYY @ HH:mm:ss.SSS';
case 'dateFormat:scaled':
return [[]];
case 'dateFormat:tz':
return 'UTC';
case 'histogram:barTarget':
return 50;
case 'histogram:maxBars':
return 100;
}
};
describe('to_esql', () => {
const { uiSettings } = createCoreSetupMock();
uiSettings.get.mockImplementation((key: string) => {
return defaultUiSettingsGet(key);
});
const layer = {
indexPatternId: 'myIndexPattern',
columns: {},
columnOrder: [],
};
const indexPattern = {
title: 'myIndexPattern',
timeFieldName: 'order_date',
getFieldByName: (field: string) => {
if (field === 'records') return undefined;
return { name: field };
},
getFormatterForField: () => ({ convert: (v: unknown) => v }),
} as unknown as IndexPattern;
it('should produce valid esql for date histogram and count', () => {
const esql = getESQLForLayer(
[
[
'1',
{
operationType: 'date_histogram',
sourceField: 'order_date',
label: 'Date histogram',
dataType: 'date',
isBucketed: true,
interval: 'auto',
},
],
[
'2',
{
operationType: 'count',
sourceField: 'records',
label: 'Count',
dataType: 'number',
isBucketed: false,
},
],
],
layer,
indexPattern,
uiSettings,
{
fromDate: '2021-01-01T00:00:00.000Z',
toDate: '2021-01-01T23:59:59.999Z',
},
new Date()
);
expect(esql?.esql).toEqual(
'FROM myIndexPattern | WHERE order_date >= ?_tstart AND order_date <= ?_tend | STATS bucket_0_0 = COUNT(*) BY order_date = BUCKET(`order_date`, 30 minutes) | SORT order_date ASC'
);
});
it('should return undefined if missing row option is set', () => {
const esql = getESQLForLayer(
[
[
'1',
{
operationType: 'date_histogram',
sourceField: 'order_date',
label: 'Date histogram',
dataType: 'date',
isBucketed: true,
params: { includeEmptyRows: true },
} as DateHistogramIndexPatternColumn,
],
[
'2',
{
operationType: 'count',
sourceField: 'records',
label: 'Count',
dataType: 'number',
isBucketed: false,
},
],
],
layer,
indexPattern,
uiSettings,
{
fromDate: '2021-01-01T00:00:00.000Z',
toDate: '2021-01-01T23:59:59.999Z',
},
new Date()
);
expect(esql?.esql).toEqual(undefined);
});
it('should return undefined if lens formula is used', () => {
const esql = getESQLForLayer(
[
[
'1',
{
operationType: 'formula',
label: 'Formula',
isBucketed: false,
params: {},
dataType: 'number',
},
],
],
layer,
indexPattern,
uiSettings,
{
fromDate: '2021-01-01T00:00:00.000Z',
toDate: '2021-01-01T23:59:59.999Z',
},
new Date()
);
expect(esql).toEqual(undefined);
});
test('it should add a where condition to esql if timeField is set', () => {
const esql = getESQLForLayer(
[
[
'1',
{
operationType: 'date_histogram',
sourceField: 'order_date',
label: 'Date histogram',
dataType: 'date',
isBucketed: true,
interval: 'auto',
},
],
[
'2',
{
operationType: 'count',
sourceField: 'records',
label: 'Count',
dataType: 'number',
isBucketed: false,
},
],
],
layer,
indexPattern,
uiSettings,
{
fromDate: '2021-01-01T00:00:00.000Z',
toDate: '2021-01-01T23:59:59.999Z',
},
new Date()
);
expect(esql?.esql).toEqual(
'FROM myIndexPattern | WHERE order_date >= ?_tstart AND order_date <= ?_tend | STATS bucket_0_0 = COUNT(*) BY order_date = BUCKET(`order_date`, 30 minutes) | SORT order_date ASC'
);
});
it('should not add a where condition to esql if timeField is not set', () => {
const esql = getESQLForLayer(
[
[
'1',
{
operationType: 'date_histogram',
sourceField: 'order_date',
label: 'Date histogram',
dataType: 'date',
isBucketed: true,
interval: 'auto',
},
],
[
'2',
{
operationType: 'count',
sourceField: 'records',
label: 'Count',
dataType: 'number',
isBucketed: false,
},
],
],
layer,
{
title: 'myIndexPattern',
getFieldByName: (field: string) => {
if (field === 'records') return undefined;
return { name: field };
},
getFormatterForField: () => ({ convert: (v: unknown) => v }),
} as unknown as IndexPattern,
uiSettings,
{
fromDate: '2021-01-01T00:00:00.000Z',
toDate: '2021-01-01T23:59:59.999Z',
},
new Date()
);
expect(esql?.esql).toEqual(
'FROM myIndexPattern | STATS bucket_0_0 = COUNT(*) BY order_date = BUCKET(`order_date`, 30 minutes) | SORT order_date ASC'
);
});
it('should return undefined if timezone is not UTC', () => {
uiSettings.get.mockImplementation((key: string) => {
if (key === 'dateFormat:tz') return 'America/Chicago';
return defaultUiSettingsGet(key);
});
const esql = getESQLForLayer(
[
[
'1',
{
operationType: 'date_histogram',
sourceField: 'order_date',
label: 'Date histogram',
dataType: 'date',
isBucketed: true,
interval: 'auto',
},
],
[
'2',
{
operationType: 'count',
sourceField: 'records',
label: 'Count',
dataType: 'number',
isBucketed: false,
},
],
],
layer,
indexPattern,
uiSettings,
{
fromDate: '2021-01-01T00:00:00.000Z',
toDate: '2021-01-01T23:59:59.999Z',
},
new Date()
);
expect(esql).toEqual(undefined);
});
it('should work with iana timezones that fall udner utc+0', () => {
uiSettings.get.mockImplementation((key: string) => {
if (key === 'dateFormat:tz') return 'Europe/London';
return defaultUiSettingsGet(key);
});
const esql = getESQLForLayer(
[
[
'1',
{
operationType: 'date_histogram',
sourceField: 'order_date',
label: 'Date histogram',
dataType: 'date',
isBucketed: true,
interval: 'auto',
},
],
[
'2',
{
operationType: 'count',
sourceField: 'records',
label: 'Count',
dataType: 'number',
isBucketed: false,
},
],
],
layer,
indexPattern,
uiSettings,
{
fromDate: '2021-01-01T00:00:00.000Z',
toDate: '2021-01-01T23:59:59.999Z',
},
new Date()
);
expect(esql?.esql).toEqual(
`FROM myIndexPattern | WHERE order_date >= ?_tstart AND order_date <= ?_tend | STATS bucket_0_0 = COUNT(*) BY order_date = BUCKET(\`order_date\`, 30 minutes) | SORT order_date ASC`
);
});
});

303
x-pack/platform/plugins/shared/lens/public/datasources/form_based/to_esql.ts Normal file
View file

@ -0,0 +1,303 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IUiSettingsClient } from '@kbn/core/public';
import { UI_SETTINGS } from '@kbn/data-plugin/public';
import { getCalculateAutoTimeExpression, getUserTimeZone } from '@kbn/data-plugin/common';
import { convertIntervalToEsInterval } from '@kbn/data-plugin/public';
import moment from 'moment';
import { partition } from 'lodash';
import { isColumnOfType } from './operations/definitions/helpers';
import { ValueFormatConfig } from './operations/definitions/column_types';
import { convertToAbsoluteDateRange } from '../../utils';
import { DateRange, OriginalColumn } from '../../../common/types';
import { GenericIndexPatternColumn } from './form_based';
import { operationDefinitionMap } from './operations';
import { DateHistogramIndexPatternColumn } from './operations/definitions';
import type { IndexPattern } from '../../types';
import { resolveTimeShift } from './time_shift_utils';
import { FormBasedLayer } from '../..';
// esAggs column ID manipulation functions
export const extractAggId = (id: string) => id.split('.')[0].split('-')[2];
// Need a more complex logic for decimals percentiles
export function getESQLForLayer(
esAggEntries: Array<readonly [string, GenericIndexPatternColumn]>,
layer: FormBasedLayer,
indexPattern: IndexPattern,
uiSettings: IUiSettingsClient,
dateRange: DateRange,
nowInstant: Date
) {
// esql mode variables
const partialRows = true;
const timeZone = getUserTimeZone((key) => uiSettings.get(key), true);
const utcOffset = moment.tz(timeZone).utcOffset() / 60;
if (utcOffset !== 0) return;
if (
Object.values(layer.columns).find(
(col) =>
col.operationType === 'formula' ||
col.timeShift ||
('sourceField' in col && indexPattern.getFieldByName(col.sourceField)?.runtime)
)
)
return;
// indexPattern.title is the actual es pattern
const esql = [`FROM ${indexPattern.title}`];
if (indexPattern.timeFieldName) {
esql.push(
`WHERE ${indexPattern.timeFieldName} >= ?_tstart AND ${indexPattern.timeFieldName} <= ?_tend`
);
}
const histogramBarsTarget = uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET);
const absDateRange = convertToAbsoluteDateRange(dateRange, nowInstant);
const firstDateHistogramColumn = esAggEntries.find(
([, col]) => col.operationType === 'date_histogram'
);
const hasDateHistogram = Boolean(firstDateHistogramColumn);
const esAggsIdMap: Record<string, OriginalColumn[]> = {};
const [metricEsAggsEntries, bucketEsAggsEntries] = partition(
esAggEntries,
([_, col]) => !col.isBucketed
);
const metrics = metricEsAggsEntries.map(([colId, col], index) => {
const def = operationDefinitionMap[col.operationType];
if (!def.toESQL) return undefined;
const aggId = String(index);
const wrapInFilter = Boolean(def.filterable && col.filter?.query);
const wrapInTimeFilter =
def.canReduceTimeRange &&
!hasDateHistogram &&
col.reducedTimeRange &&
indexPattern.timeFieldName;
const esAggsId = window.ELASTIC_LENS_DELAY_SECONDS
? `bucket_${index + 1}_${aggId}`
: `bucket_${index}_${aggId}`;
const format =
operationDefinitionMap[col.operationType].getSerializedFormat?.(
col,
col,
indexPattern,
uiSettings,
dateRange
) ??
('sourceField' in col
? col.sourceField === '___records___'
? { id: 'number' }
: indexPattern.getFormatterForField(col.sourceField)
: undefined);
esAggsIdMap[esAggsId] = [
{
...col,
id: colId,
format: format as unknown as ValueFormatConfig,
interval: undefined as never,
label: col.customLabel
? col.label
: operationDefinitionMap[col.operationType].getDefaultLabel(
col,
layer.columns,
indexPattern,
uiSettings,
dateRange
),
},
];
let metricESQL = def.toESQL(
{
...col,
timeShift: resolveTimeShift(
col.timeShift,
absDateRange,
histogramBarsTarget,
hasDateHistogram
),
},
wrapInFilter || wrapInTimeFilter ? `${aggId}-metric` : aggId,
indexPattern,
layer,
uiSettings,
dateRange
);
if (!metricESQL) return undefined;
metricESQL = `${esAggsId} = ` + metricESQL;
if (wrapInFilter || wrapInTimeFilter) {
if (wrapInFilter) {
if (col.filter?.language === 'kquery') {
return;
}
return;
}
if (wrapInTimeFilter) {
return undefined;
}
}
return metricESQL;
});
if (metrics.some((m) => !m)) return;
let stats = `STATS ${metrics.join(', ')}`;
const buckets = bucketEsAggsEntries.map(([colId, col], index) => {
const def = operationDefinitionMap[col.operationType];
if (!def.toESQL) return undefined;
const aggId = String(index);
const wrapInFilter = Boolean(def.filterable && col.filter?.query);
const wrapInTimeFilter =
def.canReduceTimeRange &&
!hasDateHistogram &&
col.reducedTimeRange &&
indexPattern.timeFieldName;
let esAggsId = window.ELASTIC_LENS_DELAY_SECONDS
? `col_${index}-${aggId}`
: `col_${index}_${aggId}`;
let interval: number | undefined;
if (isColumnOfType<DateHistogramIndexPatternColumn>('date_histogram', col)) {
const dateHistogramColumn = col as DateHistogramIndexPatternColumn;
const calcAutoInterval = getCalculateAutoTimeExpression((key) => uiSettings.get(key));
const cleanInterval = (i: string) => {
switch (i) {
case 'd':
return '1d';
case 'h':
return '1h';
case 'm':
return '1m';
case 's':
return '1s';
case 'ms':
return '1ms';
default:
return i;
}
};
esAggsId = dateHistogramColumn.sourceField;
const kibanaInterval =
dateHistogramColumn.params?.interval === 'auto'
? calcAutoInterval({ from: dateRange.fromDate, to: dateRange.toDate }) || '1h'
: dateHistogramColumn.params?.interval || '1h';
const esInterval = convertIntervalToEsInterval(cleanInterval(kibanaInterval));
interval = moment.duration(esInterval.value, esInterval.unit).as('ms');
}
const format =
operationDefinitionMap[col.operationType].getSerializedFormat?.(
col,
col,
indexPattern,
uiSettings,
dateRange
) ?? ('sourceField' in col ? indexPattern.getFormatterForField(col.sourceField) : undefined);
esAggsIdMap[esAggsId] = [
{
...col,
id: colId,
format: format as unknown as ValueFormatConfig,
interval: interval as never,
...('sourceField' in col ? { sourceField: col.sourceField! } : {}),
label: col.customLabel
? col.label
: operationDefinitionMap[col.operationType].getDefaultLabel(
col,
layer.columns,
indexPattern,
uiSettings,
dateRange
),
},
];
if (isColumnOfType<DateHistogramIndexPatternColumn>('date_histogram', col)) {
const column = col;
if (
column.params?.dropPartials &&
// set to false when detached from time picker
(indexPattern.timeFieldName === indexPattern.getFieldByName(column.sourceField)?.name ||
!column.params?.ignoreTimeRange)
) {
return undefined;
}
}
return (
`${esAggsId} = ` +
def.toESQL(
{
...col,
timeShift: resolveTimeShift(
col.timeShift,
absDateRange,
histogramBarsTarget,
hasDateHistogram
),
},
wrapInFilter || wrapInTimeFilter ? `${aggId}-metric` : aggId,
indexPattern,
layer,
uiSettings,
dateRange
)
);
});
if (buckets.some((m) => !m)) return;
if (buckets.length > 0) {
stats += ` BY ${buckets.join(', ')}`;
esql.push(stats);
if (buckets.some((b) => !b || b.includes('undefined'))) return;
const sorts = bucketEsAggsEntries.map(([colId, col], index) => {
const aggId = String(index);
let esAggsId = window.ELASTIC_LENS_DELAY_SECONDS
? `col_${index}-${aggId}`
: `col_${index}_${aggId}`;
if (isColumnOfType<DateHistogramIndexPatternColumn>('date_histogram', col)) {
esAggsId = col.sourceField;
}
return `${esAggsId} ASC`;
});
esql.push(`SORT ${sorts.join(', ')}`);
} else {
esql.push(stats);
}
return {
esql: esql.join(' | '),
partialRows,
esAggsIdMap,
};
}

360
x-pack/platform/plugins/shared/lens/public/datasources/form_based/to_expression.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import type { IUiSettingsClient } from '@kbn/core/public';
import type { FeatureFlagsStart, IUiSettingsClient } from '@kbn/core/public';
import { partition, uniq } from 'lodash';
import seedrandom from 'seedrandom';
import {
@ -22,6 +22,8 @@ import {
ExpressionAstExpressionBuilder,
ExpressionAstFunction,
} from '@kbn/expressions-plugin/public';
import { ENABLE_ESQL } from '@kbn/esql-utils';
import { getESQLForLayer } from './to_esql';
import { convertToAbsoluteDateRange } from '../../utils';
import type { DateRange } from '../../../common/types';
import { GenericIndexPatternColumn } from './form_based';
@ -69,6 +71,7 @@ function getExpressionForLayer(
layer: FormBasedLayer,
indexPattern: IndexPattern,
uiSettings: IUiSettingsClient,
featureFlags: FeatureFlagsStart,
dateRange: DateRange,
nowInstant: Date,
searchSessionId?: string,
@ -167,175 +170,195 @@ function getExpressionForLayer(
const absDateRange = convertToAbsoluteDateRange(dateRange, nowInstant);
const aggExpressionToEsAggsIdMap: Map<ExpressionAstExpressionBuilder, string> = new Map();
esAggEntries.forEach(([colId, col], index) => {
const def = operationDefinitionMap[col.operationType];
if (def.input !== 'fullReference' && def.input !== 'managedReference') {
const aggId = String(index);
const wrapInFilter = Boolean(def.filterable && col.filter?.query);
const wrapInTimeFilter =
def.canReduceTimeRange &&
!hasDateHistogram &&
col.reducedTimeRange &&
indexPattern.timeFieldName;
let aggAst = def.toEsAggsFn(
{
...col,
timeShift: resolveTimeShift(
col.timeShift,
absDateRange,
histogramBarsTarget,
hasDateHistogram
),
},
wrapInFilter || wrapInTimeFilter ? `${aggId}-metric` : aggId,
indexPattern,
layer,
uiSettings,
orderedColumnIds,
operationDefinitionMap
);
if (wrapInFilter || wrapInTimeFilter) {
aggAst = buildExpressionFunction<AggFunctionsMapping['aggFilteredMetric']>(
'aggFilteredMetric',
// esql mode variables
const lensESQLEnabled = featureFlags.getBooleanValue('lens.enable_esql', false);
const canUseESQL = lensESQLEnabled && uiSettings.get(ENABLE_ESQL) && !forceDSL; // read from a setting
const esqlLayer =
canUseESQL &&
getESQLForLayer(esAggEntries, layer, indexPattern, uiSettings, dateRange, nowInstant);
if (!esqlLayer) {
esAggEntries.forEach(([colId, col], index) => {
const def = operationDefinitionMap[col.operationType];
if (def.input !== 'fullReference' && def.input !== 'managedReference') {
const aggId = String(index);
const wrapInFilter = Boolean(def.filterable && col.filter?.query);
const wrapInTimeFilter =
def.canReduceTimeRange &&
!hasDateHistogram &&
col.reducedTimeRange &&
indexPattern.timeFieldName;
let aggAst = def.toEsAggsFn(
{
id: String(index),
enabled: true,
schema: 'metric',
customBucket: buildExpression([
buildExpressionFunction<AggFunctionsMapping['aggFilter']>('aggFilter', {
id: `${index}-filter`,
enabled: true,
schema: 'bucket',
filter: col.filter && queryToAst(col.filter),
timeWindow: wrapInTimeFilter ? col.reducedTimeRange : undefined,
timeShift: resolveTimeShift(
col.timeShift,
absDateRange,
histogramBarsTarget,
hasDateHistogram
),
}),
]),
customMetric: buildExpression({ type: 'expression', chain: [aggAst] }),
...col,
timeShift: resolveTimeShift(
col.timeShift,
absDateRange,
histogramBarsTarget,
hasDateHistogram
),
}
).toAst();
}
const expressionBuilder = buildExpression({
type: 'expression',
chain: [aggAst],
});
aggs.push(expressionBuilder);
const esAggsId = window.ELASTIC_LENS_DELAY_SECONDS
? `col-${index + (col.isBucketed ? 0 : 1)}-${aggId}`
: `col-${index}-${aggId}`;
esAggsIdMap[esAggsId] = [
{
...col,
id: colId,
label: col.customLabel
? col.label
: operationDefinitionMap[col.operationType].getDefaultLabel(
col,
layer.columns,
indexPattern
},
wrapInFilter || wrapInTimeFilter ? `${aggId}-metric` : aggId,
indexPattern,
layer,
uiSettings,
orderedColumnIds,
operationDefinitionMap
);
if (wrapInFilter || wrapInTimeFilter) {
aggAst = buildExpressionFunction<AggFunctionsMapping['aggFilteredMetric']>(
'aggFilteredMetric',
{
id: String(index),
enabled: true,
schema: 'metric',
customBucket: buildExpression([
buildExpressionFunction<AggFunctionsMapping['aggFilter']>('aggFilter', {
id: `${index}-filter`,
enabled: true,
schema: 'bucket',
filter: col.filter && queryToAst(col.filter),
timeWindow: wrapInTimeFilter ? col.reducedTimeRange : undefined,
timeShift: resolveTimeShift(
col.timeShift,
absDateRange,
histogramBarsTarget,
hasDateHistogram
),
}),
]),
customMetric: buildExpression({ type: 'expression', chain: [aggAst] }),
timeShift: resolveTimeShift(
col.timeShift,
absDateRange,
histogramBarsTarget,
hasDateHistogram
),
},
];
}
).toAst();
}
aggExpressionToEsAggsIdMap.set(expressionBuilder, esAggsId);
}
});
const expressionBuilder = buildExpression({
type: 'expression',
chain: [aggAst],
});
aggs.push(expressionBuilder);
if (window.ELASTIC_LENS_DELAY_SECONDS) {
aggs.push(
buildExpression({
type: 'expression',
chain: [
buildExpressionFunction('aggShardDelay', {
id: 'the-delay',
enabled: true,
schema: 'metric',
delay: `${window.ELASTIC_LENS_DELAY_SECONDS}s`,
}).toAst(),
],
})
);
}
const esAggsId = window.ELASTIC_LENS_DELAY_SECONDS
? `col-${index + (col.isBucketed ? 0 : 1)}-${aggId}`
: `col-${index}-${aggId}`;
const allOperations = uniq(
esAggEntries.map(([_, column]) => operationDefinitionMap[column.operationType])
);
esAggsIdMap[esAggsId] = [
{
...col,
id: colId,
label: col.customLabel
? col.label
: operationDefinitionMap[col.operationType].getDefaultLabel(
col,
layer.columns,
indexPattern
),
},
];
// De-duplicate aggs for supported operations
const dedupedResult = dedupeAggs(aggs, esAggsIdMap, aggExpressionToEsAggsIdMap, allOperations);
aggs = dedupedResult.aggs;
esAggsIdMap = dedupedResult.esAggsIdMap;
// Apply any operation-specific custom optimizations
allOperations.forEach((operation) => {
const optimizeAggs = operation.optimizeEsAggs?.bind(operation);
if (optimizeAggs) {
const { aggs: newAggs, esAggsIdMap: newIdMap } = optimizeAggs(
aggs,
esAggsIdMap,
aggExpressionToEsAggsIdMap
);
aggs = newAggs;
esAggsIdMap = newIdMap;
}
});
/*
Update ID mappings with new agg array positions.
Given this esAggs-ID-to-original-column map after percentile (for example) optimization:
col-0-0: column1
col-?-1.34: column2 (34th percentile)
col-2-2: column3
col-?-1.98: column4 (98th percentile)
and this array of aggs
0: { id: 0 }
1: { id: 2 }
2: { id: 1 }
We need to update the anticipated agg indicies to match the aggs array:
col-0-0: column1
col-2-1.34: column2 (34th percentile)
col-1-2: column3
col-3-3.98: column4 (98th percentile)
*/
const updatedEsAggsIdMap: Record<string, OriginalColumn[]> = {};
let counter = 0;
const esAggsIds = Object.keys(esAggsIdMap);
aggs.forEach((builder) => {
const esAggId = builder.functions[0].getArgument('id')?.[0];
const matchingEsAggColumnIds = esAggsIds.filter((id) => extractAggId(id) === esAggId);
matchingEsAggColumnIds.forEach((currentId) => {
const currentColumn = esAggsIdMap[currentId][0];
const aggIndex = window.ELASTIC_LENS_DELAY_SECONDS
? counter + (currentColumn.isBucketed ? 0 : 1)
: counter;
const newId = updatePositionIndex(currentId, aggIndex);
updatedEsAggsIdMap[newId] = esAggsIdMap[currentId];
counter++;
aggExpressionToEsAggsIdMap.set(expressionBuilder, esAggsId);
}
});
});
if (window.ELASTIC_LENS_DELAY_SECONDS) {
aggs.push(
buildExpression({
type: 'expression',
chain: [
buildExpressionFunction('aggShardDelay', {
id: 'the-delay',
enabled: true,
schema: 'metric',
delay: `${window.ELASTIC_LENS_DELAY_SECONDS}s`,
}).toAst(),
],
})
);
}
const allOperations = uniq(
esAggEntries.map(([_, column]) => operationDefinitionMap[column.operationType])
);
// De-duplicate aggs for supported operations
const dedupedResult = dedupeAggs(
aggs,
esAggsIdMap,
aggExpressionToEsAggsIdMap,
allOperations
);
aggs = dedupedResult.aggs;
const updatedEsAggsIdMap: Record<string, OriginalColumn[]> = {};
esAggsIdMap = dedupedResult.esAggsIdMap;
// Apply any operation-specific custom optimizations
allOperations.forEach((operation) => {
const optimizeAggs = operation.optimizeEsAggs?.bind(operation);
if (optimizeAggs) {
const { aggs: newAggs, esAggsIdMap: newIdMap } = optimizeAggs(
aggs,
esAggsIdMap,
aggExpressionToEsAggsIdMap
);
aggs = newAggs;
esAggsIdMap = newIdMap;
}
});
/*
Update ID mappings with new agg array positions.
Given this esAggs-ID-to-original-column map after percentile (for example) optimization:
col-0-0: column1
col-?-1.34: column2 (34th percentile)
col-2-2: column3
col-?-1.98: column4 (98th percentile)
and this array of aggs
0: { id: 0 }
1: { id: 2 }
2: { id: 1 }
We need to update the anticipated agg indicies to match the aggs array:
col-0-0: column1
col-2-1.34: column2 (34th percentile)
col-1-2: column3
col-3-3.98: column4 (98th percentile)
*/
let counter = 0;
const esAggsIds = Object.keys(esAggsIdMap);
aggs.forEach((builder) => {
const esAggId = builder.functions[0].getArgument('id')?.[0];
const matchingEsAggColumnIds = esAggsIds.filter((id) => extractAggId(id) === esAggId);
matchingEsAggColumnIds.forEach((currentId) => {
const currentColumn = esAggsIdMap[currentId][0];
const aggIndex = window.ELASTIC_LENS_DELAY_SECONDS
? counter + (currentColumn.isBucketed ? 0 : 1)
: counter;
const newId = updatePositionIndex(currentId, aggIndex);
updatedEsAggsIdMap[newId] = esAggsIdMap[currentId];
counter++;
});
});
esAggsIdMap = updatedEsAggsIdMap;
} else {
esAggsIdMap = esqlLayer.esAggsIdMap;
}
const columnsWithFormatters = columnEntries.filter(
([, col]) =>
@ -454,11 +477,13 @@ function getExpressionForLayer(
)
.filter((field): field is string => Boolean(field));
return {
type: 'expression',
chain: [
{ type: 'function', function: 'kibana', arguments: {} },
buildExpressionFunction<EsaggsExpressionFunctionDefinition>('esaggs', {
const dataAST = esqlLayer
? buildExpressionFunction('esql', {
query: esqlLayer.esql,
timeField: allDateHistogramFields[0],
ignoreGlobalFilters: Boolean(layer.ignoreGlobalFilters),
}).toAst()
: buildExpressionFunction<EsaggsExpressionFunctionDefinition>('esaggs', {
index: buildExpression([
buildExpressionFunction<IndexPatternLoadExpressionFunctionDefinition>(
'indexPatternLoad',
@ -472,12 +497,19 @@ function getExpressionForLayer(
probability: getSamplingValue(layer),
samplerSeed: seedrandom(searchSessionId).int32(),
ignoreGlobalFilters: Boolean(layer.ignoreGlobalFilters),
}).toAst(),
}).toAst();
return {
type: 'expression',
chain: [
{ type: 'function', function: 'kibana', arguments: {} },
dataAST,
{
type: 'function',
function: 'lens_map_to_columns',
arguments: {
idMap: [JSON.stringify(updatedEsAggsIdMap)],
idMap: [JSON.stringify(esAggsIdMap)],
isTextBased: [!!esqlLayer],
},
},
...expressions,
@ -522,6 +554,7 @@ export function toExpression(
layerId: string,
indexPatterns: IndexPatternMap,
uiSettings: IUiSettingsClient,
featureFlags: FeatureFlagsStart,
dateRange: DateRange,
nowInstant: Date,
searchSessionId?: string,
@ -532,6 +565,7 @@ export function toExpression(
state.layers[layerId],
indexPatterns[state.layers[layerId].indexPatternId],
uiSettings,
featureFlags,
dateRange,
nowInstant,
searchSessionId,