github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Lens] optimize percentiles fetching (#131875)

Browse source
This commit is contained in:
Andrew Tate 2022-06-21 10:13:54 -05:00 • committed by GitHub
parent 61be8fbe1e
commit ebe331eebb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 957 additions and 114 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/lens/common/expressions/index.ts
View file

@ -8,6 +8,6 @@
export * from './counter_rate';
export * from './collapse';
export * from './format_column';
export * from './rename_columns';
export * from './map_to_columns';
export * from './time_scale';
export * from './datatable';

2
x-pack/plugins/lens/common/expressions/rename_columns/index.ts → x-pack/plugins/lens/common/expressions/map_to_columns/index.ts
View file

@ -5,4 +5,4 @@
* 2.0.
*/
export { renameColumns } from './rename_columns';
export { mapToColumns } from './map_to_columns';

95
x-pack/plugins/lens/common/expressions/rename_columns/rename_columns.test.ts → x-pack/plugins/lens/common/expressions/map_to_columns/map_to_columns.test.ts
View file

@ -5,11 +5,11 @@
* 2.0.
*/
import { renameColumns } from './rename_columns';
import { mapToColumns } from './map_to_columns';
import { Datatable } from '@kbn/expressions-plugin/common';
import { createMockExecutionContext } from '@kbn/expressions-plugin/common/mocks';
describe('rename_columns', () => {
describe('map_to_columns', () => {
it('should rename columns of a given datatable', async () => {
const input: Datatable = {
type: 'datatable',
@ -26,17 +26,21 @@ describe('rename_columns', () => {
};
const idMap = {
a: {
id: 'b',
label: 'Austrailia',
},
b: {
id: 'c',
label: 'Boomerang',
},
a: [
{
id: 'b',
label: 'Austrailia',
},
],
b: [
{
id: 'c',
label: 'Boomerang',
},
],
};
const result = await renameColumns.fn(
const result = await mapToColumns.fn(
input,
{ idMap: JSON.stringify(idMap) },
createMockExecutionContext()
@ -99,10 +103,10 @@ describe('rename_columns', () => {
};
const idMap = {
b: { id: 'c', label: 'Catamaran' },
b: [{ id: 'c', label: 'Catamaran' }],
};
const result = await renameColumns.fn(
const result = await mapToColumns.fn(
input,
{ idMap: JSON.stringify(idMap) },
createMockExecutionContext()
@ -149,6 +153,67 @@ describe('rename_columns', () => {
`);
});
it('should map to multiple original columns', async () => {
const input: Datatable = {
type: 'datatable',
columns: [{ id: 'b', name: 'B', meta: { type: 'number' } }],
rows: [{ b: 2 }, { b: 4 }, { b: 6 }, { b: 8 }],
};
const idMap = {
b: [
{ id: 'c', label: 'Catamaran' },
{ id: 'd', label: 'Dinghy' },
],
};
const result = await mapToColumns.fn(
input,
{ idMap: JSON.stringify(idMap) },
createMockExecutionContext()
);
expect(result).toMatchInlineSnapshot(`
Object {
"columns": Array [
Object {
"id": "c",
"meta": Object {
"type": "number",
},
"name": "Catamaran",
},
Object {
"id": "d",
"meta": Object {
"type": "number",
},
"name": "Dinghy",
},
],
"rows": Array [
Object {
"c": 2,
"d": 2,
},
Object {
"c": 4,
"d": 4,
},
Object {
"c": 6,
"d": 6,
},
Object {
"c": 8,
"d": 8,
},
],
"type": "datatable",
}
`);
});
it('should rename date histograms', async () => {
const input: Datatable = {
type: 'datatable',
@ -165,10 +230,10 @@ describe('rename_columns', () => {
};
const idMap = {
b: { id: 'c', label: 'Apple', operationType: 'date_histogram', sourceField: 'banana' },
b: [{ id: 'c', label: 'Apple', operationType: 'date_histogram', sourceField: 'banana' }],
};
const result = await renameColumns.fn(
const result = await mapToColumns.fn(
input,
{ idMap: JSON.stringify(idMap) },
createMockExecutionContext()

32
x-pack/plugins/lens/common/expressions/map_to_columns/map_to_columns.ts Normal file
View file

@ -0,0 +1,32 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import type { MapToColumnsExpressionFunction } from './types';
export const mapToColumns: MapToColumnsExpressionFunction = {
name: 'lens_map_to_columns',
type: 'datatable',
help: i18n.translate('xpack.lens.functions.mapToColumns.help', {
defaultMessage: 'A helper to transform a datatable to match Lens column definitions',
}),
args: {
idMap: {
types: ['string'],
help: i18n.translate('xpack.lens.functions.mapToColumns.idMap.help', {
defaultMessage:
'A JSON encoded object in which keys are the datatable column ids and values are the Lens column definitions. Any datatable columns not mentioned within the ID map will be kept unmapped.',
}),
},
},
inputTypes: ['datatable'],
async fn(...args) {
/** Build optimization: prevent adding extra code into initial bundle **/
const { mapToOriginalColumns } = await import('./map_to_columns_fn');
return mapToOriginalColumns(...args);
},
};

37
x-pack/plugins/lens/common/expressions/rename_columns/rename_columns_fn.ts → x-pack/plugins/lens/common/expressions/map_to_columns/map_to_columns_fn.ts
View file

@ -6,7 +6,7 @@
*/
import type { DatatableColumn } from '@kbn/expressions-plugin/common';
import type { OriginalColumn, RenameColumnsExpressionFunction } from './types';
import type { OriginalColumn, MapToColumnsExpressionFunction } from './types';
function getColumnName(originalColumn: OriginalColumn, newColumn: DatatableColumn) {
if (originalColumn?.operationType === 'date_histogram') {
@ -21,23 +21,22 @@ function getColumnName(originalColumn: OriginalColumn, newColumn: DatatableColum
return originalColumn.label;
}
export const renameColumnFn: RenameColumnsExpressionFunction['fn'] = (
export const mapToOriginalColumns: MapToColumnsExpressionFunction['fn'] = (
data,
{ idMap: encodedIdMap }
) => {
const idMap = JSON.parse(encodedIdMap) as Record<string, OriginalColumn>;
const idMap = JSON.parse(encodedIdMap) as Record<string, OriginalColumn[]>;
return {
...data,
rows: data.rows.map((row) => {
const mappedRow: Record<string, unknown> = {};
Object.entries(idMap).forEach(([fromId, toId]) => {
mappedRow[toId.id] = row[fromId];
});
Object.entries(row).forEach(([id, value]) => {
if (id in idMap) {
mappedRow[idMap[id].id] = value;
idMap[id].forEach(({ id: originalId }) => {
mappedRow[originalId] = value;
});
} else {
mappedRow[id] = value;
}
@ -45,18 +44,20 @@ export const renameColumnFn: RenameColumnsExpressionFunction['fn'] = (
return mappedRow;
}),
columns: data.columns.map((column) => {
const mappedItem = idMap[column.id];
columns: data.columns
.map((column) => {
const originalColumns = idMap[column.id];
if (!mappedItem) {
return column;
}
if (!originalColumns) {
return column;
}
return {
...column,
id: mappedItem.id,
name: getColumnName(mappedItem, column),
};
}),
return originalColumns.map((originalColumn) => ({
...column,
id: originalColumn.id,
name: getColumnName(originalColumn, column),
}));
})
.flat(),
};
};

4
x-pack/plugins/lens/common/expressions/rename_columns/types.ts → x-pack/plugins/lens/common/expressions/map_to_columns/types.ts
View file

@ -12,8 +12,8 @@ export type OriginalColumn = { id: string; label: string } & (
| { operationType: string; sourceField: never }
);
export type RenameColumnsExpressionFunction = ExpressionFunctionDefinition<
'lens_rename_columns',
export type MapToColumnsExpressionFunction = ExpressionFunctionDefinition<
'lens_map_to_columns',
Datatable,
{
idMap: string;

32
x-pack/plugins/lens/common/expressions/rename_columns/rename_columns.ts
View file

@ -1,32 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import type { RenameColumnsExpressionFunction } from './types';
export const renameColumns: RenameColumnsExpressionFunction = {
name: 'lens_rename_columns',
type: 'datatable',
help: i18n.translate('xpack.lens.functions.renameColumns.help', {
defaultMessage: 'A helper to rename the columns of a datatable',
}),
args: {
idMap: {
types: ['string'],
help: i18n.translate('xpack.lens.functions.renameColumns.idMap.help', {
defaultMessage:
'A JSON encoded object in which keys are the old column ids and values are the corresponding new ones. All other columns ids are kept.',
}),
},
},
inputTypes: ['datatable'],
async fn(...args) {
/** Build optimization: prevent adding extra code into initial bundle **/
const { renameColumnFn } = await import('./rename_columns_fn');
return renameColumnFn(...args);
},
};

4
x-pack/plugins/lens/public/expressions.ts
View file

@ -8,7 +8,7 @@
import type { ExpressionsSetup } from '@kbn/expressions-plugin/public';
import { getDatatable } from '../common/expressions/datatable/datatable';
import { datatableColumn } from '../common/expressions/datatable/datatable_column';
import { renameColumns } from '../common/expressions/rename_columns/rename_columns';
import { mapToColumns } from '../common/expressions/map_to_columns/map_to_columns';
import { formatColumn } from '../common/expressions/format_column';
import { counterRate } from '../common/expressions/counter_rate';
import { getTimeScale } from '../common/expressions/time_scale/time_scale';
@ -24,7 +24,7 @@ export const setupExpressions = (
collapse,
counterRate,
formatColumn,
renameColumns,
mapToColumns,
datatableColumn,
getDatatable(formatFactory),
getTimeScale(getDatatableUtilities, getTimeZone),

154
x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts
View file

@ -33,6 +33,7 @@ import {
FormulaIndexPatternColumn,
RangeIndexPatternColumn,
FiltersIndexPatternColumn,
PercentileIndexPatternColumn,
} from './operations';
import { createMockedFullReference } from './operations/mocks';
import { cloneDeep } from 'lodash';
@ -491,10 +492,10 @@ describe('IndexPattern Data Source', () => {
Object {
"arguments": Object {
"idMap": Array [
"{\\"col-0-0\\":{\\"label\\":\\"Count of records\\",\\"dataType\\":\\"number\\",\\"isBucketed\\":false,\\"sourceField\\":\\"___records___\\",\\"operationType\\":\\"count\\",\\"id\\":\\"col1\\"},\\"col-1-1\\":{\\"label\\":\\"Date\\",\\"dataType\\":\\"date\\",\\"isBucketed\\":true,\\"operationType\\":\\"date_histogram\\",\\"sourceField\\":\\"timestamp\\",\\"params\\":{\\"interval\\":\\"1d\\"},\\"id\\":\\"col2\\"}}",
"{\\"col-0-0\\":[{\\"label\\":\\"Count of records\\",\\"dataType\\":\\"number\\",\\"isBucketed\\":false,\\"sourceField\\":\\"___records___\\",\\"operationType\\":\\"count\\",\\"id\\":\\"col1\\"}],\\"col-1-1\\":[{\\"label\\":\\"Date\\",\\"dataType\\":\\"date\\",\\"isBucketed\\":true,\\"operationType\\":\\"date_histogram\\",\\"sourceField\\":\\"timestamp\\",\\"params\\":{\\"interval\\":\\"1d\\"},\\"id\\":\\"col2\\"}]}",
],
},
"function": "lens_rename_columns",
"function": "lens_map_to_columns",
"type": "function",
},
],
@ -905,9 +906,9 @@ describe('IndexPattern Data Source', () => {
const ast = indexPatternDatasource.toExpression(state, 'first') as Ast;
expect(ast.chain[1].arguments.metricsAtAllLevels).toEqual([false]);
expect(JSON.parse(ast.chain[2].arguments.idMap[0] as string)).toEqual({
'col-0-0': expect.objectContaining({ id: 'bucket1' }),
'col-1-1': expect.objectContaining({ id: 'bucket2' }),
'col-2-2': expect.objectContaining({ id: 'metric' }),
'col-0-0': [expect.objectContaining({ id: 'bucket1' })],
'col-1-1': [expect.objectContaining({ id: 'bucket2' })],
'col-2-2': [expect.objectContaining({ id: 'metric' })],
});
});
@ -948,6 +949,140 @@ describe('IndexPattern Data Source', () => {
expect(ast.chain[1].arguments.timeFields).not.toContain('timefield');
});
it('should call optimizeEsAggs once per operation for which it is available', () => {
const queryBaseState: DataViewBaseState = {
currentIndexPatternId: '1',
layers: {
first: {
indexPatternId: '1',
columns: {
col1: {
label: 'timestamp',
dataType: 'date',
operationType: 'date_histogram',
sourceField: 'timestamp',
isBucketed: true,
scale: 'interval',
params: {
interval: 'auto',
includeEmptyRows: true,
dropPartials: false,
},
} as DateHistogramIndexPatternColumn,
col2: {
label: '95th percentile of bytes',
dataType: 'number',
operationType: 'percentile',
sourceField: 'bytes',
isBucketed: false,
scale: 'ratio',
params: {
percentile: 95,
},
} as PercentileIndexPatternColumn,
col3: {
label: '95th percentile of bytes',
dataType: 'number',
operationType: 'percentile',
sourceField: 'bytes',
isBucketed: false,
scale: 'ratio',
params: {
percentile: 95,
},
} as PercentileIndexPatternColumn,
},
columnOrder: ['col1', 'col2', 'col3'],
incompleteColumns: {},
},
},
};
const state = enrichBaseState(queryBaseState);
const optimizeMock = jest.spyOn(operationDefinitionMap.percentile, 'optimizeEsAggs');
indexPatternDatasource.toExpression(state, 'first');
expect(operationDefinitionMap.percentile.optimizeEsAggs).toHaveBeenCalledTimes(1);
optimizeMock.mockRestore();
});
it('should update anticipated esAggs column IDs based on the order of the optimized agg expression builders', () => {
const queryBaseState: DataViewBaseState = {
currentIndexPatternId: '1',
layers: {
first: {
indexPatternId: '1',
columns: {
col1: {
label: 'timestamp',
dataType: 'date',
operationType: 'date_histogram',
sourceField: 'timestamp',
isBucketed: true,
scale: 'interval',
params: {
interval: 'auto',
includeEmptyRows: true,
dropPartials: false,
},
} as DateHistogramIndexPatternColumn,
col2: {
label: '95th percentile of bytes',
dataType: 'number',
operationType: 'percentile',
sourceField: 'bytes',
isBucketed: false,
scale: 'ratio',
params: {
percentile: 95,
},
} as PercentileIndexPatternColumn,
col3: {
label: 'Count of records',
dataType: 'number',
isBucketed: false,
sourceField: '___records___',
operationType: 'count',
timeScale: 'h',
},
col4: {
label: 'Count of records2',
dataType: 'number',
isBucketed: false,
sourceField: '___records___',
operationType: 'count',
timeScale: 'h',
},
},
columnOrder: ['col1', 'col2', 'col3', 'col4'],
incompleteColumns: {},
},
},
};
const state = enrichBaseState(queryBaseState);
const optimizeMock = jest
.spyOn(operationDefinitionMap.percentile, 'optimizeEsAggs')
.mockImplementation((aggs, esAggsIdMap) => {
// change the order of the aggregations
return { aggs: aggs.reverse(), esAggsIdMap };
});
const ast = indexPatternDatasource.toExpression(state, 'first') as Ast;
expect(operationDefinitionMap.percentile.optimizeEsAggs).toHaveBeenCalledTimes(1);
const idMap = JSON.parse(ast.chain[2].arguments.idMap as unknown as string);
expect(Object.keys(idMap)).toEqual(['col-0-3', 'col-1-2', 'col-2-1', 'col-3-0']);
optimizeMock.mockRestore();
});
describe('references', () => {
beforeEach(() => {
// @ts-expect-error we are inserting an invalid type
@ -1026,10 +1161,13 @@ describe('IndexPattern Data Source', () => {
const state = enrichBaseState(queryBaseState);
const ast = indexPatternDatasource.toExpression(state, 'first') as Ast;
expect(JSON.parse(ast.chain[2].arguments.idMap[0] as string)).toEqual({
'col-0-0': expect.objectContaining({
id: 'col1',
}),
'col-0-0': [
expect.objectContaining({
id: 'col1',
}),
],
});
});

17
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts
View file

@ -12,7 +12,10 @@ import {
CoreStart,
} from '@kbn/core/public';
import { IStorageWrapper } from '@kbn/kibana-utils-plugin/public';
import { ExpressionAstFunction } from '@kbn/expressions-plugin/public';
import {
ExpressionAstExpressionBuilder,
ExpressionAstFunction,
} from '@kbn/expressions-plugin/public';
import { DataPublicPluginStart } from '@kbn/data-plugin/public';
import { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public';
import { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
@ -55,6 +58,7 @@ import { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../types'
import { DateRange, LayerType } from '../../../../common';
import { rangeOperation } from './ranges';
import { IndexPatternDimensionEditorProps, OperationSupportMatrix } from '../../dimension_panel';
import type { OriginalColumn } from '../../to_expression';
export type {
IncompleteColumn,
@ -378,6 +382,17 @@ interface BaseOperationDefinitionProps<C extends BaseIndexPatternColumn, P = {}>
* Title for the help component
*/
helpComponentTitle?: string;
/**
* Optimizes EsAggs expression. Invoked only once per operation type.
*/
optimizeEsAggs?: (
aggs: ExpressionAstExpressionBuilder[],
esAggsIdMap: Record<string, OriginalColumn[]>,
aggExpressionToEsAggsIdMap: Map<ExpressionAstExpressionBuilder, string>
) => {
aggs: ExpressionAstExpressionBuilder[];
esAggsIdMap: Record<string, OriginalColumn[]>;
};
}
interface BaseBuildColumnArgs {

441
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx
View file

@ -20,6 +20,12 @@ import { percentileOperation } from '.';
import { IndexPattern, IndexPatternLayer } from '../../types';
import { PercentileIndexPatternColumn } from './percentile';
import { TermsIndexPatternColumn } from './terms';
import {
buildExpressionFunction,
buildExpression,
ExpressionAstExpressionBuilder,
} from '@kbn/expressions-plugin/public';
import type { OriginalColumn } from '../../to_expression';
jest.mock('lodash', () => {
const original = jest.requireActual('lodash');
@ -187,6 +193,441 @@ describe('percentile', () => {
});
});
describe('optimizeEsAggs', () => {
const makeEsAggBuilder = (name: string, params: object) =>
buildExpression({
type: 'expression',
chain: [buildExpressionFunction(name, params).toAst()],
});
const buildMapsFromAggBuilders = (aggs: ExpressionAstExpressionBuilder[]) => {
const esAggsIdMap: Record<string, OriginalColumn[]> = {};
const aggsToIdsMap = new Map();
aggs.forEach((builder, i) => {
const esAggsId = `col-${i}-${i}`;
esAggsIdMap[esAggsId] = [{ id: `original-${i}` } as OriginalColumn];
aggsToIdsMap.set(builder, esAggsId);
});
return {
esAggsIdMap,
aggsToIdsMap,
};
};
it('should collapse percentile dimensions with matching parameters', () => {
const field1 = 'foo';
const field2 = 'bar';
const timeShift1 = '1d';
const timeShift2 = '2d';
const aggs = [
// group 1
makeEsAggBuilder('aggSinglePercentile', {
id: 1,
enabled: true,
schema: 'metric',
field: field1,
percentile: 10,
timeShift: undefined,
}),
makeEsAggBuilder('aggSinglePercentile', {
id: 2,
enabled: true,
schema: 'metric',
field: field1,
percentile: 20,
timeShift: undefined,
}),
makeEsAggBuilder('aggSinglePercentile', {
id: 3,
enabled: true,
schema: 'metric',
field: field1,
percentile: 30,
timeShift: undefined,
}),
// group 2
makeEsAggBuilder('aggSinglePercentile', {
id: 4,
enabled: true,
schema: 'metric',
field: field2,
percentile: 10,
timeShift: undefined,
}),
makeEsAggBuilder('aggSinglePercentile', {
id: 5,
enabled: true,
schema: 'metric',
field: field2,
percentile: 40,
timeShift: undefined,
}),
// group 3
makeEsAggBuilder('aggSinglePercentile', {
id: 6,
enabled: true,
schema: 'metric',
field: field2,
percentile: 50,
timeShift: timeShift1,
}),
makeEsAggBuilder('aggSinglePercentile', {
id: 7,
enabled: true,
schema: 'metric',
field: field2,
percentile: 60,
timeShift: timeShift1,
}),
// group 4
makeEsAggBuilder('aggSinglePercentile', {
id: 8,
enabled: true,
schema: 'metric',
field: field2,
percentile: 70,
timeShift: timeShift2,
}),
makeEsAggBuilder('aggSinglePercentile', {
id: 9,
enabled: true,
schema: 'metric',
field: field2,
percentile: 80,
timeShift: timeShift2,
}),
];
const { esAggsIdMap, aggsToIdsMap } = buildMapsFromAggBuilders(aggs);
const { esAggsIdMap: newIdMap, aggs: newAggs } = percentileOperation.optimizeEsAggs!(
aggs,
esAggsIdMap,
aggsToIdsMap
);
expect(newAggs.length).toBe(4);
expect(newAggs[0].functions[0].getArgument('field')![0]).toBe(field1);
expect(newAggs[0].functions[0].getArgument('timeShift')).toBeUndefined();
expect(newAggs[1].functions[0].getArgument('field')![0]).toBe(field2);
expect(newAggs[1].functions[0].getArgument('timeShift')).toBeUndefined();
expect(newAggs[2].functions[0].getArgument('field')![0]).toBe(field2);
expect(newAggs[2].functions[0].getArgument('timeShift')![0]).toBe(timeShift1);
expect(newAggs[3].functions[0].getArgument('field')![0]).toBe(field2);
expect(newAggs[3].functions[0].getArgument('timeShift')![0]).toBe(timeShift2);
expect(newAggs).toMatchInlineSnapshot(`
Array [
Object {
"findFunction": [Function],
"functions": Array [
Object {
"addArgument": [Function],
"arguments": Object {
"enabled": Array [
true,
],
"field": Array [
"foo",
],
"id": Array [
1,
],
"percents": Array [
10,
20,
30,
],
"schema": Array [
"metric",
],
},
"getArgument": [Function],
"name": "aggPercentiles",
"removeArgument": [Function],
"replaceArgument": [Function],
"toAst": [Function],
"toString": [Function],
"type": "expression_function_builder",
},
],
"toAst": [Function],
"toString": [Function],
"type": "expression_builder",
},
Object {
"findFunction": [Function],
"functions": Array [
Object {
"addArgument": [Function],
"arguments": Object {
"enabled": Array [
true,
],
"field": Array [
"bar",
],
"id": Array [
4,
],
"percents": Array [
10,
40,
],
"schema": Array [
"metric",
],
},
"getArgument": [Function],
"name": "aggPercentiles",
"removeArgument": [Function],
"replaceArgument": [Function],
"toAst": [Function],
"toString": [Function],
"type": "expression_function_builder",
},
],
"toAst": [Function],
"toString": [Function],
"type": "expression_builder",
},
Object {
"findFunction": [Function],
"functions": Array [
Object {
"addArgument": [Function],
"arguments": Object {
"enabled": Array [
true,
],
"field": Array [
"bar",
],
"id": Array [
6,
],
"percents": Array [
50,
60,
],
"schema": Array [
"metric",
],
"timeShift": Array [
"1d",
],
},
"getArgument": [Function],
"name": "aggPercentiles",
"removeArgument": [Function],
"replaceArgument": [Function],
"toAst": [Function],
"toString": [Function],
"type": "expression_function_builder",
},
],
"toAst": [Function],
"toString": [Function],
"type": "expression_builder",
},
Object {
"findFunction": [Function],
"functions": Array [
Object {
"addArgument": [Function],
"arguments": Object {
"enabled": Array [
true,
],
"field": Array [
"bar",
],
"id": Array [
8,
],
"percents": Array [
70,
80,
],
"schema": Array [
"metric",
],
"timeShift": Array [
"2d",
],
},
"getArgument": [Function],
"name": "aggPercentiles",
"removeArgument": [Function],
"replaceArgument": [Function],
"toAst": [Function],
"toString": [Function],
"type": "expression_function_builder",
},
],
"toAst": [Function],
"toString": [Function],
"type": "expression_builder",
},
]
`);
expect(newIdMap).toMatchInlineSnapshot(`
Object {
"col-?-1.10": Array [
Object {
"id": "original-0",
},
],
"col-?-1.20": Array [
Object {
"id": "original-1",
},
],
"col-?-1.30": Array [
Object {
"id": "original-2",
},
],
"col-?-4.10": Array [
Object {
"id": "original-3",
},
],
"col-?-4.40": Array [
Object {
"id": "original-4",
},
],
"col-?-6.50": Array [
Object {
"id": "original-5",
},
],
"col-?-6.60": Array [
Object {
"id": "original-6",
},
],
"col-?-8.70": Array [
Object {
"id": "original-7",
},
],
"col-?-8.80": Array [
Object {
"id": "original-8",
},
],
}
`);
});
it('should handle multiple identical percentiles', () => {
const field1 = 'foo';
const field2 = 'bar';
const samePercentile = 90;
const aggs = [
// group 1
makeEsAggBuilder('aggSinglePercentile', {
id: 1,
enabled: true,
schema: 'metric',
field: field1,
percentile: samePercentile,
timeShift: undefined,
}),
makeEsAggBuilder('aggSinglePercentile', {
id: 2,
enabled: true,
schema: 'metric',
field: field1,
percentile: samePercentile,
timeShift: undefined,
}),
makeEsAggBuilder('aggSinglePercentile', {
id: 4,
enabled: true,
schema: 'metric',
field: field2,
percentile: 10,
timeShift: undefined,
}),
makeEsAggBuilder('aggSinglePercentile', {
id: 3,
enabled: true,
schema: 'metric',
field: field1,
percentile: samePercentile,
timeShift: undefined,
}),
];
const { esAggsIdMap, aggsToIdsMap } = buildMapsFromAggBuilders(aggs);
const { esAggsIdMap: newIdMap, aggs: newAggs } = percentileOperation.optimizeEsAggs!(
aggs,
esAggsIdMap,
aggsToIdsMap
);
expect(newAggs.length).toBe(2);
expect(newIdMap[`col-?-1.${samePercentile}`].length).toBe(3);
expect(newIdMap).toMatchInlineSnapshot(`
Object {
"col-2-2": Array [
Object {
"id": "original-2",
},
],
"col-?-1.90": Array [
Object {
"id": "original-0",
},
Object {
"id": "original-1",
},
Object {
"id": "original-3",
},
],
}
`);
});
it("shouldn't touch non-percentile aggs or single percentiles with no siblings", () => {
const aggs = [
makeEsAggBuilder('aggSinglePercentile', {
id: 1,
enabled: true,
schema: 'metric',
field: 'foo',
percentile: 30,
}),
makeEsAggBuilder('aggMax', {
id: 1,
enabled: true,
schema: 'metric',
field: 'bar',
}),
];
const { esAggsIdMap, aggsToIdsMap } = buildMapsFromAggBuilders(aggs);
const { esAggsIdMap: newIdMap, aggs: newAggs } = percentileOperation.optimizeEsAggs!(
aggs,
esAggsIdMap,
aggsToIdsMap
);
expect(newAggs).toEqual(aggs);
expect(newIdMap).toEqual(esAggsIdMap);
});
});
describe('buildColumn', () => {
it('should set default percentile', () => {
const indexPattern = createMockedIndexPattern();

123
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx
View file

@ -8,8 +8,13 @@
import { EuiFormRow, EuiRange, EuiRangeProps } from '@elastic/eui';
import React, { useCallback } from 'react';
import { i18n } from '@kbn/i18n';
import { AggFunctionsMapping } from '@kbn/data-plugin/public';
import { buildExpressionFunction } from '@kbn/expressions-plugin/public';
import { AggFunctionsMapping, METRIC_TYPES } from '@kbn/data-plugin/public';
import {
buildExpression,
buildExpressionFunction,
ExpressionAstExpressionBuilder,
} from '@kbn/expressions-plugin/public';
import { AggExpressionFunctionArgs } from '@kbn/data-plugin/common';
import { OperationDefinition } from '.';
import {
getFormatFromPreviousColumn,
@ -143,6 +148,120 @@ export const percentileOperation: OperationDefinition<
}
).toAst();
},
optimizeEsAggs: (_aggs, _esAggsIdMap, aggExpressionToEsAggsIdMap) => {
let aggs = [..._aggs];
const esAggsIdMap = { ..._esAggsIdMap };
const percentileExpressionsByArgs: Record<string, ExpressionAstExpressionBuilder[]> = {};
// group percentile dimensions by differentiating parameters
aggs.forEach((expressionBuilder) => {
const {
functions: [fnBuilder],
} = expressionBuilder;
if (fnBuilder.name === 'aggSinglePercentile') {
const groupByKey = `${fnBuilder.getArgument('field')?.[0]}-${
fnBuilder.getArgument('timeShift')?.[0]
}`;
if (!(groupByKey in percentileExpressionsByArgs)) {
percentileExpressionsByArgs[groupByKey] = [];
}
percentileExpressionsByArgs[groupByKey].push(expressionBuilder);
}
});
// collapse them into a single esAggs expression builder
Object.values(percentileExpressionsByArgs).forEach((expressionBuilders) => {
if (expressionBuilders.length <= 1) {
// don't need to optimize if there aren't more than one
return;
}
// we're going to merge these percentile builders into a single builder, so
// remove them from the aggs array
aggs = aggs.filter((aggBuilder) => !expressionBuilders.includes(aggBuilder));
const {
functions: [firstFnBuilder],
} = expressionBuilders[0];
const esAggsColumnId = firstFnBuilder.getArgument('id')![0];
const aggPercentilesConfig: AggExpressionFunctionArgs<typeof METRIC_TYPES.PERCENTILES> = {
id: esAggsColumnId,
enabled: firstFnBuilder.getArgument('enabled')?.[0],
schema: firstFnBuilder.getArgument('schema')?.[0],
field: firstFnBuilder.getArgument('field')?.[0],
percents: [],
// time shift is added to wrapping aggFilteredMetric if filter is set
timeShift: firstFnBuilder.getArgument('timeShift')?.[0],
};
const percentileToBuilder: Record<number, ExpressionAstExpressionBuilder> = {};
for (const builder of expressionBuilders) {
const percentile = builder.functions[0].getArgument('percentile')![0] as number;
if (percentile in percentileToBuilder) {
// found a duplicate percentile so let's optimize
const duplicateExpressionBuilder = percentileToBuilder[percentile];
const idForDuplicate = aggExpressionToEsAggsIdMap.get(duplicateExpressionBuilder);
const idForThisOne = aggExpressionToEsAggsIdMap.get(builder);
if (!idForDuplicate || !idForThisOne) {
throw new Error(
"Couldn't find esAggs ID for percentile expression builder... this should never happen."
);
}
esAggsIdMap[idForDuplicate].push(...esAggsIdMap[idForThisOne]);
delete esAggsIdMap[idForThisOne];
// remove current builder
expressionBuilders = expressionBuilders.filter((b) => b !== builder);
} else {
percentileToBuilder[percentile] = builder;
aggPercentilesConfig.percents!.push(percentile);
}
}
const multiPercentilesAst = buildExpressionFunction<AggFunctionsMapping['aggPercentiles']>(
'aggPercentiles',
aggPercentilesConfig
).toAst();
aggs.push(
buildExpression({
type: 'expression',
chain: [multiPercentilesAst],
})
);
expressionBuilders.forEach((expressionBuilder) => {
const currentEsAggsId = aggExpressionToEsAggsIdMap.get(expressionBuilder);
if (currentEsAggsId === undefined) {
throw new Error('Could not find current column ID for percentile agg expression builder');
}
// esAggs appends the percent number to the agg id to make distinct column IDs in the resulting datatable.
// We're anticipating that here by adding the `.<percentile>`.
// The agg index will be assigned when we update all the indices in the ID map based on the agg order in the
// datasource's toExpression fn so we mark it as '?' for now.
const newEsAggsId = `col-?-${esAggsColumnId}.${
expressionBuilder.functions[0].getArgument('percentile')![0]
}`;
esAggsIdMap[newEsAggsId] = esAggsIdMap[currentEsAggsId];
delete esAggsIdMap[currentEsAggsId];
});
});
return {
esAggsIdMap,
aggs,
};
},
getErrorMessage: (layer, columnId, indexPattern) =>
combineErrorMessages([
getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern),

118
x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts
View file

@ -6,7 +6,7 @@
*/
import type { IUiSettingsClient } from '@kbn/core/public';
import { partition } from 'lodash';
import { partition, uniq } from 'lodash';
import {
AggFunctionsMapping,
EsaggsExpressionFunctionDefinition,
@ -27,7 +27,7 @@ import { DateHistogramIndexPatternColumn, RangeIndexPatternColumn } from './oper
import { FormattedIndexPatternColumn } from './operations/definitions/column_types';
import { isColumnFormatted, isColumnOfType } from './operations/definitions/helpers';
type OriginalColumn = { id: string } & GenericIndexPatternColumn;
export type OriginalColumn = { id: string } & GenericIndexPatternColumn;
declare global {
interface Window {
@ -38,6 +38,15 @@ declare global {
}
}
// esAggs column ID manipulation functions
const extractEsAggId = (id: string) => id.split('.')[0].split('-')[2];
const updatePositionIndex = (currentId: string, newIndex: number) => {
const [fullId, percentile] = currentId.split('.');
const idParts = fullId.split('-');
idParts[1] = String(newIndex);
return idParts.join('-') + (percentile ? `.${percentile}` : '');
};
function getExpressionForLayer(
layer: IndexPatternLayer,
indexPattern: IndexPattern,
@ -95,7 +104,7 @@ function getExpressionForLayer(
);
if (referenceEntries.length || esAggEntries.length) {
const aggs: ExpressionAstExpressionBuilder[] = [];
let aggs: ExpressionAstExpressionBuilder[] = [];
const expressions: ExpressionAstFunction[] = [];
sortedReferences(referenceEntries).forEach((colId) => {
@ -107,13 +116,17 @@ function getExpressionForLayer(
});
const orderedColumnIds = esAggEntries.map(([colId]) => colId);
let esAggsIdMap: Record<string, OriginalColumn[]> = {};
const aggExpressionToEsAggsIdMap: Map<ExpressionAstExpressionBuilder, string> = new Map();
esAggEntries.forEach(([colId, col], index) => {
const def = operationDefinitionMap[col.operationType];
if (def.input !== 'fullReference' && def.input !== 'managedReference') {
const aggId = String(index);
const wrapInFilter = Boolean(def.filterable && col.filter);
let aggAst = def.toEsAggsFn(
col,
wrapInFilter ? `${index}-metric` : String(index),
wrapInFilter ? `${aggId}-metric` : aggId,
indexPattern,
layer,
uiSettings,
@ -139,12 +152,25 @@ function getExpressionForLayer(
}
).toAst();
}
aggs.push(
buildExpression({
type: 'expression',
chain: [aggAst],
})
);
const expressionBuilder = buildExpression({
type: 'expression',
chain: [aggAst],
});
aggs.push(expressionBuilder);
const esAggsId = window.ELASTIC_LENS_DELAY_SECONDS
? `col-${index + (col.isBucketed ? 0 : 1)}-${aggId}`
: `col-${index}-${aggId}`;
esAggsIdMap[esAggsId] = [
{
...col,
id: colId,
},
];
aggExpressionToEsAggsIdMap.set(expressionBuilder, esAggsId);
}
});
@ -164,19 +190,63 @@ function getExpressionForLayer(
);
}
const idMap = esAggEntries.reduce((currentIdMap, [colId, column], index) => {
const esAggsId = window.ELASTIC_LENS_DELAY_SECONDS
? `col-${index + (column.isBucketed ? 0 : 1)}-${index}`
: `col-${index}-${index}`;
uniq(esAggEntries.map(([_, column]) => column.operationType)).forEach((type) => {
const optimizeAggs = operationDefinitionMap[type].optimizeEsAggs?.bind(
operationDefinitionMap[type]
);
if (optimizeAggs) {
const { aggs: newAggs, esAggsIdMap: newIdMap } = optimizeAggs(
aggs,
esAggsIdMap,
aggExpressionToEsAggsIdMap
);
aggs = newAggs;
esAggsIdMap = newIdMap;
}
});
/*
Update ID mappings with new agg array positions.
Given this esAggs-ID-to-original-column map after percentile (for example) optimization:
col-0-0: column1
col-?-1.34: column2 (34th percentile)
col-2-2: column3
col-?-1.98: column4 (98th percentile)
and this array of aggs
0: { id: 0 }
1: { id: 2 }
2: { id: 1 }
We need to update the anticipated agg indicies to match the aggs array:
col-0-0: column1
col-2-1.34: column2 (34th percentile)
col-1-2: column3
col-3-3.98: column4 (98th percentile)
*/
const updatedEsAggsIdMap: Record<string, OriginalColumn[]> = {};
let counter = 0;
const esAggsIds = Object.keys(esAggsIdMap);
aggs.forEach((builder) => {
const esAggId = builder.functions[0].getArgument('id')?.[0];
const matchingEsAggColumnIds = esAggsIds.filter((id) => extractEsAggId(id) === esAggId);
matchingEsAggColumnIds.forEach((currentId) => {
const currentColumn = esAggsIdMap[currentId][0];
const aggIndex = window.ELASTIC_LENS_DELAY_SECONDS
? counter + (currentColumn.isBucketed ? 0 : 1)
: counter;
const newId = updatePositionIndex(currentId, aggIndex);
updatedEsAggsIdMap[newId] = esAggsIdMap[currentId];
counter++;
});
});
return {
...currentIdMap,
[esAggsId]: {
...column,
id: colId,
},
};
}, {} as Record<string, OriginalColumn>);
const columnsWithFormatters = columnEntries.filter(
([, col]) =>
(isColumnOfType<RangeIndexPatternColumn>('range', col) && col.params?.parentFormat) ||
@ -292,9 +362,9 @@ function getExpressionForLayer(
}).toAst(),
{
type: 'function',
function: 'lens_rename_columns',
function: 'lens_map_to_columns',
arguments: {
idMap: [JSON.stringify(idMap)],
idMap: [JSON.stringify(updatedEsAggsIdMap)],
},
},
...expressions,

4
x-pack/plugins/lens/server/expressions/expressions.ts
View file

@ -10,7 +10,7 @@ import type { ExpressionsServerSetup } from '@kbn/expressions-plugin/server';
import {
counterRate,
formatColumn,
renameColumns,
mapToColumns,
getTimeScale,
getDatatable,
} from '../../common/expressions';
@ -25,7 +25,7 @@ export const setupExpressions = (
[
counterRate,
formatColumn,
renameColumns,
mapToColumns,
getDatatable(getFormatFactory(core)),
getTimeScale(getDatatableUtilitiesFactory(core), getTimeZoneFactory(core)),
].forEach((expressionFn) => expressions.registerFunction(expressionFn));

2
x-pack/plugins/translations/translations/fr-FR.json
View file

@ -365,8 +365,6 @@
"xpack.lens.functions.counterRate.args.outputColumnNameHelpText": "Nom de la colonne dans laquelle le taux de compteur résultant sera stocké",
"xpack.lens.functions.counterRate.help": "Calcule le taux de compteur d'une colonne dans un tableau de données",
"xpack.lens.functions.lastValue.missingSortField": "Cette vue de données ne contient aucun champ de date.",
"xpack.lens.functions.renameColumns.help": "Aide pour renommer les colonnes d'un tableau de données",
"xpack.lens.functions.renameColumns.idMap.help": "Un objet encodé JSON dans lequel les clés sont les anciens ID de colonne et les valeurs sont les nouveaux ID correspondants. Tous les autres ID de colonne sont conservés.",
"xpack.lens.functions.timeScale.dateColumnMissingMessage": "L'ID de colonne de date {columnId} n'existe pas.",
"xpack.lens.functions.timeScale.timeInfoMissingMessage": "Impossible de récupérer les informations d'histogramme des dates",
"xpack.lens.gauge.addLayer": "Visualisation",

2
x-pack/plugins/translations/translations/ja-JP.json
View file

@ -367,8 +367,6 @@
"xpack.lens.functions.counterRate.args.outputColumnNameHelpText": "結果のカウンターレートを格納する列の名前",
"xpack.lens.functions.counterRate.help": "データテーブルの列のカウンターレートを計算します",
"xpack.lens.functions.lastValue.missingSortField": "このデータビューには日付フィールドが含まれていません",
"xpack.lens.functions.renameColumns.help": "データベースの列の名前の変更をアシストします",
"xpack.lens.functions.renameColumns.idMap.help": "キーが古い列 ID で値が対応する新しい列 ID となるように JSON エンコーディングされたオブジェクトです。他の列 ID はすべてのそのままです。",
"xpack.lens.functions.timeScale.dateColumnMissingMessage": "指定した dateColumnId {columnId} は存在しません。",
"xpack.lens.functions.timeScale.timeInfoMissingMessage": "日付ヒストグラム情報を取得できませんでした",
"xpack.lens.gauge.addLayer": "ビジュアライゼーション",

2
x-pack/plugins/translations/translations/zh-CN.json
View file

@ -372,8 +372,6 @@
"xpack.lens.functions.counterRate.args.outputColumnNameHelpText": "要存储结果计数率的列名称",
"xpack.lens.functions.counterRate.help": "在数据表中计算列的计数率",
"xpack.lens.functions.lastValue.missingSortField": "此数据视图不包含任何日期字段",
"xpack.lens.functions.renameColumns.help": "用于重命名数据表列的助手",
"xpack.lens.functions.renameColumns.idMap.help": "旧列 ID 为键且相应新列 ID 为值的 JSON 编码对象。所有其他列 ID 都将保留。",
"xpack.lens.functions.timeScale.dateColumnMissingMessage": "指定的 dateColumnId {columnId} 不存在。",
"xpack.lens.functions.timeScale.timeInfoMissingMessage": "无法获取日期直方图信息",
"xpack.lens.gauge.addLayer": "可视化",