[Lens] Time scaling without date histogram (#140107)

* [Lens] Time scaling without date histogram

Closes: #79656

* cleanup

* fix observability tests

* fix scale_fn jest

* add test

* fix PR comments

* fix PR comment

* remove adjustTimeScaleOnOtherColumnChange

* add reducedTimeRange argument into time_scale

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Alexey Antonov 2022-09-15 13:14:23 +03:00 committed by GitHub
parent c0c161d709
commit a60b730d0e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 193 additions and 210 deletions

View file

@ -6,7 +6,7 @@
*/
import moment from 'moment';
import type { Datatable } from '@kbn/expressions-plugin/common';
import type { Datatable, ExecutionContext } from '@kbn/expressions-plugin/common';
import type { TimeRange } from '@kbn/es-query';
import { createDatatableUtilitiesMock } from '@kbn/data-plugin/common/mocks';
@ -28,7 +28,12 @@ import { getTimeScale } from './time_scale';
import type { TimeScaleArgs } from './types';
describe('time_scale', () => {
let timeScaleWrapped: (input: Datatable, args: TimeScaleArgs) => Promise<Datatable>;
let timeScaleWrapped: (
input: Datatable,
args: TimeScaleArgs,
context?: ExecutionContext
) => Promise<Datatable>;
const timeScale = getTimeScale(createDatatableUtilitiesMock, () => 'UTC');
const emptyTable: Datatable = {
@ -391,6 +396,65 @@ describe('time_scale', () => {
expect(result.rows.map(({ scaledMetric }) => scaledMetric)).toEqual([1, 1, 1, 1, 1]);
});
it('should apply fn for non-histogram fields', async () => {
const result = await timeScaleWrapped(
{
...emptyTable,
rows: [
{
date: moment('2010-01-01T00:00:00.000Z').valueOf(),
metric: 300,
},
],
},
{
inputColumnId: 'metric',
outputColumnId: 'scaledMetric',
targetUnit: 'd',
},
{
getSearchContext: () => ({
timeRange: {
from: '2010-01-01T00:00:00.000Z',
to: '2010-01-05T00:00:00.000Z',
},
}),
} as unknown as ExecutionContext
);
expect(result.rows.map(({ scaledMetric }) => scaledMetric)).toEqual([75]);
});
it('should apply fn for non-histogram fields (with Reduced time range)', async () => {
const result = await timeScaleWrapped(
{
...emptyTable,
rows: [
{
date: moment('2010-01-04T00:00:00.000Z').valueOf(),
metric: 300,
},
],
},
{
inputColumnId: 'metric',
outputColumnId: 'scaledMetric',
targetUnit: 'd',
reducedTimeRange: '4d',
},
{
getSearchContext: () => ({
timeRange: {
from: '2009-01-01T00:00:00.000Z',
to: '2010-01-05T00:00:00.000Z',
},
}),
} as unknown as ExecutionContext
);
expect(result.rows.map(({ scaledMetric }) => scaledMetric)).toEqual([75]);
});
it('should be sync except for timezone getter to prevent timezone leakage', async () => {
let resolveTimezonePromise: (value: string | PromiseLike<string>) => void;
const timezonePromise = new Promise<string>((res) => {

View file

@ -18,7 +18,6 @@ export const getTimeScale = (
dateColumnId: {
types: ['string'],
help: '',
required: true,
},
inputColumnId: {
types: ['string'],
@ -40,6 +39,10 @@ export const getTimeScale = (
help: '',
required: true,
},
reducedTimeRange: {
types: ['string'],
help: '',
},
},
inputTypes: ['datatable'],
async fn(...args) {

View file

@ -5,10 +5,16 @@
* 2.0.
*/
import moment from 'moment-timezone';
import moment, { Moment } from 'moment-timezone';
import { i18n } from '@kbn/i18n';
import { buildResultColumns, Datatable, ExecutionContext } from '@kbn/expressions-plugin/common';
import { calculateBounds, DatatableUtilitiesService, parseInterval } from '@kbn/data-plugin/common';
import { buildResultColumns, DatatableRow, ExecutionContext } from '@kbn/expressions-plugin/common';
import {
calculateBounds,
DatatableUtilitiesService,
parseInterval,
TimeRangeBounds,
TimeRange,
} from '@kbn/data-plugin/common';
import type { TimeScaleExpressionFunction, TimeScaleUnit, TimeScaleArgs } from './types';
const unitInMs: Record<TimeScaleUnit, number> = {
@ -27,20 +33,79 @@ export const timeScaleFn =
): TimeScaleExpressionFunction['fn'] =>
async (
input,
{ dateColumnId, inputColumnId, outputColumnId, outputColumnName, targetUnit }: TimeScaleArgs,
{
dateColumnId,
inputColumnId,
outputColumnId,
outputColumnName,
targetUnit,
reducedTimeRange,
}: TimeScaleArgs,
context
) => {
const dateColumnDefinition = input.columns.find((column) => column.id === dateColumnId);
let timeBounds: TimeRangeBounds | undefined;
const contextTimeZone = await getTimezone(context);
if (!dateColumnDefinition) {
throw new Error(
i18n.translate('xpack.lens.functions.timeScale.dateColumnMissingMessage', {
defaultMessage: 'Specified dateColumnId {columnId} does not exist.',
values: {
columnId: dateColumnId,
},
})
);
let getStartEndOfBucketMeta: (row: DatatableRow) => {
startOfBucket: Moment;
endOfBucket: Moment;
};
if (dateColumnId) {
const dateColumnDefinition = input.columns.find((column) => column.id === dateColumnId);
if (!dateColumnDefinition) {
throw new Error(
i18n.translate('xpack.lens.functions.timeScale.dateColumnMissingMessage', {
defaultMessage: 'Specified dateColumnId {columnId} does not exist.',
values: {
columnId: dateColumnId,
},
})
);
}
const datatableUtilities = await getDatatableUtilities(context);
const timeInfo = datatableUtilities.getDateHistogramMeta(dateColumnDefinition, {
timeZone: contextTimeZone,
});
const intervalDuration = timeInfo?.interval && parseInterval(timeInfo.interval);
timeBounds = timeInfo?.timeRange && calculateBounds(timeInfo.timeRange);
getStartEndOfBucketMeta = (row) => {
const startOfBucket = moment.tz(row[dateColumnId], timeInfo?.timeZone ?? contextTimeZone);
return {
startOfBucket,
endOfBucket: startOfBucket.clone().add(intervalDuration),
};
};
if (!timeInfo || !intervalDuration) {
throw new Error(
i18n.translate('xpack.lens.functions.timeScale.timeInfoMissingMessage', {
defaultMessage: 'Could not fetch date histogram information',
})
);
}
} else {
const timeRange = context.getSearchContext().timeRange as TimeRange;
const endOfBucket = moment.tz(timeRange.to, contextTimeZone);
let startOfBucket = moment.tz(timeRange.from, contextTimeZone);
if (reducedTimeRange) {
const reducedStartOfBucket = endOfBucket.clone().subtract(parseInterval(reducedTimeRange));
if (reducedStartOfBucket > startOfBucket) {
startOfBucket = reducedStartOfBucket;
}
}
timeBounds = calculateBounds(timeRange);
getStartEndOfBucketMeta = () => ({
startOfBucket,
endOfBucket,
});
}
const resultColumns = buildResultColumns(
@ -57,61 +122,30 @@ export const timeScaleFn =
return input;
}
const targetUnitInMs = unitInMs[targetUnit];
const datatableUtilities = await getDatatableUtilities(context);
const timeInfo = datatableUtilities.getDateHistogramMeta(dateColumnDefinition, {
timeZone: await getTimezone(context),
});
const intervalDuration = timeInfo?.interval && parseInterval(timeInfo.interval);
return {
...input,
columns: resultColumns,
rows: input.rows.map((row) => {
const newRow = { ...row };
if (!timeInfo || !intervalDuration) {
throw new Error(
i18n.translate('xpack.lens.functions.timeScale.timeInfoMissingMessage', {
defaultMessage: 'Could not fetch date histogram information',
})
);
}
// the datemath plugin always parses dates by using the current default moment time zone.
// to use the configured time zone, we are switching just for the bounds calculation.
let { startOfBucket, endOfBucket } = getStartEndOfBucketMeta(row);
// The code between this call and the reset in the finally block is not allowed to get async,
// otherwise the timezone setting can leak out of this function.
const defaultTimezone = moment().zoneName();
let result: Datatable;
try {
moment.tz.setDefault(timeInfo.timeZone);
if (timeBounds && timeBounds.min) {
startOfBucket = moment.max(startOfBucket, timeBounds.min);
}
if (timeBounds && timeBounds.max) {
endOfBucket = moment.min(endOfBucket, timeBounds.max);
}
const timeBounds = timeInfo.timeRange && calculateBounds(timeInfo.timeRange);
const bucketSize = endOfBucket.diff(startOfBucket);
const factor = bucketSize / unitInMs[targetUnit];
const currentValue = newRow[inputColumnId];
result = {
...input,
columns: resultColumns,
rows: input.rows.map((row) => {
const newRow = { ...row };
if (currentValue != null) {
newRow[outputColumnId] = Number(currentValue) / factor;
}
let startOfBucket = moment(row[dateColumnId]);
let endOfBucket = startOfBucket.clone().add(intervalDuration);
if (timeBounds && timeBounds.min) {
startOfBucket = moment.max(startOfBucket, timeBounds.min);
}
if (timeBounds && timeBounds.max) {
endOfBucket = moment.min(endOfBucket, timeBounds.max);
}
const bucketSize = endOfBucket.diff(startOfBucket);
const factor = bucketSize / targetUnitInMs;
const currentValue = newRow[inputColumnId];
if (currentValue != null) {
newRow[outputColumnId] = Number(currentValue) / factor;
}
return newRow;
}),
};
} finally {
// reset default moment timezone
moment.tz.setDefault(defaultTimezone);
}
return result;
return newRow;
}),
};
};

View file

@ -10,11 +10,12 @@ import type { Datatable, ExpressionFunctionDefinition } from '@kbn/expressions-p
export type TimeScaleUnit = 's' | 'm' | 'h' | 'd';
export interface TimeScaleArgs {
dateColumnId: string;
inputColumnId: string;
outputColumnId: string;
targetUnit: TimeScaleUnit;
dateColumnId?: string;
outputColumnName?: string;
reducedTimeRange?: string;
}
export type TimeScaleExpressionFunction = ExpressionFunctionDefinition<

View file

@ -66,15 +66,9 @@ export function TimeScaling({
layer: IndexPatternLayer;
updateLayer: (newLayer: IndexPatternLayer) => void;
}) {
const hasDateHistogram = layer.columnOrder.some(
(colId) => layer.columns[colId].operationType === 'date_histogram'
);
const selectedOperation = operationDefinitionMap[selectedColumn.operationType];
if (
!selectedOperation.timeScalingMode ||
selectedOperation.timeScalingMode === 'disabled' ||
!hasDateHistogram
) {
if (!selectedOperation.timeScalingMode || selectedOperation.timeScalingMode === 'disabled') {
return null;
}

View file

@ -782,6 +782,7 @@ describe('IndexPattern Data Source', () => {
"outputColumnName": Array [
"Count of records",
],
"reducedTimeRange": Array [],
"targetUnit": Array [
"h",
],

View file

@ -16,7 +16,6 @@ import {
hasDateField,
checkForDataLayerType,
} from './utils';
import { adjustTimeScaleOnOtherColumnChange } from '../../time_scale_utils';
import { OperationDefinition } from '..';
import { getFormatFromPreviousColumn, getFilter, combineErrorMessages } from '../helpers';
import { getDisallowedPreviousShiftMessage } from '../../../time_shift_utils';
@ -93,7 +92,6 @@ export const derivativeOperation: OperationDefinition<
isTransferable: (column, newIndexPattern) => {
return hasDateField(newIndexPattern);
},
onOtherColumnChanged: adjustTimeScaleOnOtherColumnChange,
getErrorMessage: (layer: IndexPatternLayer, columnId: string) => {
return combineErrorMessages([
getErrorsForDateReference(

View file

@ -27,7 +27,6 @@ import {
getFilter,
combineErrorMessages,
} from '../helpers';
import { adjustTimeScaleOnOtherColumnChange } from '../../time_scale_utils';
import type { OperationDefinition, ParamEditorProps } from '..';
import { getDisallowedPreviousShiftMessage } from '../../../time_shift_utils';
@ -115,7 +114,6 @@ export const movingAverageOperation: OperationDefinition<
isTransferable: (column, newIndexPattern) => {
return hasDateField(newIndexPattern);
},
onOtherColumnChanged: adjustTimeScaleOnOtherColumnChange,
getErrorMessage: (layer: IndexPatternLayer, columnId: string) => {
return combineErrorMessages([
getErrorsForDateReference(

View file

@ -65,17 +65,21 @@ export const timeScaleOperation: OperationDefinition<TimeScaleIndexPatternColumn
const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed);
const dateColumn = buckets.find(
(colId) => layer.columns[colId].operationType === 'date_histogram'
)!;
);
return [
{
type: 'function',
function: 'lens_time_scale',
arguments: {
dateColumnId: [dateColumn],
dateColumnId: dateColumn ? [dateColumn] : [],
inputColumnId: [currentColumn.references[0]],
outputColumnId: [columnId],
outputColumnName: [currentColumn.label],
targetUnit: [currentColumn.params.unit!],
reducedTimeRange: currentColumn.reducedTimeRange
? [currentColumn.reducedTimeRange]
: [],
},
},
];

View file

@ -22,10 +22,7 @@ import {
getFormatFromPreviousColumn,
isColumnOfType,
} from './helpers';
import {
adjustTimeScaleLabelSuffix,
adjustTimeScaleOnOtherColumnChange,
} from '../time_scale_utils';
import { adjustTimeScaleLabelSuffix } from '../time_scale_utils';
import { getDisallowedPreviousShiftMessage } from '../../time_shift_utils';
import { updateColumnParam } from '../layer_helpers';
import { getColumnReducedTimeRangeError } from '../../reduced_time_range_utils';
@ -186,8 +183,6 @@ export const countOperation: OperationDefinition<CountIndexPatternColumn, 'field
},
];
},
onOtherColumnChanged: (layer, thisColumnId) =>
adjustTimeScaleOnOtherColumnChange<CountIndexPatternColumn>(layer, thisColumnId),
toEsAggsFn: (column, columnId, indexPattern) => {
const field = indexPattern.getFieldByName(column.sourceField);
if (field?.type === 'document') {

View file

@ -24,10 +24,7 @@ import {
BaseIndexPatternColumn,
ValueFormatConfig,
} from './column_types';
import {
adjustTimeScaleLabelSuffix,
adjustTimeScaleOnOtherColumnChange,
} from '../time_scale_utils';
import { adjustTimeScaleLabelSuffix } from '../time_scale_utils';
import { getDisallowedPreviousShiftMessage } from '../../time_shift_utils';
import { updateColumnParam } from '../layer_helpers';
import { getColumnReducedTimeRangeError } from '../../reduced_time_range_utils';
@ -117,10 +114,6 @@ function buildMetricOperation<T extends MetricColumn<string>>({
(!newField.aggregationRestrictions || newField.aggregationRestrictions![type])
);
},
onOtherColumnChanged: (layer, thisColumnId) =>
optionalTimeScaling
? (adjustTimeScaleOnOtherColumnChange(layer, thisColumnId) as T)
: (layer.columns[thisColumnId] as T),
getDefaultLabel: (column, indexPattern, columns) =>
labelLookup(getSafeName(column.sourceField, indexPattern), column),
buildColumn: ({ field, previousColumn }, columnParams) => {

View file

@ -5,10 +5,8 @@
* 2.0.
*/
import type { IndexPatternLayer } from '../types';
import type { TimeScaleUnit } from '../../../common/expressions';
import type { DateHistogramIndexPatternColumn, GenericIndexPatternColumn } from './definitions';
import { adjustTimeScaleLabelSuffix, adjustTimeScaleOnOtherColumnChange } from './time_scale_utils';
import { adjustTimeScaleLabelSuffix } from './time_scale_utils';
export const DEFAULT_TIME_SCALE = 's' as TimeScaleUnit;
@ -271,68 +269,4 @@ describe('time scale utils', () => {
).toEqual('abc per day');
});
});
describe('adjustTimeScaleOnOtherColumnChange', () => {
const baseColumn: GenericIndexPatternColumn = {
operationType: 'count',
sourceField: '___records___',
label: 'Count of records per second',
dataType: 'number',
isBucketed: false,
timeScale: 's',
};
const baseLayer: IndexPatternLayer = {
columns: { col1: baseColumn },
columnOrder: [],
indexPatternId: '',
};
it('should keep column if there is no time scale', () => {
const column = { ...baseColumn, timeScale: undefined };
expect(
adjustTimeScaleOnOtherColumnChange({ ...baseLayer, columns: { col1: column } }, 'col1')
).toBe(column);
});
it('should keep time scale if there is a date histogram', () => {
expect(
adjustTimeScaleOnOtherColumnChange(
{
...baseLayer,
columns: {
col1: baseColumn,
col2: {
operationType: 'date_histogram',
dataType: 'date',
isBucketed: true,
label: '',
sourceField: 'date',
params: { interval: 'auto' },
} as DateHistogramIndexPatternColumn,
},
},
'col1'
)
).toBe(baseColumn);
});
it('should remove time scale if there is no date histogram', () => {
expect(adjustTimeScaleOnOtherColumnChange(baseLayer, 'col1')).toHaveProperty(
'timeScale',
undefined
);
});
it('should remove suffix from label', () => {
expect(
adjustTimeScaleOnOtherColumnChange({ ...baseLayer, columns: { col1: baseColumn } }, 'col1')
).toHaveProperty('label', 'Count of records');
});
it('should keep custom label', () => {
const column = { ...baseColumn, label: 'abc', customLabel: true };
expect(
adjustTimeScaleOnOtherColumnChange({ ...baseLayer, columns: { col1: column } }, 'col1')
).toHaveProperty('label', 'abc');
});
});
});

View file

@ -8,8 +8,6 @@
import { i18n } from '@kbn/i18n';
import { unitSuffixesLong } from '../../../common/suffix_formatter';
import type { TimeScaleUnit } from '../../../common/expressions';
import type { IndexPatternLayer } from '../types';
import type { GenericIndexPatternColumn } from './definitions';
export const DEFAULT_TIME_SCALE = 's' as TimeScaleUnit;
@ -57,36 +55,3 @@ export function adjustTimeScaleLabelSuffix(
// add new suffix if column has a time scale now
return `${cleanedLabel}${getSuffix(newTimeScale, newShift, newReducedTimeRange)}`;
}
export function adjustTimeScaleOnOtherColumnChange<T extends GenericIndexPatternColumn>(
layer: IndexPatternLayer,
thisColumnId: string
): T {
const columns = layer.columns;
const column = columns[thisColumnId] as T;
if (!column.timeScale) {
return column;
}
const hasDateHistogram = Object.values(columns).some(
(col) => col?.operationType === 'date_histogram'
);
if (hasDateHistogram) {
return column;
}
if (column.customLabel) {
return column;
}
return {
...column,
timeScale: undefined,
label: adjustTimeScaleLabelSuffix(
column.label,
column.timeScale,
undefined,
column.timeShift,
column.timeShift,
column.reducedTimeRange,
column.reducedTimeRange
),
};
}

View file

@ -57,7 +57,6 @@ function getExpressionForLayer(
if (columnOrder.length === 0 || !indexPattern) {
return null;
}
const columns = { ...layer.columns };
Object.keys(columns).forEach((columnId) => {
const column = columns[columnId];
@ -289,25 +288,25 @@ function getExpressionForLayer(
([, col]) => col.operationType === 'date_histogram'
);
const columnsWithTimeScale = firstDateHistogramColumn
? columnEntries.filter(
([, col]) =>
col.timeScale &&
operationDefinitionMap[col.operationType].timeScalingMode &&
operationDefinitionMap[col.operationType].timeScalingMode !== 'disabled'
)
: [];
const columnsWithTimeScale = columnEntries.filter(
([, col]) =>
col.timeScale &&
operationDefinitionMap[col.operationType].timeScalingMode &&
operationDefinitionMap[col.operationType].timeScalingMode !== 'disabled'
);
const timeScaleFunctions: ExpressionAstFunction[] = columnsWithTimeScale.flatMap(
([id, col]) => {
const scalingCall: ExpressionAstFunction = {
type: 'function',
function: 'lens_time_scale',
arguments: {
dateColumnId: [firstDateHistogramColumn![0]],
dateColumnId: firstDateHistogramColumn?.length ? [firstDateHistogramColumn[0]] : [],
inputColumnId: [id],
outputColumnId: [id],
outputColumnName: [col.label],
targetUnit: [col.timeScale!],
reducedTimeRange: col.reducedTimeRange ? [col.reducedTimeRange] : [],
},
};