[ML] Transforms: Fixes chart histograms for runtime fields. (#93028)

Fixes chart histograms for runtime fields. The runtime field configurations were not passed on to the endpoint to fetch the charts data, so charts ended up being empty with a 0 documents legend.
This commit is contained in:
Walter Rafelsberger 2021-03-02 13:37:29 +01:00 committed by GitHub
parent c0535abc06
commit 8201d4fd01
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 250 additions and 119 deletions

View file

@ -6,6 +6,7 @@
*/
export { HitsTotalRelation, SearchResponse7, HITS_TOTAL_RELATION } from './types/es_client';
export { ChartData } from './types/field_histograms';
export { ANOMALY_SEVERITY, ANOMALY_THRESHOLD, SEVERITY_COLORS } from './constants/anomalies';
export { getSeverityColor, getSeverityType } from './util/anomaly_utils';
export { composeValidators, patternValidator } from './util/validators';

View file

@ -0,0 +1,68 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export interface NumericDataItem {
key: number;
key_as_string?: string | number;
doc_count: number;
}
export interface NumericChartData {
data: NumericDataItem[];
id: string;
interval: number;
stats: [number, number];
type: 'numeric';
}
export const isNumericChartData = (arg: any): arg is NumericChartData => {
return (
typeof arg === 'object' &&
arg.hasOwnProperty('data') &&
arg.hasOwnProperty('id') &&
arg.hasOwnProperty('interval') &&
arg.hasOwnProperty('stats') &&
arg.hasOwnProperty('type') &&
arg.type === 'numeric'
);
};
export interface OrdinalDataItem {
key: string;
key_as_string?: string;
doc_count: number;
}
export interface OrdinalChartData {
cardinality: number;
data: OrdinalDataItem[];
id: string;
type: 'ordinal' | 'boolean';
}
export const isOrdinalChartData = (arg: any): arg is OrdinalChartData => {
return (
typeof arg === 'object' &&
arg.hasOwnProperty('data') &&
arg.hasOwnProperty('cardinality') &&
arg.hasOwnProperty('id') &&
arg.hasOwnProperty('type') &&
(arg.type === 'ordinal' || arg.type === 'boolean')
);
};
export interface UnsupportedChartData {
id: string;
type: 'unsupported';
}
export const isUnsupportedChartData = (arg: any): arg is UnsupportedChartData => {
return typeof arg === 'object' && arg.hasOwnProperty('type') && arg.type === 'unsupported';
};
export type ChartDataItem = NumericDataItem | OrdinalDataItem;
export type ChartData = NumericChartData | OrdinalChartData | UnsupportedChartData;

View file

@ -13,7 +13,9 @@ import { EuiDataGridColumn } from '@elastic/eui';
import './column_chart.scss';
import { isUnsupportedChartData, useColumnChart, ChartData } from './use_column_chart';
import { isUnsupportedChartData, ChartData } from '../../../../common/types/field_histograms';
import { useColumnChart } from './use_column_chart';
interface Props {
chartData: ChartData;

View file

@ -16,7 +16,7 @@ export {
useRenderCellValue,
getProcessedFields,
} from './common';
export { getFieldType, ChartData } from './use_column_chart';
export { getFieldType } from './use_column_chart';
export { useDataGrid } from './use_data_grid';
export { DataGrid } from './data_grid';
export {

View file

@ -11,10 +11,10 @@ import { EuiDataGridPaginationProps, EuiDataGridSorting, EuiDataGridColumn } fro
import { Dictionary } from '../../../../common/types/common';
import { HitsTotalRelation } from '../../../../common/types/es_client';
import { ChartData } from '../../../../common/types/field_histograms';
import { INDEX_STATUS } from '../../data_frame_analytics/common/analytics';
import { ChartData } from './use_column_chart';
import { FeatureImportanceBaseline } from '../../../../common/types/feature_importance';
export type ColumnId = string;

View file

@ -13,17 +13,15 @@ import '@testing-library/jest-dom/extend-expect';
import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
import {
getFieldType,
getLegendText,
getXScaleType,
isNumericChartData,
isOrdinalChartData,
isUnsupportedChartData,
useColumnChart,
NumericChartData,
OrdinalChartData,
UnsupportedChartData,
} from './use_column_chart';
} from '../../../../common/types/field_histograms';
import { getFieldType, getLegendText, getXScaleType, useColumnChart } from './use_column_chart';
describe('getFieldType()', () => {
it('should return the Kibana field type for a given EUI data grid schema', () => {

View file

@ -17,6 +17,15 @@ import { i18n } from '@kbn/i18n';
import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
import {
isNumericChartData,
isOrdinalChartData,
ChartData,
ChartDataItem,
NumericDataItem,
OrdinalDataItem,
} from '../../../../common/types/field_histograms';
import { NON_AGGREGATABLE } from './common';
export const hoveredRow$ = new BehaviorSubject<any | null>(null);
@ -66,68 +75,6 @@ export const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYP
return fieldType;
};
interface NumericDataItem {
key: number;
key_as_string?: string | number;
doc_count: number;
}
export interface NumericChartData {
data: NumericDataItem[];
id: string;
interval: number;
stats: [number, number];
type: 'numeric';
}
export const isNumericChartData = (arg: any): arg is NumericChartData => {
return (
typeof arg === 'object' &&
arg.hasOwnProperty('data') &&
arg.hasOwnProperty('id') &&
arg.hasOwnProperty('interval') &&
arg.hasOwnProperty('stats') &&
arg.hasOwnProperty('type') &&
arg.type === 'numeric'
);
};
export interface OrdinalDataItem {
key: string;
key_as_string?: string;
doc_count: number;
}
export interface OrdinalChartData {
cardinality: number;
data: OrdinalDataItem[];
id: string;
type: 'ordinal' | 'boolean';
}
export const isOrdinalChartData = (arg: any): arg is OrdinalChartData => {
return (
typeof arg === 'object' &&
arg.hasOwnProperty('data') &&
arg.hasOwnProperty('cardinality') &&
arg.hasOwnProperty('id') &&
arg.hasOwnProperty('type') &&
(arg.type === 'ordinal' || arg.type === 'boolean')
);
};
export interface UnsupportedChartData {
id: string;
type: 'unsupported';
}
export const isUnsupportedChartData = (arg: any): arg is UnsupportedChartData => {
return typeof arg === 'object' && arg.hasOwnProperty('type') && arg.type === 'unsupported';
};
export type ChartDataItem = NumericDataItem | OrdinalDataItem;
export type ChartData = NumericChartData | OrdinalChartData | UnsupportedChartData;
type LegendText = string | JSX.Element;
export const getLegendText = (
chartData: ChartData,

View file

@ -10,6 +10,7 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react';
import { EuiDataGridSorting, EuiDataGridColumn } from '@elastic/eui';
import { HITS_TOTAL_RELATION } from '../../../../common/types/es_client';
import { ChartData } from '../../../../common/types/field_histograms';
import { INDEX_STATUS } from '../../data_frame_analytics/common';
@ -26,7 +27,6 @@ import {
RowCountRelation,
UseDataGridReturnType,
} from './types';
import { ChartData } from './use_column_chart';
export const useDataGrid = (
columns: EuiDataGridColumn[],

View file

@ -7,10 +7,10 @@
import React, { FC, useMemo } from 'react';
import { EuiDataGridColumn } from '@elastic/eui';
import { OrdinalChartData } from '../../../../../../common/types/field_histograms';
import { ColumnChart } from '../../../../components/data_grid/column_chart';
import { FieldDataRowProps } from '../../types';
import { getTFPercentage } from '../../utils';
import { ColumnChart } from '../../../../components/data_grid/column_chart';
import { OrdinalChartData } from '../../../../components/data_grid/use_column_chart';
export const BooleanContentPreview: FC<FieldDataRowProps> = ({ config }) => {
const chartData = useMemo(() => {

View file

@ -7,10 +7,9 @@
import React, { FC } from 'react';
import { EuiDataGridColumn } from '@elastic/eui';
import type { FieldDataRowProps } from '../../types/field_data_row';
import { ChartData, OrdinalDataItem } from '../../../../../../common/types/field_histograms';
import { ColumnChart } from '../../../../components/data_grid/column_chart';
import { ChartData } from '../../../../components/data_grid';
import { OrdinalDataItem } from '../../../../components/data_grid/use_column_chart';
import type { FieldDataRowProps } from '../../types/field_data_row';
export const TopValuesPreview: FC<FieldDataRowProps> = ({ config }) => {
const { stats } = config;

View file

@ -16,7 +16,7 @@ import {
buildSamplerAggregation,
getSamplerAggregationsResponsePath,
} from '../../lib/query_utils';
import { AggCardinality } from '../../../common/types/fields';
import { AggCardinality, RuntimeMappings } from '../../../common/types/fields';
import { getDatafeedAggregations } from '../../../common/util/datafeed_utils';
import { Datafeed } from '../../../common/types/anomaly_detection_jobs';
import { isPopulatedObject } from '../../../common/util/object_utils';
@ -183,7 +183,8 @@ const getAggIntervals = async (
indexPatternTitle: string,
query: any,
fields: HistogramField[],
samplerShardSize: number
samplerShardSize: number,
runtimeMappings?: RuntimeMappings
): Promise<NumericColumnStatsMap> => {
const numericColumns = fields.filter((field) => {
return field.type === KBN_FIELD_TYPES.NUMBER || field.type === KBN_FIELD_TYPES.DATE;
@ -210,6 +211,7 @@ const getAggIntervals = async (
query,
aggs: buildSamplerAggregation(minMaxAggs, samplerShardSize),
size: 0,
...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}),
},
});
@ -240,7 +242,8 @@ export const getHistogramsForFields = async (
indexPatternTitle: string,
query: any,
fields: HistogramField[],
samplerShardSize: number
samplerShardSize: number,
runtimeMappings?: RuntimeMappings
) => {
const { asCurrentUser } = client;
const aggIntervals = await getAggIntervals(
@ -248,7 +251,8 @@ export const getHistogramsForFields = async (
indexPatternTitle,
query,
fields,
samplerShardSize
samplerShardSize,
runtimeMappings
);
const chartDataAggs = fields.reduce((aggs, field) => {
@ -293,6 +297,7 @@ export const getHistogramsForFields = async (
query,
aggs: buildSamplerAggregation(chartDataAggs, samplerShardSize),
size: 0,
...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}),
},
});
@ -607,7 +612,7 @@ export class DataVisualizer {
// Value count aggregation faster way of checking if field exists than using
// filter aggregation with exists query.
const aggs: Aggs = datafeedAggregations !== undefined ? { ...datafeedAggregations } : {};
const runtimeMappings: any = {};
const runtimeMappings: { runtime_mappings?: RuntimeMappings } = {};
aggregatableFields.forEach((field, i) => {
const safeFieldName = getSafeAggregationName(field, i);

View file

@ -53,3 +53,27 @@ export interface ResponseStatus {
export interface CommonResponseStatusSchema {
[key: string]: ResponseStatus;
}
export const runtimeMappingsSchema = schema.maybe(
schema.recordOf(
schema.string(),
schema.object({
type: schema.oneOf([
schema.literal('keyword'),
schema.literal('long'),
schema.literal('double'),
schema.literal('date'),
schema.literal('ip'),
schema.literal('boolean'),
]),
script: schema.maybe(
schema.oneOf([
schema.string(),
schema.object({
source: schema.string(),
}),
])
),
})
)
);

View file

@ -7,14 +7,20 @@
import { schema, TypeOf } from '@kbn/config-schema';
import { ChartData } from '../shared_imports';
import { runtimeMappingsSchema } from './common';
export const fieldHistogramsRequestSchema = schema.object({
/** Query to match documents in the index. */
query: schema.any(),
/** The fields to return histogram data. */
fields: schema.arrayOf(schema.any()),
/** Optional runtime mappings */
runtimeMappings: runtimeMappingsSchema,
/** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */
samplerShardSize: schema.number(),
});
export type FieldHistogramsRequestSchema = TypeOf<typeof fieldHistogramsRequestSchema>;
export type FieldHistogramsResponseSchema = any[];
export type FieldHistogramsResponseSchema = ChartData[];

View file

@ -14,7 +14,7 @@ import type { PivotAggDict } from '../types/pivot_aggs';
import type { PivotGroupByDict } from '../types/pivot_group_by';
import type { TransformId, TransformPivotConfig } from '../types/transform';
import { transformStateSchema } from './common';
import { transformStateSchema, runtimeMappingsSchema } from './common';
// GET transforms
export const getTransformsRequestSchema = schema.arrayOf(
@ -64,30 +64,6 @@ export const settingsSchema = schema.object({
docs_per_second: schema.maybe(schema.nullable(schema.number())),
});
export const runtimeMappingsSchema = schema.maybe(
schema.recordOf(
schema.string(),
schema.object({
type: schema.oneOf([
schema.literal('keyword'),
schema.literal('long'),
schema.literal('double'),
schema.literal('date'),
schema.literal('ip'),
schema.literal('boolean'),
]),
script: schema.maybe(
schema.oneOf([
schema.string(),
schema.object({
source: schema.string(),
}),
])
),
})
)
);
export const sourceSchema = schema.object({
runtime_mappings: runtimeMappingsSchema,
index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),

View file

@ -6,5 +6,9 @@
*/
export type { HitsTotalRelation, SearchResponse7 } from '../../ml/common';
export { HITS_TOTAL_RELATION } from '../../ml/common';
export { composeValidators, patternValidator } from '../../ml/common';
export {
composeValidators,
patternValidator,
ChartData,
HITS_TOTAL_RELATION,
} from '../../ml/common';

View file

@ -16,7 +16,10 @@ import type {
DeleteTransformsRequestSchema,
DeleteTransformsResponseSchema,
} from '../../../common/api_schemas/delete_transforms';
import type { FieldHistogramsResponseSchema } from '../../../common/api_schemas/field_histograms';
import type {
FieldHistogramsRequestSchema,
FieldHistogramsResponseSchema,
} from '../../../common/api_schemas/field_histograms';
import type {
StartTransformsRequestSchema,
StartTransformsResponseSchema,
@ -194,6 +197,7 @@ export const useApi = () => {
indexPatternTitle: string,
fields: FieldHistogramRequestConfig[],
query: string | SavedSearchQuery,
runtimeMappings?: FieldHistogramsRequestSchema['runtimeMappings'],
samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE
): Promise<FieldHistogramsResponseSchema | HttpFetchError> {
try {
@ -202,6 +206,7 @@ export const useApi = () => {
query,
fields,
samplerShardSize,
...(runtimeMappings !== undefined ? { runtimeMappings } : {}),
}),
});
} catch (e) {

View file

@ -150,7 +150,8 @@ export const useIndexData = (
fieldName: cT.id,
type: getFieldType(cT.schema),
})),
isDefaultQuery(query) ? matchAllQuery : query
isDefaultQuery(query) ? matchAllQuery : query,
combinedRuntimeMappings
);
if (!isFieldHistogramsResponseSchema(columnChartsData)) {

View file

@ -13,6 +13,7 @@ import { EuiCodeEditor } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { StepDefineFormHook } from '../step_define';
import { isRuntimeMappings } from '../step_define/common/types';
export const AdvancedRuntimeMappingsEditor: FC<StepDefineFormHook['runtimeMappingsEditor']> = memo(
({
@ -43,8 +44,8 @@ export const AdvancedRuntimeMappingsEditor: FC<StepDefineFormHook['runtimeMappin
// Try to parse the string passed on from the editor.
// If parsing fails, the "Apply"-Button will be disabled
try {
JSON.parse(convertToJson(d));
setRuntimeMappingsEditorApplyButtonEnabled(true);
const parsedJson = JSON.parse(convertToJson(d));
setRuntimeMappingsEditorApplyButtonEnabled(isRuntimeMappings(parsedJson));
} catch (e) {
setRuntimeMappingsEditorApplyButtonEnabled(false);
}

View file

@ -26,7 +26,7 @@ import {
import { getDefaultAggregationConfig } from './get_default_aggregation_config';
import { getDefaultGroupByConfig } from './get_default_group_by_config';
import type { Field, StepDefineExposedState } from './types';
import { isPopulatedObject } from '../../../../../common/utils/object_utils';
import { isRuntimeMappings } from './types';
const illegalEsAggNameChars = /[[\]>]/g;
@ -77,7 +77,7 @@ export function getPivotDropdownOptions(
// Support for runtime_mappings that are defined by queries
let runtimeFields: Field[] = [];
if (isPopulatedObject(runtimeMappings)) {
if (isRuntimeMappings(runtimeMappings)) {
runtimeFields = Object.keys(runtimeMappings).map((fieldName) => {
const field = runtimeMappings[fieldName];
return { name: fieldName, type: getKibanaFieldTypeFromEsType(field.type) };

View file

@ -0,0 +1,71 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isRuntimeField, isRuntimeMappings } from './types';
describe('Transform: step_define type guards', () => {
it('isRuntimeField()', () => {
expect(isRuntimeField(1)).toBe(false);
expect(isRuntimeField(null)).toBe(false);
expect(isRuntimeField([])).toBe(false);
expect(isRuntimeField({})).toBe(false);
expect(isRuntimeField({ someAttribute: 'someValue' })).toBe(false);
expect(isRuntimeField({ type: 'wrong-type' })).toBe(false);
expect(isRuntimeField({ type: 'keyword', someAttribute: 'some value' })).toBe(false);
expect(isRuntimeField({ type: 'keyword' })).toBe(true);
expect(isRuntimeField({ type: 'keyword', script: 'some script' })).toBe(true);
});
it('isRuntimeMappings()', () => {
expect(isRuntimeMappings(1)).toBe(false);
expect(isRuntimeMappings(null)).toBe(false);
expect(isRuntimeMappings([])).toBe(false);
expect(isRuntimeMappings({})).toBe(false);
expect(isRuntimeMappings({ someAttribute: 'someValue' })).toBe(false);
expect(isRuntimeMappings({ fieldName1: { type: 'keyword' }, fieldName2: 'someValue' })).toBe(
false
);
expect(
isRuntimeMappings({
fieldName1: { type: 'keyword' },
fieldName2: { type: 'keyword', someAttribute: 'some value' },
})
).toBe(false);
expect(
isRuntimeMappings({
fieldName: { type: 'long', script: 1234 },
})
).toBe(false);
expect(
isRuntimeMappings({
fieldName: { type: 'long', script: { someAttribute: 'some value' } },
})
).toBe(false);
expect(
isRuntimeMappings({
fieldName: { type: 'long', script: { source: 1234 } },
})
).toBe(false);
expect(isRuntimeMappings({ fieldName: { type: 'keyword' } })).toBe(true);
expect(
isRuntimeMappings({ fieldName1: { type: 'keyword' }, fieldName2: { type: 'keyword' } })
).toBe(true);
expect(
isRuntimeMappings({
fieldName1: { type: 'keyword' },
fieldName2: { type: 'keyword', script: 'some script as script' },
})
).toBe(true);
expect(
isRuntimeMappings({
fieldName: { type: 'long', script: { source: 'some script as source' } },
})
).toBe(true);
});
});

View file

@ -24,6 +24,8 @@ import {
} from '../../../../../../../common/types/transform';
import { LatestFunctionConfig } from '../../../../../../../common/api_schemas/transforms';
import { isPopulatedObject } from '../../../../../common/utils/object_utils';
export interface ErrorMessage {
query: string;
message: string;
@ -70,10 +72,30 @@ export interface StepDefineExposedState {
isRuntimeMappingsEditorEnabled: boolean;
}
export function isRuntimeField(arg: any): arg is RuntimeField {
return (
isPopulatedObject(arg) &&
((Object.keys(arg).length === 1 && arg.hasOwnProperty('type')) ||
(Object.keys(arg).length === 2 &&
arg.hasOwnProperty('type') &&
arg.hasOwnProperty('script') &&
(typeof arg.script === 'string' ||
(isPopulatedObject(arg.script) &&
Object.keys(arg.script).length === 1 &&
arg.script.hasOwnProperty('source') &&
typeof arg.script.source === 'string')))) &&
RUNTIME_FIELD_TYPES.includes(arg.type)
);
}
export function isRuntimeMappings(arg: any): arg is RuntimeMappings {
return isPopulatedObject(arg) && Object.values(arg).every((d) => isRuntimeField(d));
}
export function isPivotPartialRequest(arg: any): arg is { pivot: PivotConfigDefinition } {
return typeof arg === 'object' && arg.hasOwnProperty('pivot');
return isPopulatedObject(arg) && arg.hasOwnProperty('pivot');
}
export function isLatestPartialRequest(arg: any): arg is { latest: LatestFunctionConfig } {
return typeof arg === 'object' && arg.hasOwnProperty('latest');
return isPopulatedObject(arg) && arg.hasOwnProperty('latest');
}

View file

@ -32,7 +32,7 @@ export function registerFieldHistogramsRoutes({ router, license }: RouteDependen
license.guardApiRoute<IndexPatternTitleSchema, undefined, FieldHistogramsRequestSchema>(
async (ctx, req, res) => {
const { indexPatternTitle } = req.params;
const { query, fields, samplerShardSize } = req.body;
const { query, fields, runtimeMappings, samplerShardSize } = req.body;
try {
const resp = await getHistogramsForFields(
@ -40,7 +40,8 @@ export function registerFieldHistogramsRoutes({ router, license }: RouteDependen
indexPatternTitle,
query,
fields,
samplerShardSize
samplerShardSize,
runtimeMappings
);
return res.ok({ body: resp });