adding TSDB support to DataViews (#137677)

This commit is contained in:
Peter Pisljar 2022-08-14 06:23:06 +02:00 committed by GitHub
parent e15e9320a7
commit 9799dbba27
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 144 additions and 175 deletions

View file

@ -43,6 +43,7 @@ export const createKbnFieldTypes = (): KbnFieldType[] => [
ES_FIELD_TYPES.SHORT,
ES_FIELD_TYPES.BYTE,
ES_FIELD_TYPES.TOKEN_COUNT,
ES_FIELD_TYPES.AGGREGATE_METRIC_DOUBLE,
],
}),
new KbnFieldType({

View file

@ -46,6 +46,8 @@ export enum ES_FIELD_TYPES {
SHORT = 'short',
UNSIGNED_LONG = 'unsigned_long',
AGGREGATE_METRIC_DOUBLE = 'aggregate_metric_double',
FLOAT_RANGE = 'float_range',
DOUBLE_RANGE = 'double_range',
INTEGER_RANGE = 'integer_range',

View file

@ -219,9 +219,12 @@ export const getDateHistogramBucketAgg = ({
// this DSL will anyway not be used before we're passing this code with an actual interval.
return;
}
const shouldForceFixedInterval: boolean = agg.params.field?.meta?.fixed_interval?.length;
output.params = {
...output.params,
...dateHistogramInterval(interval.expression),
...dateHistogramInterval(interval.expression, shouldForceFixedInterval),
};
const scaleMetrics =
@ -272,7 +275,10 @@ export const getDateHistogramBucketAgg = ({
getConfig,
aggExecutionContext
);
output.params.time_zone = tz;
const shouldForceTimeZone = agg.params.field?.meta?.time_zone?.includes('UTC');
output.params.time_zone = shouldForceTimeZone ? 'UTC' : tz;
},
},
{

View file

@ -10,6 +10,12 @@ import { parseEsInterval } from './parse_es_interval';
type Interval = { fixed_interval: string } | { calendar_interval: string };
const calendarFixedConversion = {
w: '7d',
M: '30d',
y: '365d',
};
/**
* Checks whether a given Elasticsearch interval is a calendar or fixed interval
* and returns an object containing the appropriate date_histogram property for that
@ -28,11 +34,14 @@ type Interval = { fixed_interval: string } | { calendar_interval: string };
*
* @param interval The interval string to return the appropriate date_histogram key for.
*/
export function dateHistogramInterval(interval: string): Interval {
const { type } = parseEsInterval(interval);
if (type === 'calendar') {
export function dateHistogramInterval(interval: string, shouldForceFixed?: boolean): Interval {
const { type, unit } = parseEsInterval(interval);
if (type === 'calendar' && !shouldForceFixed) {
return { calendar_interval: interval };
} else {
if (type === 'calendar') {
return { fixed_interval: calendarFixedConversion[unit as 'w' | 'M' | 'y'] || interval };
}
return { fixed_interval: interval };
}
}

View file

@ -57,9 +57,7 @@ Object {
"fields": Object {
"@tags": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"keyword",
],
@ -70,22 +68,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "@tags",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "string",
},
"@timestamp": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 30,
"customLabel": undefined,
"esTypes": Array [
"date",
],
@ -96,22 +88,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "@timestamp",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "date",
},
"_id": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"_id",
],
@ -122,22 +108,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "_id",
"readFromDocValues": false,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "string",
},
"_source": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"_source",
],
@ -148,22 +128,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "_source",
"readFromDocValues": false,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "_source",
},
"_type": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"_type",
],
@ -174,22 +148,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "_type",
"readFromDocValues": false,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "string",
},
"area": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"geo_shape",
],
@ -200,22 +168,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "area",
"readFromDocValues": false,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "geo_shape",
},
"bytes": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 10,
"customLabel": undefined,
"esTypes": Array [
"long",
],
@ -226,22 +188,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "bytes",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "number",
},
"custom_user_field": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"conflict",
],
@ -252,22 +208,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "custom_user_field",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "conflict",
},
"extension": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"text",
],
@ -278,22 +228,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "extension",
"readFromDocValues": false,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "string",
},
"extension.keyword": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"keyword",
],
@ -304,11 +248,8 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "extension.keyword",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
@ -321,9 +262,7 @@ Object {
},
"geo.coordinates": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"geo_point",
],
@ -334,22 +273,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "geo.coordinates",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "geo_point",
},
"geo.src": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"keyword",
],
@ -360,22 +293,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "geo.src",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "string",
},
"hashed": Object {
"aggregatable": false,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"murmur3",
],
@ -386,22 +313,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "hashed",
"readFromDocValues": false,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "murmur3",
},
"ip": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"ip",
],
@ -412,22 +333,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "ip",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "ip",
},
"machine.os": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"text",
],
@ -438,22 +353,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "machine.os",
"readFromDocValues": false,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "string",
},
"machine.os.raw": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"keyword",
],
@ -464,11 +373,8 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "machine.os.raw",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
@ -481,9 +387,7 @@ Object {
},
"non-filterable": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"text",
],
@ -494,22 +398,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "non-filterable",
"readFromDocValues": false,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": false,
"shortDotsEnable": false,
"subType": undefined,
"type": "string",
},
"non-sortable": Object {
"aggregatable": false,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"text",
],
@ -520,22 +418,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "non-sortable",
"readFromDocValues": false,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": false,
"shortDotsEnable": false,
"subType": undefined,
"type": "string",
},
"phpmemory": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"integer",
],
@ -546,22 +438,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "phpmemory",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "number",
},
"point": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"geo_point",
],
@ -572,22 +458,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "point",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "geo_point",
},
"request_body": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"attachment",
],
@ -598,22 +478,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "request_body",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "attachment",
},
"runtime_field": Object {
"aggregatable": false,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"keyword",
],
@ -623,8 +497,6 @@ Object {
"pattern": "$0,0.[00]",
},
},
"isMapped": undefined,
"lang": undefined,
"name": "runtime_field",
"readFromDocValues": false,
"runtimeField": Object {
@ -633,18 +505,14 @@ Object {
},
"type": "keyword",
},
"script": undefined,
"scripted": false,
"searchable": false,
"shortDotsEnable": false,
"subType": undefined,
"type": "string",
},
"script date": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"date",
],
@ -658,19 +526,15 @@ Object {
"lang": "painless",
"name": "script date",
"readFromDocValues": false,
"runtimeField": undefined,
"script": "1234",
"scripted": true,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "date",
},
"script murmur3": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"murmur3",
],
@ -684,19 +548,15 @@ Object {
"lang": "expression",
"name": "script murmur3",
"readFromDocValues": false,
"runtimeField": undefined,
"script": "1234",
"scripted": true,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "murmur3",
},
"script number": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"long",
],
@ -710,19 +570,15 @@ Object {
"lang": "expression",
"name": "script number",
"readFromDocValues": false,
"runtimeField": undefined,
"script": "1234",
"scripted": true,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "number",
},
"script string": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"text",
],
@ -736,19 +592,15 @@ Object {
"lang": "expression",
"name": "script string",
"readFromDocValues": false,
"runtimeField": undefined,
"script": "'i am a string'",
"scripted": true,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "string",
},
"ssl": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 20,
"customLabel": undefined,
"esTypes": Array [
"boolean",
],
@ -759,22 +611,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "ssl",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "boolean",
},
"time": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 30,
"customLabel": undefined,
"esTypes": Array [
"date",
],
@ -785,22 +631,16 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "time",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "date",
},
"utc_time": Object {
"aggregatable": true,
"conflictDescriptions": undefined,
"count": 0,
"customLabel": undefined,
"esTypes": Array [
"date",
],
@ -811,15 +651,11 @@ Object {
},
},
"isMapped": true,
"lang": undefined,
"name": "utc_time",
"readFromDocValues": true,
"runtimeField": undefined,
"script": undefined,
"scripted": false,
"searchable": true,
"shortDotsEnable": false,
"subType": undefined,
"type": "date",
},
},

View file

@ -13,7 +13,7 @@ import { last, map } from 'lodash';
import { stubbedSavedObjectIndexPattern } from '../data_view.stub';
import { stubLogstashFields } from '../field.stub';
import { DataViewField } from '../fields';
import { RuntimeField, RuntimeTypeExceptComposite } from '../types';
import { FieldSpec, RuntimeField, RuntimeTypeExceptComposite } from '../types';
import { DataView } from './data_view';
class MockFieldFormatter {}
@ -96,6 +96,29 @@ describe('IndexPattern', () => {
});
});
describe('isTSDBMode', () => {
const tsdbField: FieldSpec = {
name: 'tsdb-metric-field',
type: 'number',
aggregatable: true,
searchable: true,
timeSeriesMetric: 'gauge',
};
test('should return false if no fields are tsdb fields', () => {
expect(indexPattern.isTSDBMode()).toBe(false);
});
test('should return true if some fields are tsdb fields', () => {
indexPattern.fields.add(tsdbField);
expect(indexPattern.isTSDBMode()).toBe(true);
});
afterAll(() => {
indexPattern.fields.remove(tsdbField);
});
});
describe('getScriptedFields', () => {
test('should return all scripted fields', () => {
const scriptedNames = stubLogstashFields

View file

@ -348,6 +348,13 @@ export class DataView implements DataViewBase {
return [...this.fields.getAll().filter((field) => field.scripted)];
}
/**
* returns true if dataview contains TSDB fields
*/
isTSDBMode() {
return this.fields.some((field) => field.timeSeriesDimension || field.timeSeriesMetric);
}
/**
* Does the data view have a timestamp field?
*/

View file

@ -47,7 +47,6 @@ Object {
],
},
"count": 1,
"customLabel": undefined,
"esTypes": Array [
"keyword",
],
@ -57,7 +56,6 @@ Object {
"pattern": "$0,0.[00]",
},
},
"isMapped": undefined,
"lang": "java",
"name": "name",
"readFromDocValues": false,
@ -70,7 +68,6 @@ Object {
"script": "script",
"scripted": true,
"searchable": true,
"shortDotsEnable": undefined,
"subType": Object {
"multi": Object {
"parent": "parent",

View file

@ -207,6 +207,40 @@ export class DataViewField implements DataViewFieldBase {
return !!(this.spec.aggregatable || this.scripted);
}
/**
* returns true if field is a TSDB dimension field
*/
public get timeSeriesDimension() {
return this.spec.timeSeriesDimension || false;
}
/**
* returns type of TSDB metric or undefined
*/
public get timeSeriesMetric() {
return this.spec.timeSeriesMetric;
}
/**
* returns list of alloeed fixed intervals
*/
public get fixedInterval() {
return this.spec.fixedInterval;
}
/**
* returns true if the field is of rolled up type
*/
public get isRolledUpField() {
return this.esTypes?.includes('aggregate_metric_double');
}
/**
* return list of allowed time zones
*/
public get timeZone() {
return this.spec.timeZone;
}
/**
* Returns true if field is available via doc values
*/
@ -339,7 +373,7 @@ export class DataViewField implements DataViewFieldBase {
public toSpec(config: ToSpecConfig = {}): FieldSpec {
const { getFormatterForField } = config;
return {
const spec = {
count: this.count,
script: this.script,
lang: this.lang,
@ -357,7 +391,16 @@ export class DataViewField implements DataViewFieldBase {
shortDotsEnable: this.spec.shortDotsEnable,
runtimeField: this.runtimeField,
isMapped: this.isMapped,
timeSeriesDimension: this.spec.timeSeriesDimension,
timeSeriesMetric: this.spec.timeSeriesMetric,
timeZone: this.spec.timeZone,
fixedInterval: this.spec.fixedInterval,
};
// Filter undefined values from the spec
return Object.fromEntries(
Object.entries(spec).filter(([, v]) => typeof v !== 'undefined')
) as FieldSpec;
}
/**

View file

@ -410,6 +410,26 @@ export type FieldSpec = DataViewFieldBase & {
*/
runtimeField?: RuntimeFieldSpec;
/**
* list of allowed field intervals for the field
*/
fixedInterval?: string[];
/**
* List of allowed timezones for the field
*/
timeZone?: string[];
/**
* set to true if field is a TSDB dimension field
*/
timeSeriesDimension?: boolean;
/**
* set if field is a TSDB metric field
*/
timeSeriesMetric?: 'histogram' | 'summary' | 'gauge' | 'counter';
// not persisted
/**

View file

@ -25,6 +25,10 @@ export interface FieldDescriptor {
esTypes: string[];
subType?: FieldSubType;
metadata_field?: boolean;
fixedInterval?: string[];
timeZone?: string[];
timeSeriesMetric?: 'histogram' | 'summary' | 'counter' | 'gauge';
timeSeriesDimension?: boolean;
}
interface FieldSubType {

View file

@ -39,7 +39,15 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
const fields = readFieldCapsResponse(responseClone);
fields.forEach((field) => {
const fieldWithoutOptionalKeys = omit(field, 'conflictDescriptions', 'subType');
const fieldWithoutOptionalKeys = omit(
field,
'conflictDescriptions',
'subType',
'fixedInterval',
'timeZone',
'timeSeriesMetric',
'timeSeriesDimension'
);
expect(Object.keys(fieldWithoutOptionalKeys)).toEqual([
'name',

View file

@ -98,6 +98,8 @@ export function readFieldCapsResponse(
);
});
const timeSeriesMetricProp = uniq(types.map((t) => capsByType[t].time_series_metric));
// If there are multiple types but they all resolve to the same kibana type
// ignore the conflict and carry on (my wayward son)
const uniqueKibanaTypes = uniq(types.map(castEsToKbnFieldTypeName));
@ -124,6 +126,13 @@ export function readFieldCapsResponse(
return agg;
}
let timeSeriesMetricType: 'gauge' | 'counter' | undefined;
if (timeSeriesMetricProp.length === 1 && timeSeriesMetricProp[0] === 'gauge') {
timeSeriesMetricType = 'gauge';
}
if (timeSeriesMetricProp.length === 1 && timeSeriesMetricProp[0] === 'counter') {
timeSeriesMetricType = 'counter';
}
const esType = types[0];
const field = {
name: fieldName,
@ -133,6 +142,10 @@ export function readFieldCapsResponse(
aggregatable: isAggregatable,
readFromDocValues: shouldReadFieldFromDocValues(isAggregatable, esType),
metadata_field: capsByType[types[0]].metadata_field,
fixedInterval: capsByType[types[0]].meta?.fixed_interval,
timeZone: capsByType[types[0]].meta?.time_zone,
timeSeriesMetric: timeSeriesMetricType,
timeSeriesDimension: capsByType[types[0]].time_series_dimension,
};
// This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
agg.array.push(field);