[6.x] Enable KQL support for TSVB (#26006) (#29828)

* Enable KQL support for TSVB (#26006)

* Use buildEsQuery for table, series and annotations

* Fix query test.

Using the buildEsQuery changed a bit the order of the must.bool array on the query

* Remove console.log

* Remove console.error and comment

* Fix wrong merge of PR #26510

* Fix default/empty index_pattern

When the user save the visualization without configuring
manually an index_pattern, a default empty string is used
instead of the default pattern. This leads to an empty
visualization after the refactoring done in #24832.
Now it will update the index_pattern field with the default
index pattern in visualize editor and on dashboard.

* Remove unnecessary wrapping query in an array

* Enable query bar on tsvb

* Remove unnecessary setDefaultIndexPattern

After fixing the null index pattern issue in kql
there is no need to use set the default index pattern
before rendering.

* fix(tsvb-server): Ignore query bar search on ignore_global_filters param

This commit mimic the behaviour of the ignore_global_filters currently used with lucene syntax:
if you enable the ignore_global_filter option on data or on annotations, data or annotations
are filtered out when using the filter bar and the search bar

* Disable showQueryBar

* Fix non array query value on request
This commit is contained in:
Marco Vettorello 2019-02-05 15:45:12 +01:00 committed by GitHub
parent c4129152c7
commit deccf5d794
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 276 additions and 181 deletions

View file

@ -5,7 +5,7 @@
display: flex;
align-items: center;
justify-content: center;
flex-direction: column;
// Calculate colors similar to EuiCallout
$tempBackgroundColor: tintOrShade($euiColorDanger, 90%, 70%);
background-color: $tempBackgroundColor;

View file

@ -20,7 +20,6 @@
import { validateInterval } from '../lib/validate_interval';
import { timezoneProvider } from 'ui/vis/lib/timezone';
import { timefilter } from 'ui/timefilter';
import { buildEsQuery } from '@kbn/es-query';
const MetricsRequestHandlerProvider = function (Private, Notifier, config, $http, i18n) {
const notify = new Notifier({ location: i18n('tsvb.requestHandler.notifier.locationNameTitle', { defaultMessage: 'Metrics' }) });
@ -35,14 +34,11 @@ const MetricsRequestHandlerProvider = function (Private, Notifier, config, $http
const parsedTimeRange = timefilter.calculateBounds(timeRange);
const scaledDataFormat = config.get('dateFormat:scaled');
const dateFormat = config.get('dateFormat');
const esQueryConfigs = {
allowLeadingWildcards: config.get('query:allowLeadingWildcards'),
queryStringOptions: config.get('query:queryString:options'),
};
if (panel && panel.id) {
const params = {
timerange: { timezone, ...parsedTimeRange },
filters: [buildEsQuery(undefined, [query], filters, esQueryConfigs)],
query: Array.isArray(query) ? query : [query],
filters,
panels: [panel],
state: uiStateObj
};

View file

@ -20,8 +20,8 @@
import buildProcessorFunction from './build_processor_function';
import processors from './request_processors/annotations';
export default function buildAnnotationRequest(req, panel, annotation) {
const processor = buildProcessorFunction(processors, req, panel, annotation);
export default function buildAnnotationRequest(req, panel, annotation, esQueryConfig, indexPattern) {
const processor = buildProcessorFunction(processors, req, panel, annotation, esQueryConfig, indexPattern);
const doc = processor({});
return doc;
}

View file

@ -19,6 +19,7 @@
import buildAnnotationRequest from './build_annotation_request';
import handleAnnotationResponse from './handle_annotation_response';
import { getIndexPatternObject } from './helpers/get_index_pattern';
function validAnnotation(annotation) {
return annotation.index_pattern &&
@ -28,27 +29,19 @@ function validAnnotation(annotation) {
annotation.template;
}
export default async (req, panel) => {
export default async (req, panel, esQueryConfig) => {
const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data');
const bodies = panel.annotations
const bodiesPromises = panel.annotations
.filter(validAnnotation)
.map(annotation => {
const indexPattern = annotation.index_pattern;
const bodies = [];
bodies.push({
index: indexPattern,
ignoreUnavailable: true,
});
const body = buildAnnotationRequest(req, panel, annotation);
body.timeout = '90s';
bodies.push(body);
return bodies;
return getAnnotationBody(req, panel, annotation, esQueryConfig);
});
if (!bodies.length) return { responses: [] };
const bodies = await Promise.all(bodiesPromises);
if (!bodies.length) {
return {
responses: [],
};
}
try {
const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen');
const resp = await callWithRequest(req, 'msearch', {
@ -68,6 +61,18 @@ export default async (req, panel) => {
if (error.message === 'missing-indices') return { responses: [] };
throw error;
}
};
async function getAnnotationBody(req, panel, annotation, esQueryConfig) {
const indexPatternString = annotation.index_pattern;
const indexPatternObject = await getIndexPatternObject(req, indexPatternString);
const request = buildAnnotationRequest(req, panel, annotation, esQueryConfig, indexPatternObject);
request.timeout = '90s';
return [
{
index: indexPatternString,
ignoreUnavailable: true,
},
request,
];
}

View file

@ -21,7 +21,9 @@ import { getTableData } from './get_table_data';
import { getSeriesData } from './get_series_data';
export default function getPanelData(req) {
return panel => {
if (panel.type === 'table') return getTableData(req, panel);
if (panel.type === 'table') {
return getTableData(req, panel);
}
return getSeriesData(req, panel);
};
}

View file

@ -21,36 +21,45 @@ import getRequestParams from './series/get_request_params';
import handleResponseBody from './series/handle_response_body';
import handleErrorResponse from './handle_error_response';
import getAnnotations from './get_annotations';
import { getEsQueryConfig } from './helpers/get_es_query_uisettings';
export async function getSeriesData(req, panel) {
const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data');
const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen');
const bodies = panel.series.map(series => getRequestParams(req, panel, series));
const params = {
rest_total_hits_as_int: true,
ignore_throttled: !includeFrozen,
body: bodies.reduce((acc, items) => acc.concat(items), [])
};
return callWithRequest(req, 'msearch', params)
.then(resp => {
const series = resp.responses.map(handleResponseBody(panel));
return {
[panel.id]: {
id: panel.id,
series: series.reduce((acc, series) => acc.concat(series), [])
}
};
})
.then(resp => {
if (!panel.annotations || panel.annotations.length === 0) return resp;
return getAnnotations(req, panel).then(annotations => {
resp[panel.id].annotations = annotations;
const esQueryConfig = await getEsQueryConfig(req);
try {
const bodiesPromises = panel.series.map(series => getRequestParams(req, panel, series, esQueryConfig));
const bodies = await Promise.all(bodiesPromises);
const params = {
rest_total_hits_as_int: true,
ignore_throttled: !includeFrozen,
body: bodies.reduce((acc, items) => acc.concat(items), [])
};
return callWithRequest(req, 'msearch', params)
.then(resp => {
const series = resp.responses.map(handleResponseBody(panel));
return {
[panel.id]: {
id: panel.id,
series: series.reduce((acc, series) => acc.concat(series), [])
}
};
})
.then(resp => {
if (!panel.annotations || panel.annotations.length === 0) return resp;
return getAnnotations(req, panel, esQueryConfig).then(annotations => {
resp[panel.id].annotations = annotations;
return resp;
});
})
.then(resp => {
resp.type = panel.type;
return resp;
});
})
.then(resp => {
resp.type = panel.type;
return resp;
})
.catch(handleErrorResponse(panel));
})
.catch(handleErrorResponse(panel));
} catch(e) {
return handleErrorResponse(e);
}
}

View file

@ -16,18 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
import { get } from 'lodash';
import buildRequestBody from './table/build_request_body';
import handleErrorResponse from './handle_error_response';
import { get } from 'lodash';
import processBucket from './table/process_bucket';
import { getIndexPatternObject } from './helpers/get_index_pattern';
import { getEsQueryConfig } from './helpers/get_es_query_uisettings';
export async function getTableData(req, panel) {
const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data');
const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen');
const indexPatternString = panel.index_pattern;
const esQueryConfig = await getEsQueryConfig(req);
const indexPatternObject = await getIndexPatternObject(req, indexPatternString);
const params = {
index: panel.index_pattern,
index: indexPatternString,
ignore_throttled: !includeFrozen,
body: buildRequestBody(req, panel)
body: buildRequestBody(req, panel, esQueryConfig, indexPatternObject)
};
try {
const resp = await callWithRequest(req, 'search', params);

View file

@ -0,0 +1,28 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export async function getEsQueryConfig(req) {
const uiSettings = req.getUiSettingsService();
const allowLeadingWildcards = await uiSettings.get('query:allowLeadingWildcards');
const queryStringOptions = await uiSettings.get('query:queryString:options');
return {
allowLeadingWildcards,
queryStringOptions: JSON.parse(queryStringOptions),
};
}

View file

@ -0,0 +1,41 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export async function getIndexPatternObject(req, indexPatternString) {
// getting the matching index pattern
const savedObjectClient = req.getSavedObjectsClient();
const indexPatternObjects = await savedObjectClient.find({
type: 'index-pattern',
fields: ['title', 'fields'],
search: `"${indexPatternString}"`,
search_fields: ['title'],
});
// getting the index pattern fields
const indexPatterns = indexPatternObjects.saved_objects
.filter(obj => obj.attributes.title === indexPatternString)
.map(indexPattern => {
const { title, fields } = indexPattern.attributes;
return {
title,
fields: JSON.parse(fields),
};
});
return indexPatterns.length === 1 ? indexPatterns[0] : null;
}

View file

@ -19,29 +19,26 @@
import getBucketSize from '../../helpers/get_bucket_size';
import getTimerange from '../../helpers/get_timerange';
export default function query(req, panel, annotation) {
import { buildEsQuery } from '@kbn/es-query';
export default function query(req, panel, annotation, esQueryConfig, indexPattern) {
return next => doc => {
const timeField = annotation.time_field;
const {
bucketSize
} = getBucketSize(req, 'auto');
const { bucketSize } = getBucketSize(req, 'auto');
const { from, to } = getTimerange(req);
doc.size = 0;
doc.query = {
bool: {
must: []
}
};
const queries = !annotation.ignore_global_filters ? req.payload.query : [];
const filters = !annotation.ignore_global_filters ? req.payload.filters : [];
doc.query = buildEsQuery(indexPattern, queries, filters, esQueryConfig);
const timerange = {
range: {
[timeField]: {
gte: from.valueOf(),
lte: to.valueOf() - (bucketSize * 1000),
lte: to.valueOf() - bucketSize * 1000,
format: 'epoch_millis',
}
}
},
},
};
doc.query.bool.must.push(timerange);
@ -49,22 +46,17 @@ export default function query(req, panel, annotation) {
doc.query.bool.must.push({
query_string: {
query: annotation.query_string,
analyze_wildcard: true
}
analyze_wildcard: true,
},
});
}
const globalFilters = req.payload.filters;
if (!annotation.ignore_global_filters) {
doc.query.bool.must = doc.query.bool.must.concat(globalFilters);
}
if (!annotation.ignore_panel_filters && panel.filter) {
doc.query.bool.must.push({
query_string: {
query: panel.filter,
analyze_wildcard: true
}
analyze_wildcard: true,
},
});
}
@ -76,7 +68,5 @@ export default function query(req, panel, annotation) {
}
return next(doc);
};
}

View file

@ -26,6 +26,10 @@ describe('query(req, panel, series)', () => {
let panel;
let series;
let req;
const config = {
allowLeadingWildcards: true,
queryStringOptions: {},
};
beforeEach(() => {
req = {
payload: {
@ -45,17 +49,18 @@ describe('query(req, panel, series)', () => {
it('calls next when finished', () => {
const next = sinon.spy();
query(req, panel, series)(next)({});
query(req, panel, series, config)(next)({});
expect(next.calledOnce).to.equal(true);
});
it('returns doc with query for timerange', () => {
const next = doc => doc;
const doc = query(req, panel, series)(next)({});
const doc = query(req, panel, series, config)(next)({});
expect(doc).to.eql({
size: 0,
query: {
bool: {
filter: [],
must: [
{
range: {
@ -66,7 +71,9 @@ describe('query(req, panel, series)', () => {
}
}
}
]
],
must_not: [],
should: [],
}
}
});
@ -75,11 +82,12 @@ describe('query(req, panel, series)', () => {
it('returns doc with query for timerange (offset by 1h)', () => {
series.offset_time = '1h';
const next = doc => doc;
const doc = query(req, panel, series)(next)({});
const doc = query(req, panel, series, config)(next)({});
expect(doc).to.eql({
size: 0,
query: {
bool: {
filter: [],
must: [
{
range: {
@ -90,7 +98,9 @@ describe('query(req, panel, series)', () => {
}
}
}
]
],
must_not: [],
should: [],
}
}
});
@ -111,21 +121,13 @@ describe('query(req, panel, series)', () => {
}
];
const next = doc => doc;
const doc = query(req, panel, series)(next)({});
const doc = query(req, panel, series, config)(next)({});
expect(doc).to.eql({
size: 0,
query: {
bool: {
filter: [],
must: [
{
range: {
timestamp: {
gte: 1483228800000,
lte: 1483232400000,
format: 'epoch_millis'
}
}
},
{
bool: {
must: [
@ -136,8 +138,19 @@ describe('query(req, panel, series)', () => {
}
]
}
}
]
},
{
range: {
timestamp: {
gte: 1483228800000,
lte: 1483232400000,
format: 'epoch_millis'
}
}
},
],
must_not: [],
should: [],
}
}
});
@ -146,11 +159,12 @@ describe('query(req, panel, series)', () => {
it('returns doc with series filter', () => {
series.filter = 'host:web-server';
const next = doc => doc;
const doc = query(req, panel, series)(next)({});
const doc = query(req, panel, series, config)(next)({});
expect(doc).to.eql({
size: 0,
query: {
bool: {
filter: [],
must: [
{
range: {
@ -166,8 +180,10 @@ describe('query(req, panel, series)', () => {
query: series.filter,
analyze_wildcard: true
}
}
]
},
],
must_not: [],
should: [],
}
}
});
@ -188,21 +204,13 @@ describe('query(req, panel, series)', () => {
];
panel.filter = 'host:web-server';
const next = doc => doc;
const doc = query(req, panel, series)(next)({});
const doc = query(req, panel, series, config)(next)({});
expect(doc).to.eql({
size: 0,
query: {
bool: {
filter: [],
must: [
{
range: {
timestamp: {
gte: 1483228800000,
lte: 1483232400000,
format: 'epoch_millis'
}
}
},
{
bool: {
must: [
@ -214,13 +222,24 @@ describe('query(req, panel, series)', () => {
]
}
},
{
range: {
timestamp: {
gte: 1483228800000,
lte: 1483232400000,
format: 'epoch_millis'
}
}
},
{
query_string: {
query: panel.filter,
analyze_wildcard: true
}
}
]
],
must_not: [],
should: [],
}
}
});
@ -243,11 +262,12 @@ describe('query(req, panel, series)', () => {
panel.filter = 'host:web-server';
panel.ignore_global_filter = true;
const next = doc => doc;
const doc = query(req, panel, series)(next)({});
const doc = query(req, panel, series, config)(next)({});
expect(doc).to.eql({
size: 0,
query: {
bool: {
filter: [],
must: [
{
range: {
@ -263,8 +283,10 @@ describe('query(req, panel, series)', () => {
query: panel.filter,
analyze_wildcard: true
}
}
]
},
],
must_not: [],
should: [],
}
}
});

View file

@ -19,17 +19,17 @@
import offsetTime from '../../offset_time';
import getIntervalAndTimefield from '../../get_interval_and_timefield';
export default function query(req, panel, series) {
import { buildEsQuery } from '@kbn/es-query';
export default function query(req, panel, series, esQueryConfig, indexPattern) {
return next => doc => {
const { timeField } = getIntervalAndTimefield(panel, series);
const { from, to } = offsetTime(req, series.offset_time);
doc.size = 0;
doc.query = {
bool: {
must: []
}
};
const queries = !panel.ignore_global_filter ? req.payload.query : [];
const filters = !panel.ignore_global_filter ? req.payload.filters : [];
doc.query = buildEsQuery(indexPattern, queries, filters, esQueryConfig);
const timerange = {
range: {
@ -37,22 +37,17 @@ export default function query(req, panel, series) {
gte: from.valueOf(),
lte: to.valueOf(),
format: 'epoch_millis',
}
}
},
},
};
doc.query.bool.must.push(timerange);
const globalFilters = req.payload.filters;
if (globalFilters && !panel.ignore_global_filter) {
doc.query.bool.must = doc.query.bool.must.concat(globalFilters);
}
if (panel.filter) {
doc.query.bool.must.push({
query_string: {
query: panel.filter,
analyze_wildcard: true
}
analyze_wildcard: true,
},
});
}
@ -60,12 +55,11 @@ export default function query(req, panel, series) {
doc.query.bool.must.push({
query_string: {
query: series.filter,
analyze_wildcard: true
}
analyze_wildcard: true,
},
});
}
return next(doc);
};
}

View file

@ -16,20 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
import { buildEsQuery } from '@kbn/es-query';
import getTimerange from '../../helpers/get_timerange';
import getIntervalAndTimefield from '../../get_interval_and_timefield';
export default function query(req, panel) {
export default function query(req, panel, esQueryConfig, indexPattern) {
return next => doc => {
const { timeField } = getIntervalAndTimefield(panel);
const { from, to } = getTimerange(req);
doc.size = 0;
doc.query = {
bool: {
must: []
}
};
const queries = !panel.ignore_global_filter ? req.payload.query : [];
const filters = !panel.ignore_global_filter ? req.payload.filters : [];
doc.query = buildEsQuery(indexPattern, queries, filters, esQueryConfig);
const timerange = {
range: {
@ -37,26 +37,20 @@ export default function query(req, panel) {
gte: from.valueOf(),
lte: to.valueOf(),
format: 'epoch_millis',
}
}
},
},
};
doc.query.bool.must.push(timerange);
const globalFilters = req.payload.filters;
if (globalFilters && !panel.ignore_global_filter) {
doc.query.bool.must = doc.query.bool.must.concat(globalFilters);
}
if (panel.filter) {
doc.query.bool.must.push({
query_string: {
query: panel.filter,
analyze_wildcard: true
}
analyze_wildcard: true,
},
});
}
return next(doc);
};
}

View file

@ -82,21 +82,17 @@ describe('buildRequestBody(req)', () => {
it('returns a valid body', () => {
const panel = body.panels[0];
const series = panel.series[0];
const doc = buildRequestBody({ payload: body }, panel, series);
const config = {
allowLeadingWildcards: true,
queryStringOptions: {},
};
const doc = buildRequestBody({ payload: body }, panel, series, config);
expect(doc).to.eql({
size: 0,
query: {
bool: {
filter: [],
must: [
{
range: {
'@timestamp': {
gte: 1485463055881,
lte: 1485463955881,
format: 'epoch_millis'
}
}
},
{
bool: {
must: [
@ -109,8 +105,19 @@ describe('buildRequestBody(req)', () => {
],
must_not: []
}
}
]
},
{
range: {
'@timestamp': {
gte: 1485463055881,
lte: 1485463955881,
format: 'epoch_millis'
}
}
},
],
must_not: [],
should: [],
}
},
aggs: {

View file

@ -20,8 +20,8 @@
import buildProcessorFunction from '../build_processor_function';
import processors from '../request_processors/series';
function buildRequestBody(req, panel, series) {
const processor = buildProcessorFunction(processors, req, panel, series);
function buildRequestBody(req, panel, series, esQueryConfig, indexPattern) {
const processor = buildProcessorFunction(processors, req, panel, series, esQueryConfig, indexPattern);
const doc = processor({});
return doc;
}

View file

@ -18,18 +18,18 @@
*/
import buildRequestBody from './build_request_body';
import { getIndexPatternObject } from '../helpers/get_index_pattern';
export default (req, panel, series) => {
const indexPattern = series.override_index_pattern && series.series_index_pattern || panel.index_pattern;
const bodies = [];
bodies.push({
index: indexPattern,
ignoreUnavailable: true,
});
const body = buildRequestBody(req, panel, series);
body.timeout = '90s';
bodies.push(body);
return bodies;
export default async (req, panel, series, esQueryConfig) => {
const indexPatternString = series.override_index_pattern && series.series_index_pattern || panel.index_pattern;
const indexPatternObject = await getIndexPatternObject(req, indexPatternString);
const request = buildRequestBody(req, panel, series, esQueryConfig, indexPatternObject);
request.timeout = '90s';
return [
{
index: indexPatternString,
ignoreUnavailable: true,
},
request,
];
};

View file

@ -20,8 +20,8 @@
import buildProcessorFunction from '../build_processor_function';
import processors from '../request_processors/table';
function buildRequestBody(req, panel) {
const processor = buildProcessorFunction(processors, req, panel);
function buildRequestBody(req, panel, esQueryConfig, indexPattern) {
const processor = buildProcessorFunction(processors, req, panel, esQueryConfig, indexPattern);
const doc = processor({});
return doc;
}