[data views] Allow passing filter through to field caps for contextual field list (#121367)

* first pass at providing filtering

* fix functional tests

* add plugin functional test

* fix test and remove comments

* typescript fixes

* move test to api integration

* move test to api integration

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Matthew Kime 2021-12-21 07:37:00 -06:00 committed by GitHub
parent 45f20e9e37
commit e9a4d6b231
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 270 additions and 114 deletions

View file

@ -10,6 +10,7 @@ import React, { useState, useEffect, useCallback, useRef } from 'react';
import { EuiTitle, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiLoadingSpinner } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import memoizeOne from 'memoize-one';
import { DataViewField } from '../../../data_views/public';
import {
IndexPatternSpec,
@ -200,7 +201,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
}
const fields = await ensureMinimumTime(dataViews.getFieldsForWildcard(getFieldsOptions));
timestampOptions = extractTimeFields(fields, requireTimestampField);
timestampOptions = extractTimeFields(fields as DataViewField[], requireTimestampField);
}
if (currentLoadingTimestampFieldsIdx === currentLoadingTimestampFieldsRef.current) {
setIsLoadingTimestampFields(false);

View file

@ -251,7 +251,7 @@ export class DataViewsService {
* @param options
* @returns FieldSpec[]
*/
getFieldsForWildcard = async (options: GetFieldsOptions) => {
getFieldsForWildcard = async (options: GetFieldsOptions): Promise<FieldSpec[]> => {
const metaFields = await this.config.get(META_FIELDS);
return this.apiClient.getFieldsForWildcard({
pattern: options.pattern,
@ -259,6 +259,7 @@ export class DataViewsService {
type: options.type,
rollupIndex: options.rollupIndex,
allowNoIndex: options.allowNoIndex,
filter: options.filter,
});
};

View file

@ -11,11 +11,14 @@ import { ToastInputFields, ErrorToastOptions } from 'src/core/public/notificatio
// eslint-disable-next-line
import type { SavedObject } from 'src/core/server';
import { KBN_FIELD_TYPES } from '@kbn/field-types';
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { IFieldType } from './fields';
import { RUNTIME_FIELD_TYPES } from './constants';
import { DataViewField } from './fields';
import { FieldFormat, SerializedFieldFormat } from '../../field_formats/common';
export type { QueryDslQueryContainer };
export type FieldFormatMap = Record<string, SerializedFieldFormat>;
export type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];
@ -127,6 +130,7 @@ export interface GetFieldsOptions {
metaFields?: string[];
rollupIndex?: string;
allowNoIndex?: boolean;
filter?: QueryDslQueryContainer;
}
export interface GetFieldsOptionsTimePattern {

View file

@ -49,13 +49,21 @@ export class DataViewsApiClient implements IDataViewsApiClient {
}).then((resp: any) => resp.fields);
}
getFieldsForWildcard({ pattern, metaFields, type, rollupIndex, allowNoIndex }: GetFieldsOptions) {
getFieldsForWildcard({
pattern,
metaFields,
type,
rollupIndex,
allowNoIndex,
filter,
}: GetFieldsOptions) {
return this._request(this._getUrl(['_fields_for_wildcard']), {
pattern,
meta_fields: metaFields,
type,
rollup_index: rollupIndex,
allow_no_index: allowNoIndex,
filter,
}).then((resp: any) => resp.fields || []);
}

View file

@ -8,6 +8,7 @@
import { ElasticsearchClient } from 'kibana/server';
import { keyBy } from 'lodash';
import type { QueryDslQueryContainer } from '../../common/types';
import {
getFieldCapabilities,
@ -55,8 +56,9 @@ export class IndexPatternsFetcher {
fieldCapsOptions?: { allow_no_indices: boolean };
type?: string;
rollupIndex?: string;
filter?: QueryDslQueryContainer;
}): Promise<FieldDescriptor[]> {
const { pattern, metaFields, fieldCapsOptions, type, rollupIndex } = options;
const { pattern, metaFields = [], fieldCapsOptions, type, rollupIndex, filter } = options;
const patternList = Array.isArray(pattern) ? pattern : pattern.split(',');
const allowNoIndices = fieldCapsOptions
? fieldCapsOptions.allow_no_indices
@ -66,14 +68,15 @@ export class IndexPatternsFetcher {
if (patternList.length > 1 && !allowNoIndices) {
patternListActive = await this.validatePatternListActive(patternList);
}
const fieldCapsResponse = await getFieldCapabilities(
this.elasticsearchClient,
patternListActive,
const fieldCapsResponse = await getFieldCapabilities({
callCluster: this.elasticsearchClient,
indices: patternListActive,
metaFields,
{
fieldCapsOptions: {
allow_no_indices: allowNoIndices,
}
);
},
filter,
});
if (type === 'rollup' && rollupIndex) {
const rollupFields: FieldDescriptor[] = [];
const rollupIndexCapabilities = getCapabilitiesForRollupIndices(
@ -120,7 +123,11 @@ export class IndexPatternsFetcher {
if (indices.length === 0) {
throw createNoMatchingIndicesError(pattern);
}
return await getFieldCapabilities(this.elasticsearchClient, indices, metaFields);
return await getFieldCapabilities({
callCluster: this.elasticsearchClient,
indices,
metaFields,
});
}
/**

View file

@ -117,12 +117,12 @@ describe('server/index_patterns/service/lib/es_api', () => {
},
fieldCaps,
};
await callFieldCapsApi(callCluster);
await callFieldCapsApi({ callCluster });
sinon.assert.calledOnce(fieldCaps);
});
it('passes indices directly to es api', async () => {
const football = {};
const indices = ['indexA', 'indexB'];
const fieldCaps = sinon.stub();
const callCluster = {
indices: {
@ -130,9 +130,9 @@ describe('server/index_patterns/service/lib/es_api', () => {
},
fieldCaps,
};
await callFieldCapsApi(callCluster, football);
await callFieldCapsApi({ callCluster, indices });
sinon.assert.calledOnce(fieldCaps);
expect(fieldCaps.args[0][0].index).toBe(football);
expect(fieldCaps.args[0][0].index).toBe(indices);
});
it('returns the es response directly', async () => {
@ -144,7 +144,7 @@ describe('server/index_patterns/service/lib/es_api', () => {
},
fieldCaps,
};
const resp = await callFieldCapsApi(callCluster);
const resp = await callFieldCapsApi({ callCluster });
sinon.assert.calledOnce(fieldCaps);
expect(resp).toBe(football);
});
@ -157,7 +157,7 @@ describe('server/index_patterns/service/lib/es_api', () => {
},
fieldCaps,
};
await callFieldCapsApi(callCluster);
await callFieldCapsApi({ callCluster });
sinon.assert.calledOnce(fieldCaps);
const passedOpts = fieldCaps.args[0][0];
@ -182,7 +182,7 @@ describe('server/index_patterns/service/lib/es_api', () => {
fieldCaps,
};
try {
await callFieldCapsApi(callCluster, indices);
await callFieldCapsApi({ callCluster, indices });
throw new Error('expected callFieldCapsApi() to throw');
} catch (error) {
expect(error).toBe(convertedError);

View file

@ -7,6 +7,7 @@
*/
import { ElasticsearchClient } from 'kibana/server';
import { QueryDslQueryContainer } from '../../../common/types';
import { convertEsError } from './errors';
/**
@ -38,6 +39,13 @@ export async function callIndexAliasApi(
}
}
interface FieldCapsApiParams {
callCluster: ElasticsearchClient;
indices: string[] | string;
fieldCapsOptions?: { allow_no_indices: boolean };
filter?: QueryDslQueryContainer;
}
/**
* Call the fieldCaps API for a list of indices.
*
@ -50,16 +58,21 @@ export async function callIndexAliasApi(
* @param {Object} fieldCapsOptions
* @return {Promise<FieldCapsResponse>}
*/
export async function callFieldCapsApi(
callCluster: ElasticsearchClient,
indices: string[] | string,
fieldCapsOptions: { allow_no_indices: boolean } = { allow_no_indices: false }
) {
export async function callFieldCapsApi(params: FieldCapsApiParams) {
const {
callCluster,
indices,
filter,
fieldCapsOptions = {
allow_no_indices: false,
},
} = params;
try {
return await callCluster.fieldCaps({
index: indices,
fields: '*',
ignore_unavailable: true,
index_filter: filter,
...fieldCapsOptions,
});
} catch (error) {

View file

@ -31,10 +31,15 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
{ 'used to verify that values are directly passed through': true },
];
// assert that the stub was called with the exact `args`, using === matching
const calledWithExactly = (stub, args, matcher = sinon.match.same) => {
sinon.assert.calledWithExactly(stub, ...args.map((arg) => matcher(arg)));
};
const fillUndefinedParams = (args) => ({
callCluster: undefined,
indices: undefined,
fieldCapsOptions: undefined,
filter: undefined,
...args,
});
const getArgsWithCallCluster = (args = {}) => ({ callCluster: callFieldCapsApi, ...args });
const stubDeps = (options = {}) => {
const { esResponse = [], fieldsFromFieldCaps = [], mergeOverrides = identity } = options;
@ -50,9 +55,11 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
it('passes exact `callCluster` and `indices` args through', async () => {
stubDeps();
await getFieldCapabilities(footballs[0], footballs[1]);
const args = getArgsWithCallCluster({ indices: ['index1', 'index2'] });
await getFieldCapabilities(args);
sinon.assert.calledOnce(callFieldCapsApi);
calledWithExactly(callFieldCapsApi, [footballs[0], footballs[1], undefined]);
sinon.assert.calledWithExactly(callFieldCapsApi, fillUndefinedParams(args));
});
});
@ -62,9 +69,11 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
esResponse: footballs[0],
});
await getFieldCapabilities();
const args = getArgsWithCallCluster({ indices: ['index1', 'index2'] });
await getFieldCapabilities(args);
sinon.assert.calledOnce(readFieldCapsResponse);
calledWithExactly(readFieldCapsResponse, [footballs[0]]);
sinon.assert.calledWithExactly(readFieldCapsResponse, footballs[0]);
});
});
@ -76,7 +85,9 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
fieldsFromFieldCaps: fields.map((name) => ({ name })),
});
const fieldNames = (await getFieldCapabilities()).map((field) => field.name);
const fieldNames = (await getFieldCapabilities(getArgsWithCallCluster())).map(
(field) => field.name
);
expect(fieldNames).toEqual(fields);
});
@ -88,7 +99,9 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
fieldsFromFieldCaps: shuffle(letters.map((name) => ({ name }))),
});
const fieldNames = (await getFieldCapabilities()).map((field) => field.name);
const fieldNames = (await getFieldCapabilities(getArgsWithCallCluster())).map(
(field) => field.name
);
expect(fieldNames).toEqual(sortedLetters);
});
});
@ -99,7 +112,9 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
fieldsFromFieldCaps: [{ name: 'foo' }, { name: 'bar' }],
});
const resp = await getFieldCapabilities(undefined, undefined, ['meta1', 'meta2']);
const args = getArgsWithCallCluster({ metaFields: ['meta1', 'meta2'] });
const resp = await getFieldCapabilities(args);
expect(resp).toHaveLength(4);
expect(resp.map((field) => field.name)).toEqual(['bar', 'foo', 'meta1', 'meta2']);
});
@ -126,7 +141,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
fieldsFromFieldCaps: [field],
});
const resp = await getFieldCapabilities();
const resp = await getFieldCapabilities(getArgsWithCallCluster());
expect(resp).toHaveLength(1);
expect(resp[0]).toHaveProperty(property);
expect(resp[0][property]).not.toBe(footballs[0]);
@ -149,7 +164,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
stubDeps({ fieldsFromFieldCaps });
sinon.assert.notCalled(mergeOverrides);
await getFieldCapabilities();
await getFieldCapabilities(getArgsWithCallCluster());
sinon.assert.calledThrice(mergeOverrides);
expect(mergeOverrides.args[0][0]).toHaveProperty('name', 'foo');
@ -170,7 +185,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
},
});
expect(await getFieldCapabilities()).toEqual([
expect(await getFieldCapabilities(getArgsWithCallCluster())).toEqual([
{ notFieldAnymore: 1 },
{ notFieldAnymore: 1 },
]);

View file

@ -13,6 +13,15 @@ import { callFieldCapsApi } from '../es_api';
import { readFieldCapsResponse } from './field_caps_response';
import { mergeOverrides } from './overrides';
import { FieldDescriptor } from '../../index_patterns_fetcher';
import { QueryDslQueryContainer } from '../../../../common/types';
interface FieldCapabilitiesParams {
callCluster: ElasticsearchClient;
indices: string | string[];
metaFields: string[];
fieldCapsOptions?: { allow_no_indices: boolean };
filter?: QueryDslQueryContainer;
}
/**
* Get the field capabilities for field in `indices`, excluding
@ -24,13 +33,9 @@ import { FieldDescriptor } from '../../index_patterns_fetcher';
* @param {Object} fieldCapsOptions
* @return {Promise<Array<FieldDescriptor>>}
*/
export async function getFieldCapabilities(
callCluster: ElasticsearchClient,
indices: string | string[] = [],
metaFields: string[] = [],
fieldCapsOptions?: { allow_no_indices: boolean }
) {
const esFieldCaps = await callFieldCapsApi(callCluster, indices, fieldCapsOptions);
export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
const { callCluster, indices = [], fieldCapsOptions, filter, metaFields = [] } = params;
const esFieldCaps = await callFieldCapsApi({ callCluster, indices, fieldCapsOptions, filter });
const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps.body), 'name');
const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName)

View file

@ -0,0 +1,121 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { schema } from '@kbn/config-schema';
import {
IRouter,
StartServicesAccessor,
RequestHandler,
RouteValidatorFullConfig,
} from '../../../core/server';
import type { DataViewsServerPluginStart, DataViewsServerPluginStartDependencies } from './types';
import { IndexPatternsFetcher } from './fetcher';
const parseMetaFields = (metaFields: string | string[]) => {
let parsedFields: string[] = [];
if (typeof metaFields === 'string') {
parsedFields = JSON.parse(metaFields);
} else {
parsedFields = metaFields;
}
return parsedFields;
};
const path = '/api/index_patterns/_fields_for_wildcard';
type IBody = { index_filter?: any } | undefined;
interface IQuery {
pattern: string;
meta_fields: string[];
type?: string;
rollup_index?: string;
allow_no_index?: boolean;
}
const validate: RouteValidatorFullConfig<{}, IQuery, IBody> = {
query: schema.object({
pattern: schema.string(),
meta_fields: schema.oneOf([schema.string(), schema.arrayOf(schema.string())], {
defaultValue: [],
}),
type: schema.maybe(schema.string()),
rollup_index: schema.maybe(schema.string()),
allow_no_index: schema.maybe(schema.boolean()),
}),
// not available to get request
body: schema.maybe(schema.object({ index_filter: schema.any() })),
};
const handler: RequestHandler<{}, IQuery, IBody> = async (context, request, response) => {
const { asCurrentUser } = context.core.elasticsearch.client;
const indexPatterns = new IndexPatternsFetcher(asCurrentUser);
const {
pattern,
meta_fields: metaFields,
type,
rollup_index: rollupIndex,
allow_no_index: allowNoIndex,
} = request.query;
// not available to get request
const filter = request.body?.index_filter;
let parsedFields: string[] = [];
try {
parsedFields = parseMetaFields(metaFields);
} catch (error) {
return response.badRequest();
}
try {
const fields = await indexPatterns.getFieldsForWildcard({
pattern,
metaFields: parsedFields,
type,
rollupIndex,
fieldCapsOptions: {
allow_no_indices: allowNoIndex || false,
},
filter,
});
return response.ok({
body: { fields },
headers: {
'content-type': 'application/json',
},
});
} catch (error) {
if (
typeof error === 'object' &&
!!error?.isBoom &&
!!error?.output?.payload &&
typeof error?.output?.payload === 'object'
) {
const payload = error?.output?.payload;
return response.notFound({
body: {
message: payload.message,
attributes: payload,
},
});
} else {
return response.notFound();
}
}
};
export const registerFieldForWildcard = (
router: IRouter,
getStartServices: StartServicesAccessor<
DataViewsServerPluginStartDependencies,
DataViewsServerPluginStart
>
) => {
router.put({ path, validate }, handler);
router.get({ path, validate }, handler);
};

View file

@ -30,6 +30,7 @@ export class IndexPatternsApiServer implements IIndexPatternsApiClient {
type,
rollupIndex,
allowNoIndex,
filter,
}: GetFieldsOptions) {
const indexPatterns = new IndexPatternsFetcher(this.esClient, allowNoIndex);
return await indexPatterns
@ -38,6 +39,7 @@ export class IndexPatternsApiServer implements IIndexPatternsApiClient {
metaFields,
type,
rollupIndex,
filter,
})
.catch((err) => {
if (

View file

@ -27,6 +27,7 @@ import { registerDeleteRuntimeFieldRoute } from './routes/runtime_fields/delete_
import { registerPutRuntimeFieldRoute } from './routes/runtime_fields/put_runtime_field';
import { registerUpdateRuntimeFieldRoute } from './routes/runtime_fields/update_runtime_field';
import { registerHasUserIndexPatternRoute } from './routes/has_user_index_pattern';
import { registerFieldForWildcard } from './fields_for';
export function registerRoutes(
http: HttpServiceSetup,
@ -72,76 +73,7 @@ export function registerRoutes(
registerPutRuntimeFieldRoute(router, getStartServices);
registerUpdateRuntimeFieldRoute(router, getStartServices);
router.get(
{
path: '/api/index_patterns/_fields_for_wildcard',
validate: {
query: schema.object({
pattern: schema.string(),
meta_fields: schema.oneOf([schema.string(), schema.arrayOf(schema.string())], {
defaultValue: [],
}),
type: schema.maybe(schema.string()),
rollup_index: schema.maybe(schema.string()),
allow_no_index: schema.maybe(schema.boolean()),
}),
},
},
async (context, request, response) => {
const { asCurrentUser } = context.core.elasticsearch.client;
const indexPatterns = new IndexPatternsFetcher(asCurrentUser);
const {
pattern,
meta_fields: metaFields,
type,
rollup_index: rollupIndex,
allow_no_index: allowNoIndex,
} = request.query;
let parsedFields: string[] = [];
try {
parsedFields = parseMetaFields(metaFields);
} catch (error) {
return response.badRequest();
}
try {
const fields = await indexPatterns.getFieldsForWildcard({
pattern,
metaFields: parsedFields,
type,
rollupIndex,
fieldCapsOptions: {
allow_no_indices: allowNoIndex || false,
},
});
return response.ok({
body: { fields },
headers: {
'content-type': 'application/json',
},
});
} catch (error) {
if (
typeof error === 'object' &&
!!error?.isBoom &&
!!error?.output?.payload &&
typeof error?.output?.payload === 'object'
) {
const payload = error?.output?.payload;
return response.notFound({
body: {
message: payload.message,
attributes: payload,
},
});
} else {
return response.notFound();
}
}
}
);
registerFieldForWildcard(router, getStartServices);
router.get(
{

View file

@ -0,0 +1,45 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const es = getService('es');
describe('filter fields', () => {
before(async () => {
await es.index({
index: 'helloworld1',
refresh: true,
id: 'helloworld',
body: { hello: 'world' },
});
await es.index({
index: 'helloworld2',
refresh: true,
id: 'helloworld2',
body: { bye: 'world' },
});
});
it('can filter', async () => {
const a = await supertest
.put('/api/index_patterns/_fields_for_wildcard')
.query({ pattern: 'helloworld*' })
.send({ index_filter: { exists: { field: 'bye' } } });
const fieldNames = a.body.fields.map((fld: { name: string }) => fld.name);
expect(fieldNames.indexOf('bye') > -1).to.be(true);
expect(fieldNames.indexOf('hello') === -1).to.be(true);
});
});
}

View file

@ -11,5 +11,6 @@ export default function ({ loadTestFile }) {
loadTestFile(require.resolve('./params'));
loadTestFile(require.resolve('./conflicts'));
loadTestFile(require.resolve('./response'));
loadTestFile(require.resolve('./filter'));
});
}

View file

@ -65,6 +65,7 @@ const resolveLegacyReference = async (
timestampField: TIMESTAMP_FIELD,
tiebreakerField: TIEBREAKER_FIELD,
messageField: sourceConfiguration.fields.message,
// @ts-ignore
fields,
runtimeMappings: {},
columns: sourceConfiguration.logColumns,