github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Lens] Move field existence from Lens to UnifiedFieldList plugin (#139453)

Browse source 
* [Lens] move field existence from to unified field list plugin

* [Lens] update readme, move integration tests

* [Lens] update wording paths

* [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix'

* [Discover] fix loader tests, clean up code

* [Discover] update datapanel tests, clean up code

* [Discover] remove comments

* [Discover] fix problem with filters

* [Lens] apply suggestions

* [Discover] remove spread

* [Discover] fix type checks

Co-authored-by: Joe Reuter <johannes.reuter@elastic.co>
This commit is contained in:
Dmitry Tomashevich 2022-09-08 16:59:33 +03:00 • committed by GitHub
parent 5c9a11a25f
commit e11bea9178
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 778 additions and 614 deletions
Download patch file Download diff file Expand all files Collapse all files

4
x-pack/plugins/lens/server/index.ts
View file

@ -8,12 +8,10 @@
// TODO: https://github.com/elastic/kibana/issues/110891
/* eslint-disable @kbn/eslint/no_export_all */
import { PluginInitializerContext } from '@kbn/core/server';
import { LensServerPlugin } from './plugin';
export type { LensServerPluginSetup } from './plugin';
export * from './plugin';
export * from './migrations/types';
export const plugin = (initializerContext: PluginInitializerContext) =>
new LensServerPlugin(initializerContext);
export const plugin = () => new LensServerPlugin();

8
x-pack/plugins/lens/server/plugin.tsx
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { Plugin, CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/server';
import { Plugin, CoreSetup, CoreStart } from '@kbn/core/server';
import { PluginStart as DataViewsServerPluginStart } from '@kbn/data-views-plugin/server';
import {
PluginStart as DataPluginStart,
@ -21,8 +21,6 @@ import {
} from '@kbn/task-manager-plugin/server';
import { EmbeddableSetup } from '@kbn/embeddable-plugin/server';
import { DataViewPersistableStateService } from '@kbn/data-views-plugin/common';
import { setupRoutes } from './routes';
import { getUiSettings } from './ui_settings';
import { setupSavedObjects } from './saved_objects';
import { setupExpressions } from './expressions';
import { makeLensEmbeddableFactory } from './embeddable/make_lens_embeddable_factory';
@ -59,16 +57,14 @@ export interface LensServerPluginSetup {
export class LensServerPlugin implements Plugin<LensServerPluginSetup, {}, {}, {}> {
private customVisualizationMigrations: CustomVisualizationMigrations = {};
constructor(private initializerContext: PluginInitializerContext) {}
constructor() {}
setup(core: CoreSetup<PluginStartContract>, plugins: PluginSetupContract) {
const getFilterMigrations = plugins.data.query.filterManager.getAllMigrations.bind(
plugins.data.query.filterManager
);
setupSavedObjects(core, getFilterMigrations, this.customVisualizationMigrations);
setupRoutes(core, this.initializerContext.logger.get());
setupExpressions(core, plugins.expressions);
core.uiSettings.register(getUiSettings());
const lensEmbeddableFactory = makeLensEmbeddableFactory(
getFilterMigrations,

171
x-pack/plugins/lens/server/routes/existing_fields.test.ts
View file

@ -1,171 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { DataView } from '@kbn/data-views-plugin/common';
import { legacyExistingFields, existingFields, Field, buildFieldList } from './existing_fields';
describe('existingFields', () => {
it('should remove missing fields by matching names', () => {
expect(
existingFields(
[
{ name: 'a', aggregatable: true, searchable: true, type: 'string' },
{ name: 'b', aggregatable: true, searchable: true, type: 'string' },
],
[
{ name: 'a', isScript: false, isMeta: false },
{ name: 'b', isScript: false, isMeta: true },
{ name: 'c', isScript: false, isMeta: false },
]
)
).toEqual(['a', 'b']);
});
it('should keep scripted and runtime fields', () => {
expect(
existingFields(
[{ name: 'a', aggregatable: true, searchable: true, type: 'string' }],
[
{ name: 'a', isScript: false, isMeta: false },
{ name: 'b', isScript: true, isMeta: false },
{ name: 'c', runtimeField: { type: 'keyword' }, isMeta: false, isScript: false },
{ name: 'd', isMeta: true, isScript: false },
]
)
).toEqual(['a', 'b', 'c']);
});
});
describe('legacyExistingFields', () => {
function field(opts: string | Partial<Field>): Field {
const obj = typeof opts === 'object' ? opts : {};
const name = (typeof opts === 'string' ? opts : opts.name) || 'test';
return {
name,
isScript: false,
isMeta: false,
...obj,
};
}
function searchResults(fields: Record<string, unknown[]> = {}) {
return { fields, _index: '_index', _id: '_id' };
}
it('should handle root level fields', () => {
const result = legacyExistingFields(
[searchResults({ foo: ['bar'] }), searchResults({ baz: [0] })],
[field('foo'), field('bar'), field('baz')]
);
expect(result).toEqual(['foo', 'baz']);
});
it('should handle basic arrays, ignoring empty ones', () => {
const result = legacyExistingFields(
[searchResults({ stuff: ['heyo', 'there'], empty: [] })],
[field('stuff'), field('empty')]
);
expect(result).toEqual(['stuff']);
});
it('should handle objects with dotted fields', () => {
const result = legacyExistingFields(
[searchResults({ 'geo.country_name': ['US'] })],
[field('geo.country_name')]
);
expect(result).toEqual(['geo.country_name']);
});
it('supports scripted fields', () => {
const result = legacyExistingFields(
[searchResults({ bar: ['scriptvalue'] })],
[field({ name: 'bar', isScript: true })]
);
expect(result).toEqual(['bar']);
});
it('supports runtime fields', () => {
const result = legacyExistingFields(
[searchResults({ runtime_foo: ['scriptvalue'] })],
[
field({
name: 'runtime_foo',
runtimeField: { type: 'long', script: { source: '2+2' } },
}),
]
);
expect(result).toEqual(['runtime_foo']);
});
it('supports meta fields', () => {
const result = legacyExistingFields(
[
{
// @ts-expect-error _mymeta is not defined on estypes.SearchHit
_mymeta: 'abc',
...searchResults({ bar: ['scriptvalue'] }),
},
],
[field({ name: '_mymeta', isMeta: true })]
);
expect(result).toEqual(['_mymeta']);
});
});
describe('buildFieldList', () => {
const indexPattern = {
title: 'testpattern',
type: 'type',
typeMeta: 'typemeta',
fields: [
{ name: 'foo', scripted: true, lang: 'painless', script: '2+2' },
{
name: 'runtime_foo',
isMapped: false,
runtimeField: { type: 'long', script: { source: '2+2' } },
},
{ name: 'bar' },
{ name: '@bar' },
{ name: 'baz' },
{ name: '_mymeta' },
],
};
it('supports scripted fields', () => {
const fields = buildFieldList(indexPattern as unknown as DataView, []);
expect(fields.find((f) => f.isScript)).toMatchObject({
isScript: true,
name: 'foo',
lang: 'painless',
script: '2+2',
});
});
it('supports runtime fields', () => {
const fields = buildFieldList(indexPattern as unknown as DataView, []);
expect(fields.find((f) => f.runtimeField)).toMatchObject({
name: 'runtime_foo',
runtimeField: { type: 'long', script: { source: '2+2' } },
});
});
it('supports meta fields', () => {
const fields = buildFieldList(indexPattern as unknown as DataView, ['_mymeta']);
expect(fields.find((f) => f.isMeta)).toMatchObject({
isScript: false,
isMeta: true,
name: '_mymeta',
});
});
});

356
x-pack/plugins/lens/server/routes/existing_fields.ts
View file

@ -1,356 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import Boom from '@hapi/boom';
import { errors } from '@elastic/elasticsearch';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { schema } from '@kbn/config-schema';
import { RequestHandlerContext, ElasticsearchClient } from '@kbn/core/server';
import { CoreSetup, Logger } from '@kbn/core/server';
import { RuntimeField } from '@kbn/data-views-plugin/common';
import { DataViewsService, DataView, FieldSpec, DataViewSpec } from '@kbn/data-views-plugin/common';
import { UI_SETTINGS } from '@kbn/data-plugin/server';
import { BASE_API_URL } from '../../common';
import { FIELD_EXISTENCE_SETTING } from '../ui_settings';
import { PluginStartContract } from '../plugin';
export function isBoomError(error: { isBoom?: boolean }): error is Boom.Boom {
return error.isBoom === true;
}
/**
* The number of docs to sample to determine field empty status.
*/
const SAMPLE_SIZE = 500;
export interface Field {
name: string;
isScript: boolean;
isMeta: boolean;
lang?: estypes.ScriptLanguage;
script?: string;
runtimeField?: RuntimeField;
}
export async function existingFieldsRoute(setup: CoreSetup<PluginStartContract>, logger: Logger) {
const router = setup.http.createRouter();
router.post(
{
path: `${BASE_API_URL}/existing_fields/{indexPatternId}`,
validate: {
params: schema.object({
indexPatternId: schema.string(),
}),
body: schema.object({
dslQuery: schema.object({}, { unknowns: 'allow' }),
fromDate: schema.maybe(schema.string()),
toDate: schema.maybe(schema.string()),
timeFieldName: schema.maybe(schema.string()),
spec: schema.object({}, { unknowns: 'allow' }),
}),
},
},
async (context, req, res) => {
const [{ savedObjects, elasticsearch, uiSettings }, { dataViews }] =
await setup.getStartServices();
const savedObjectsClient = savedObjects.getScopedClient(req);
const uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient);
const [includeFrozen, useSampling]: boolean[] = await Promise.all([
uiSettingsClient.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN),
uiSettingsClient.get(FIELD_EXISTENCE_SETTING),
]);
const esClient = elasticsearch.client.asScoped(req).asCurrentUser;
try {
return res.ok({
body: await fetchFieldExistence({
...req.params,
...req.body,
dataViewsService: await dataViews.dataViewsServiceFactory(savedObjectsClient, esClient),
context,
includeFrozen,
useSampling,
}),
});
} catch (e) {
if (e instanceof errors.TimeoutError) {
logger.info(`Field existence check timed out on ${req.params.indexPatternId}`);
// 408 is Request Timeout
return res.customError({ statusCode: 408, body: e.message });
}
logger.info(
`Field existence check failed on ${req.params.indexPatternId}: ${
isBoomError(e) ? e.output.payload.message : e.message
}`
);
if (e instanceof errors.ResponseError && e.statusCode === 404) {
return res.notFound({ body: e.message });
}
if (isBoomError(e)) {
if (e.output.statusCode === 404) {
return res.notFound({ body: e.output.payload.message });
}
throw new Error(e.output.payload.message);
} else {
throw e;
}
}
}
);
}
async function fetchFieldExistence({
context,
indexPatternId,
dataViewsService,
dslQuery = { match_all: {} },
fromDate,
toDate,
timeFieldName,
spec,
includeFrozen,
useSampling,
}: {
indexPatternId: string;
context: RequestHandlerContext;
dataViewsService: DataViewsService;
dslQuery: object;
fromDate?: string;
toDate?: string;
timeFieldName?: string;
spec?: DataViewSpec;
includeFrozen: boolean;
useSampling: boolean;
}) {
if (useSampling) {
return legacyFetchFieldExistenceSampling({
context,
indexPatternId,
dataViewsService,
dslQuery,
fromDate,
toDate,
timeFieldName,
spec,
includeFrozen,
});
}
const uiSettingsClient = (await context.core).uiSettings.client;
const metaFields: string[] = await uiSettingsClient.get(UI_SETTINGS.META_FIELDS);
const dataView =
spec && Object.keys(spec).length !== 0
? await dataViewsService.create(spec)
: await dataViewsService.get(indexPatternId);
const allFields = buildFieldList(dataView, metaFields);
const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, {
// filled in by data views service
pattern: '',
filter: toQuery(timeFieldName, fromDate, toDate, dslQuery),
});
return {
indexPatternTitle: dataView.title,
existingFieldNames: existingFields(existingFieldList, allFields),
};
}
async function legacyFetchFieldExistenceSampling({
context,
indexPatternId,
dataViewsService,
dslQuery,
fromDate,
toDate,
timeFieldName,
spec,
includeFrozen,
}: {
indexPatternId: string;
context: RequestHandlerContext;
dataViewsService: DataViewsService;
dslQuery: object;
fromDate?: string;
toDate?: string;
timeFieldName?: string;
spec?: DataViewSpec;
includeFrozen: boolean;
}) {
const coreContext = await context.core;
const metaFields: string[] = await coreContext.uiSettings.client.get(UI_SETTINGS.META_FIELDS);
const indexPattern =
spec && Object.keys(spec).length !== 0
? await dataViewsService.create(spec)
: await dataViewsService.get(indexPatternId);
const fields = buildFieldList(indexPattern, metaFields);
const runtimeMappings = indexPattern.getRuntimeMappings();
const docs = await fetchIndexPatternStats({
fromDate,
toDate,
dslQuery,
client: coreContext.elasticsearch.client.asCurrentUser,
index: indexPattern.title,
timeFieldName: timeFieldName || indexPattern.timeFieldName,
fields,
runtimeMappings,
includeFrozen,
});
return {
indexPatternTitle: indexPattern.title,
existingFieldNames: legacyExistingFields(docs, fields),
};
}
/**
* Exported only for unit tests.
*/
export function buildFieldList(indexPattern: DataView, metaFields: string[]): Field[] {
return indexPattern.fields.map((field) => {
return {
name: field.name,
isScript: !!field.scripted,
lang: field.lang,
script: field.script,
// id is a special case - it doesn't show up in the meta field list,
// but as it's not part of source, it has to be handled separately.
isMeta: metaFields.includes(field.name) || field.name === '_id',
runtimeField: !field.isMapped ? field.runtimeField : undefined,
};
});
}
async function fetchIndexPatternStats({
client,
index,
dslQuery,
timeFieldName,
fromDate,
toDate,
fields,
runtimeMappings,
includeFrozen,
}: {
client: ElasticsearchClient;
index: string;
dslQuery: object;
timeFieldName?: string;
fromDate?: string;
toDate?: string;
fields: Field[];
runtimeMappings: estypes.MappingRuntimeFields;
includeFrozen: boolean;
}) {
const query = toQuery(timeFieldName, fromDate, toDate, dslQuery);
const scriptedFields = fields.filter((f) => f.isScript);
const result = await client.search(
{
index,
...(includeFrozen ? { ignore_throttled: false } : {}),
body: {
size: SAMPLE_SIZE,
query,
// Sorted queries are usually able to skip entire shards that don't match
sort: timeFieldName && fromDate && toDate ? [{ [timeFieldName]: 'desc' }] : [],
fields: ['*'],
_source: false,
runtime_mappings: runtimeMappings,
script_fields: scriptedFields.reduce((acc, field) => {
acc[field.name] = {
script: {
lang: field.lang!,
source: field.script!,
},
};
return acc;
}, {} as Record<string, estypes.ScriptField>),
// Small improvement because there is overhead in counting
track_total_hits: false,
// Per-shard timeout, must be lower than overall. Shards return partial results on timeout
timeout: '4500ms',
},
},
{
// Global request timeout. Will cancel the request if exceeded. Overrides the elasticsearch.requestTimeout
requestTimeout: '5000ms',
// Fails fast instead of retrying- default is to retry
maxRetries: 0,
}
);
return result.hits.hits;
}
function toQuery(
timeFieldName: string | undefined,
fromDate: string | undefined,
toDate: string | undefined,
dslQuery: object
) {
const filter =
timeFieldName && fromDate && toDate
? [
{
range: {
[timeFieldName]: {
format: 'strict_date_optional_time',
gte: fromDate,
lte: toDate,
},
},
},
dslQuery,
]
: [dslQuery];
const query = {
bool: {
filter,
},
};
return query;
}
/**
* Exported only for unit tests.
*/
export function existingFields(filteredFields: FieldSpec[], allFields: Field[]): string[] {
const filteredFieldsSet = new Set(filteredFields.map((f) => f.name));
return allFields
.filter((field) => field.isScript || field.runtimeField || filteredFieldsSet.has(field.name))
.map((f) => f.name);
}
/**
* Exported only for unit tests.
*/
export function legacyExistingFields(docs: estypes.SearchHit[], fields: Field[]): string[] {
const missingFields = new Set(fields);
for (const doc of docs) {
if (missingFields.size === 0) {
break;
}
missingFields.forEach((field) => {
let fieldStore = doc.fields!;
if (field.isMeta) {
fieldStore = doc;
}
const value = fieldStore[field.name];
if (Array.isArray(value) && value.length) {
missingFields.delete(field);
} else if (!Array.isArray(value) && value) {
missingFields.delete(field);
}
});
}
return fields.filter((field) => !missingFields.has(field)).map((f) => f.name);
}

14
x-pack/plugins/lens/server/routes/index.ts
View file

@ -1,14 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { CoreSetup, Logger } from '@kbn/core/server';
import { PluginStartContract } from '../plugin';
import { existingFieldsRoute } from './existing_fields';
export function setupRoutes(setup: CoreSetup<PluginStartContract>, logger: Logger) {
existingFieldsRoute(setup, logger);
}

37
x-pack/plugins/lens/server/ui_settings.ts
View file

@ -1,37 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import { schema } from '@kbn/config-schema';
import { UiSettingsParams } from '@kbn/core/server';
export const FIELD_EXISTENCE_SETTING = 'lens:useFieldExistenceSampling';
export const getUiSettings: () => Record<string, UiSettingsParams> = () => ({
[FIELD_EXISTENCE_SETTING]: {
name: i18n.translate('xpack.lens.advancedSettings.useFieldExistenceSampling.title', {
defaultMessage: 'Use field existence sampling',
}),
value: false,
description: i18n.translate(
'xpack.lens.advancedSettings.useFieldExistenceSampling.description',
{
defaultMessage:
'If enabled, document sampling is used to determine field existence (available or empty) for the Lens field list instead of relying on index mappings.',
}
),
deprecation: {
message: i18n.translate('xpack.lens.advancedSettings.useFieldExistenceSampling.deprecation', {
defaultMessage: 'This setting is deprecated and will not be supported as of 8.6.',
}),
docLinksKey: 'visualizationSettings',
},
category: ['visualization'],
schema: schema.boolean(),
},
});