github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Lens] Filtered field list using field caps API (#122915)

Browse source
This commit is contained in:
Joe Reuter 2022-01-27 13:32:00 +01:00 • committed by GitHub
parent 95e6dd695e
commit e0bbd3c4e4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 1021 additions and 249 deletions
Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/lens/kibana.json
View file

@ -6,6 +6,7 @@
"ui": true,
"requiredPlugins": [
"data",
"dataViews",
"charts",
"expressions",
"fieldFormats",

4
x-pack/plugins/lens/server/plugin.tsx
View file

@ -7,6 +7,7 @@
import { Plugin, CoreSetup, CoreStart, PluginInitializerContext, Logger } from 'src/core/server';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { PluginStart as DataViewsServerPluginStart } from 'src/plugins/data_views/server';
import {
PluginStart as DataPluginStart,
PluginSetup as DataPluginSetup,
@ -15,6 +16,7 @@ import { ExpressionsServerSetup } from 'src/plugins/expressions/server';
import { FieldFormatsStart } from 'src/plugins/field_formats/server';
import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server';
import { setupRoutes } from './routes';
import { getUiSettings } from './ui_settings';
import {
registerLensUsageCollector,
initializeLensTelemetry,
@ -37,6 +39,7 @@ export interface PluginStartContract {
taskManager?: TaskManagerStartContract;
fieldFormats: FieldFormatsStart;
data: DataPluginStart;
dataViews: DataViewsServerPluginStart;
}
export interface LensServerPluginSetup {
@ -55,6 +58,7 @@ export class LensServerPlugin implements Plugin<LensServerPluginSetup, {}, {}, {
setupSavedObjects(core, filterMigrations);
setupRoutes(core, this.initializerContext.logger.get());
setupExpressions(core, plugins.expressions);
core.uiSettings.register(getUiSettings());
if (plugins.usageCollection && plugins.taskManager) {
registerLensUsageCollector(

46
x-pack/plugins/lens/server/routes/existing_fields.test.ts
View file

@ -6,9 +6,41 @@
*/
import { IndexPattern } from 'src/plugins/data/common';
import { existingFields, Field, buildFieldList } from './existing_fields';
import { legacyExistingFields, existingFields, Field, buildFieldList } from './existing_fields';
describe('existingFields', () => {
it('should remove missing fields by matching names', () => {
expect(
existingFields(
[
{ name: 'a', aggregatable: true, searchable: true, type: 'string' },
{ name: 'b', aggregatable: true, searchable: true, type: 'string' },
],
[
{ name: 'a', isScript: false, isMeta: false },
{ name: 'b', isScript: false, isMeta: true },
{ name: 'c', isScript: false, isMeta: false },
]
)
).toEqual(['a', 'b']);
});
it('should keep scripted and runtime fields', () => {
expect(
existingFields(
[{ name: 'a', aggregatable: true, searchable: true, type: 'string' }],
[
{ name: 'a', isScript: false, isMeta: false },
{ name: 'b', isScript: true, isMeta: false },
{ name: 'c', runtimeField: { type: 'keyword' }, isMeta: false, isScript: false },
{ name: 'd', isMeta: true, isScript: false },
]
)
).toEqual(['a', 'b', 'c']);
});
});
describe('legacyExistingFields', () => {
function field(opts: string | Partial<Field>): Field {
const obj = typeof opts === 'object' ? opts : {};
const name = (typeof opts === 'string' ? opts : opts.name) || 'test';
@ -26,7 +58,7 @@ describe('existingFields', () => {
}
it('should handle root level fields', () => {
const result = existingFields(
const result = legacyExistingFields(
[searchResults({ foo: ['bar'] }), searchResults({ baz: [0] })],
[field('foo'), field('bar'), field('baz')]
);
@ -35,7 +67,7 @@ describe('existingFields', () => {
});
it('should handle basic arrays, ignoring empty ones', () => {
const result = existingFields(
const result = legacyExistingFields(
[searchResults({ stuff: ['heyo', 'there'], empty: [] })],
[field('stuff'), field('empty')]
);
@ -44,7 +76,7 @@ describe('existingFields', () => {
});
it('should handle objects with dotted fields', () => {
const result = existingFields(
const result = legacyExistingFields(
[searchResults({ 'geo.country_name': ['US'] })],
[field('geo.country_name')]
);
@ -53,7 +85,7 @@ describe('existingFields', () => {
});
it('supports scripted fields', () => {
const result = existingFields(
const result = legacyExistingFields(
[searchResults({ bar: ['scriptvalue'] })],
[field({ name: 'bar', isScript: true })]
);
@ -62,7 +94,7 @@ describe('existingFields', () => {
});
it('supports runtime fields', () => {
const result = existingFields(
const result = legacyExistingFields(
[searchResults({ runtime_foo: ['scriptvalue'] })],
[
field({
@ -76,7 +108,7 @@ describe('existingFields', () => {
});
it('supports meta fields', () => {
const result = existingFields(
const result = legacyExistingFields(
[
{
// @ts-expect-error _mymeta is not defined on estypes.SearchHit

143
x-pack/plugins/lens/server/routes/existing_fields.ts
View file

@ -11,9 +11,11 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { schema } from '@kbn/config-schema';
import { RequestHandlerContext, ElasticsearchClient } from 'src/core/server';
import { CoreSetup, Logger } from 'src/core/server';
import { IndexPattern, IndexPatternsService, RuntimeField } from 'src/plugins/data/common';
import { RuntimeField } from 'src/plugins/data/common';
import { DataViewsService, DataView, FieldSpec } from 'src/plugins/data_views/common';
import { BASE_API_URL } from '../../common';
import { UI_SETTINGS } from '../../../../../src/plugins/data/server';
import { FIELD_EXISTENCE_SETTING } from '../ui_settings';
import { PluginStartContract } from '../plugin';
export function isBoomError(error: { isBoom?: boolean }): error is Boom.Boom {
@ -53,24 +55,24 @@ export async function existingFieldsRoute(setup: CoreSetup<PluginStartContract>,
},
},
async (context, req, res) => {
const [{ savedObjects, elasticsearch, uiSettings }, { data }] =
const [{ savedObjects, elasticsearch, uiSettings }, { dataViews }] =
await setup.getStartServices();
const savedObjectsClient = savedObjects.getScopedClient(req);
const includeFrozen: boolean = await uiSettings
.asScopedToClient(savedObjectsClient)
.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN);
const uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient);
const [includeFrozen, useSampling]: boolean[] = await Promise.all([
uiSettingsClient.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN),
uiSettingsClient.get(FIELD_EXISTENCE_SETTING),
]);
const esClient = elasticsearch.client.asScoped(req).asCurrentUser;
try {
return res.ok({
body: await fetchFieldExistence({
...req.params,
...req.body,
indexPatternsService: await data.indexPatterns.indexPatternsServiceFactory(
savedObjectsClient,
esClient
),
dataViewsService: await dataViews.dataViewsServiceFactory(savedObjectsClient, esClient),
context,
includeFrozen,
useSampling,
}),
});
} catch (e) {
@ -103,16 +105,64 @@ export async function existingFieldsRoute(setup: CoreSetup<PluginStartContract>,
async function fetchFieldExistence({
context,
indexPatternId,
indexPatternsService,
dataViewsService,
dslQuery = { match_all: {} },
fromDate,
toDate,
timeFieldName,
includeFrozen,
useSampling,
}: {
indexPatternId: string;
context: RequestHandlerContext;
dataViewsService: DataViewsService;
dslQuery: object;
fromDate?: string;
toDate?: string;
timeFieldName?: string;
includeFrozen: boolean;
useSampling: boolean;
}) {
if (useSampling) {
return legacyFetchFieldExistenceSampling({
context,
indexPatternId,
dataViewsService,
dslQuery,
fromDate,
toDate,
timeFieldName,
includeFrozen,
});
}
const metaFields: string[] = await context.core.uiSettings.client.get(UI_SETTINGS.META_FIELDS);
const dataView = await dataViewsService.get(indexPatternId);
const allFields = buildFieldList(dataView, metaFields);
const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, {
// filled in by data views service
pattern: '',
filter: toQuery(timeFieldName, fromDate, toDate, dslQuery),
});
return {
indexPatternTitle: dataView.title,
existingFieldNames: existingFields(existingFieldList, allFields),
};
}
async function legacyFetchFieldExistenceSampling({
context,
indexPatternId,
dataViewsService,
dslQuery,
fromDate,
toDate,
timeFieldName,
includeFrozen,
}: {
indexPatternId: string;
context: RequestHandlerContext;
indexPatternsService: IndexPatternsService;
dataViewsService: DataViewsService;
dslQuery: object;
fromDate?: string;
toDate?: string;
@ -120,7 +170,7 @@ async function fetchFieldExistence({
includeFrozen: boolean;
}) {
const metaFields: string[] = await context.core.uiSettings.client.get(UI_SETTINGS.META_FIELDS);
const indexPattern = await indexPatternsService.get(indexPatternId);
const indexPattern = await dataViewsService.get(indexPatternId);
const fields = buildFieldList(indexPattern, metaFields);
const docs = await fetchIndexPatternStats({
@ -136,14 +186,14 @@ async function fetchFieldExistence({
return {
indexPatternTitle: indexPattern.title,
existingFieldNames: existingFields(docs, fields),
existingFieldNames: legacyExistingFields(docs, fields),
};
}
/**
* Exported only for unit tests.
*/
export function buildFieldList(indexPattern: IndexPattern, metaFields: string[]): Field[] {
export function buildFieldList(indexPattern: DataView, metaFields: string[]): Field[] {
return indexPattern.fields.map((field) => {
return {
name: field.name,
@ -177,27 +227,7 @@ async function fetchIndexPatternStats({
fields: Field[];
includeFrozen: boolean;
}) {
const filter =
timeFieldName && fromDate && toDate
? [
{
range: {
[timeFieldName]: {
format: 'strict_date_optional_time',
gte: fromDate,
lte: toDate,
},
},
},
dslQuery,
]
: [dslQuery];
const query = {
bool: {
filter,
},
};
const query = toQuery(timeFieldName, fromDate, toDate, dslQuery);
const scriptedFields = fields.filter((f) => f.isScript);
const runtimeFields = fields.filter((f) => f.runtimeField);
@ -242,10 +272,51 @@ async function fetchIndexPatternStats({
return result.hits.hits;
}
function toQuery(
timeFieldName: string | undefined,
fromDate: string | undefined,
toDate: string | undefined,
dslQuery: object
) {
const filter =
timeFieldName && fromDate && toDate
? [
{
range: {
[timeFieldName]: {
format: 'strict_date_optional_time',
gte: fromDate,
lte: toDate,
},
},
},
dslQuery,
]
: [dslQuery];
const query = {
bool: {
filter,
},
};
return query;
}
/**
* Exported only for unit tests.
*/
export function existingFields(docs: estypes.SearchHit[], fields: Field[]): string[] {
export function existingFields(filteredFields: FieldSpec[], allFields: Field[]): string[] {
const filteredFieldsSet = new Set(filteredFields.map((f) => f.name));
return allFields
.filter((field) => field.isScript || field.runtimeField || filteredFieldsSet.has(field.name))
.map((f) => f.name);
}
/**
* Exported only for unit tests.
*/
export function legacyExistingFields(docs: estypes.SearchHit[], fields: Field[]): string[] {
const missingFields = new Set(fields);
for (const doc of docs) {

31
x-pack/plugins/lens/server/ui_settings.ts Normal file
View file

@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import { schema } from '@kbn/config-schema';
import { UiSettingsParams } from 'kibana/server';
export const FIELD_EXISTENCE_SETTING = 'lens:useFieldExistenceSampling';
export const getUiSettings: () => Record<string, UiSettingsParams> = () => ({
[FIELD_EXISTENCE_SETTING]: {
name: i18n.translate('xpack.lens.advancedSettings.useFieldExistenceSampling.title', {
defaultMessage: 'Use field existence sampling',
}),
value: false,
description: i18n.translate(
'xpack.lens.advancedSettings.useFieldExistenceSampling.description',
{
defaultMessage:
'If enabled, document sampling is used to determine field existence (available or empty) for the Lens field list instead of relying on index mappings.',
}
),
category: ['visualization'],
schema: schema.boolean(),
},
});

286
x-pack/test/api_integration/apis/lens/existing_fields.ts
View file

@ -9,171 +9,46 @@ import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
const TEST_START_TIME = '2015-09-19T06:31:44.000';
const TEST_END_TIME = '2015-09-23T18:31:44.000';
const TEST_START_TIME = '2010-09-19T06:31:44.000';
const TEST_END_TIME = '2023-09-23T18:31:44.000';
const COMMON_HEADERS = {
'kbn-xsrf': 'some-xsrf-token',
};
const metaFields = ['_id', '_index', '_score', '_source', '_type'];
const fieldsWithData = [
'@message',
'@message.raw',
'@tags',
'@tags.raw',
'@timestamp',
'_id',
'_index',
'agent',
'agent.raw',
'bytes',
'clientip',
'extension',
'extension.raw',
'geo.coordinates',
'geo.dest',
'geo.src',
'geo.srcdest',
'headings',
'headings.raw',
'host',
'host.raw',
'index',
'index.raw',
'ip',
'links',
'links.raw',
'machine.os',
'machine.os.raw',
'machine.ram',
'machine.ram_range',
'memory',
'phpmemory',
'referer',
'request',
'request.raw',
'response',
'response.raw',
'spaces',
'spaces.raw',
'type',
'url',
'url.raw',
'utc_time',
'xss',
'xss.raw',
'runtime_number',
'relatedContent.article:modified_time',
'relatedContent.article:published_time',
'relatedContent.article:section',
'relatedContent.article:section.raw',
'relatedContent.article:tag',
'relatedContent.article:tag.raw',
'relatedContent.og:description',
'relatedContent.og:description.raw',
'relatedContent.og:image',
'relatedContent.og:image.raw',
'relatedContent.og:image:height',
'relatedContent.og:image:height.raw',
'relatedContent.og:image:width',
'relatedContent.og:image:width.raw',
'relatedContent.og:site_name',
'relatedContent.og:site_name.raw',
'relatedContent.og:title',
'relatedContent.og:title.raw',
'relatedContent.og:type',
'relatedContent.og:type.raw',
'relatedContent.og:url',
'relatedContent.og:url.raw',
'relatedContent.twitter:card',
'relatedContent.twitter:card.raw',
'relatedContent.twitter:description',
'relatedContent.twitter:description.raw',
'relatedContent.twitter:image',
'relatedContent.twitter:image.raw',
'relatedContent.twitter:site',
'relatedContent.twitter:site.raw',
'relatedContent.twitter:title',
'relatedContent.twitter:title.raw',
'relatedContent.url',
'relatedContent.url.raw',
];
const metricBeatData = [
'@timestamp',
'_id',
'_index',
'agent.ephemeral_id',
'agent.ephemeral_id.keyword',
'agent.hostname',
'agent.hostname.keyword',
'agent.id',
'agent.id.keyword',
'agent.type',
'agent.type.keyword',
'agent.version',
'agent.version.keyword',
'ecs.version',
'ecs.version.keyword',
'event.dataset',
'event.dataset.keyword',
'event.duration',
'event.module',
'event.module.keyword',
'host.architecture',
'host.architecture.keyword',
'host.hostname',
'host.hostname.keyword',
'host.id',
'host.id.keyword',
'host.name',
'host.name.keyword',
'host.os.build',
'host.os.build.keyword',
'host.os.family',
'host.os.family.keyword',
'host.os.kernel',
'host.os.kernel.keyword',
'host.os.name',
'host.os.name.keyword',
'host.os.platform',
'host.os.platform.keyword',
'host.os.version',
'host.os.version.keyword',
'metricset.name',
'metricset.name.keyword',
'service.type',
'service.type.keyword',
'system.cpu.cores',
'system.cpu.idle.pct',
'system.cpu.iowait.pct',
'system.cpu.irq.pct',
'system.cpu.nice.pct',
'system.cpu.softirq.pct',
'system.cpu.steal.pct',
'system.cpu.system.pct',
'system.cpu.total.pct',
'system.cpu.user.pct',
'ts',
'filter_field',
'textfield1',
'textfield2',
'mapping_runtime_field',
'data_view_runtime_field',
];
export default ({ getService }: FtrProviderContext) => {
const esArchiver = getService('esArchiver');
const supertest = getService('supertest');
const kibanaServer = getService('kibanaServer');
describe('existing_fields apis', () => {
before(async () => {
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional');
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/visualize/default');
await esArchiver.load('x-pack/test/api_integration/es_archives/lens/constant_keyword');
await kibanaServer.importExport.load(
'x-pack/test/api_integration/fixtures/kbn_archiver/lens/constant_keyword.json'
);
});
after(async () => {
await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional');
await esArchiver.unload('x-pack/test/functional/es_archives/visualize/default');
await esArchiver.unload('x-pack/test/api_integration/es_archives/lens/constant_keyword');
await kibanaServer.importExport.unload(
'x-pack/test/api_integration/fixtures/kbn_archiver/lens/constant_keyword.json'
);
});
describe('existence', () => {
it('should find which fields exist in the sample documents', async () => {
const { body } = await supertest
.post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`)
.post(`/api/lens/existing_fields/existence_index`)
.set(COMMON_HEADERS)
.send({
dslQuery: {
@ -186,76 +61,89 @@ export default ({ getService }: FtrProviderContext) => {
})
.expect(200);
expect(body.indexPatternTitle).to.eql('logstash-*');
expect(body.existingFieldNames.sort()).to.eql(fieldsWithData.sort());
expect(body.indexPatternTitle).to.eql('existence_index_*');
expect(body.existingFieldNames.sort()).to.eql([...metaFields, ...fieldsWithData].sort());
});
it('should succeed for thousands of fields', async () => {
const { body } = await supertest
.post(`/api/lens/existing_fields/${encodeURIComponent('metricbeat-*')}`)
.set(COMMON_HEADERS)
.send({
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
})
.expect(200);
expect(body.indexPatternTitle).to.eql('metricbeat-*');
expect(body.existingFieldNames.sort()).to.eql(metricBeatData.sort());
});
it('should return fields filtered by query and filters', async () => {
it('should return fields filtered by term query', async () => {
const expectedFieldNames = [
'@message',
'@message.raw',
'@tags',
'@tags.raw',
'@timestamp',
'_id',
'_index',
'agent',
'agent.raw',
'bytes',
'clientip',
'extension',
'extension.raw',
'headings',
'headings.raw',
'host',
'host.raw',
'index',
'index.raw',
'referer',
'request',
'request.raw',
'response',
'response.raw',
'runtime_number',
'spaces',
'spaces.raw',
'type',
'url',
'url.raw',
'utc_time',
'xss',
'xss.raw',
'ts',
'filter_field',
'textfield1',
// textfield2 and mapping_runtime_field are defined on the other index
'data_view_runtime_field',
];
const { body } = await supertest
.post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`)
.post(`/api/lens/existing_fields/existence_index`)
.set(COMMON_HEADERS)
.send({
dslQuery: {
bool: {
filter: [{ match: { referer: 'https://www.taylorswift.com/' } }],
filter: [{ term: { filter_field: 'a' } }],
},
},
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
})
.expect(200);
expect(body.existingFieldNames.sort()).to.eql(expectedFieldNames.sort());
expect(body.existingFieldNames.sort()).to.eql(
[...metaFields, ...expectedFieldNames].sort()
);
});
it('should return fields filtered by match_phrase query', async () => {
const expectedFieldNames = [
'ts',
'filter_field',
'textfield1',
// textfield2 and mapping_runtime_field are defined on the other index
'data_view_runtime_field',
];
const { body } = await supertest
.post(`/api/lens/existing_fields/existence_index`)
.set(COMMON_HEADERS)
.send({
dslQuery: {
bool: {
filter: [{ match_phrase: { filter_field: 'a' } }],
},
},
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
})
.expect(200);
expect(body.existingFieldNames.sort()).to.eql(
[...metaFields, ...expectedFieldNames].sort()
);
});
it('should return fields filtered by time range', async () => {
const expectedFieldNames = [
'ts',
'filter_field',
'textfield1',
// textfield2 and mapping_runtime_field are defined on the other index
'data_view_runtime_field',
];
const { body } = await supertest
.post(`/api/lens/existing_fields/existence_index`)
.set(COMMON_HEADERS)
.send({
dslQuery: {
bool: {
filter: [{ term: { filter_field: 'a' } }],
},
},
fromDate: TEST_START_TIME,
toDate: '2021-12-12',
})
.expect(200);
expect(body.existingFieldNames.sort()).to.eql(
[...metaFields, ...expectedFieldNames].sort()
);
});
});
});

1
x-pack/test/api_integration/apis/lens/index.ts
View file

@ -10,6 +10,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
export default function lensApiIntegrationTests({ loadTestFile }: FtrProviderContext) {
describe('Lens', () => {
loadTestFile(require.resolve('./existing_fields'));
loadTestFile(require.resolve('./legacy_existing_fields'));
loadTestFile(require.resolve('./field_stats'));
loadTestFile(require.resolve('./telemetry'));
});

269
x-pack/test/api_integration/apis/lens/legacy_existing_fields.ts Normal file
View file

@ -0,0 +1,269 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
const TEST_START_TIME = '2015-09-19T06:31:44.000';
const TEST_END_TIME = '2015-09-23T18:31:44.000';
const COMMON_HEADERS = {
'kbn-xsrf': 'some-xsrf-token',
};
const fieldsWithData = [
'@message',
'@message.raw',
'@tags',
'@tags.raw',
'@timestamp',
'_id',
'_index',
'agent',
'agent.raw',
'bytes',
'clientip',
'extension',
'extension.raw',
'geo.coordinates',
'geo.dest',
'geo.src',
'geo.srcdest',
'headings',
'headings.raw',
'host',
'host.raw',
'index',
'index.raw',
'ip',
'links',
'links.raw',
'machine.os',
'machine.os.raw',
'machine.ram',
'machine.ram_range',
'memory',
'phpmemory',
'referer',
'request',
'request.raw',
'response',
'response.raw',
'spaces',
'spaces.raw',
'type',
'url',
'url.raw',
'utc_time',
'xss',
'xss.raw',
'runtime_number',
'relatedContent.article:modified_time',
'relatedContent.article:published_time',
'relatedContent.article:section',
'relatedContent.article:section.raw',
'relatedContent.article:tag',
'relatedContent.article:tag.raw',
'relatedContent.og:description',
'relatedContent.og:description.raw',
'relatedContent.og:image',
'relatedContent.og:image.raw',
'relatedContent.og:image:height',
'relatedContent.og:image:height.raw',
'relatedContent.og:image:width',
'relatedContent.og:image:width.raw',
'relatedContent.og:site_name',
'relatedContent.og:site_name.raw',
'relatedContent.og:title',
'relatedContent.og:title.raw',
'relatedContent.og:type',
'relatedContent.og:type.raw',
'relatedContent.og:url',
'relatedContent.og:url.raw',
'relatedContent.twitter:card',
'relatedContent.twitter:card.raw',
'relatedContent.twitter:description',
'relatedContent.twitter:description.raw',
'relatedContent.twitter:image',
'relatedContent.twitter:image.raw',
'relatedContent.twitter:site',
'relatedContent.twitter:site.raw',
'relatedContent.twitter:title',
'relatedContent.twitter:title.raw',
'relatedContent.url',
'relatedContent.url.raw',
];
const metricBeatData = [
'@timestamp',
'_id',
'_index',
'agent.ephemeral_id',
'agent.ephemeral_id.keyword',
'agent.hostname',
'agent.hostname.keyword',
'agent.id',
'agent.id.keyword',
'agent.type',
'agent.type.keyword',
'agent.version',
'agent.version.keyword',
'ecs.version',
'ecs.version.keyword',
'event.dataset',
'event.dataset.keyword',
'event.duration',
'event.module',
'event.module.keyword',
'host.architecture',
'host.architecture.keyword',
'host.hostname',
'host.hostname.keyword',
'host.id',
'host.id.keyword',
'host.name',
'host.name.keyword',
'host.os.build',
'host.os.build.keyword',
'host.os.family',
'host.os.family.keyword',
'host.os.kernel',
'host.os.kernel.keyword',
'host.os.name',
'host.os.name.keyword',
'host.os.platform',
'host.os.platform.keyword',
'host.os.version',
'host.os.version.keyword',
'metricset.name',
'metricset.name.keyword',
'service.type',
'service.type.keyword',
'system.cpu.cores',
'system.cpu.idle.pct',
'system.cpu.iowait.pct',
'system.cpu.irq.pct',
'system.cpu.nice.pct',
'system.cpu.softirq.pct',
'system.cpu.steal.pct',
'system.cpu.system.pct',
'system.cpu.total.pct',
'system.cpu.user.pct',
];
export default ({ getService }: FtrProviderContext) => {
const esArchiver = getService('esArchiver');
const supertest = getService('supertest');
const kibanaServer = getService('kibanaServer');
describe('existing_fields apis legacy', () => {
before(async () => {
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional');
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/visualize/default');
await kibanaServer.uiSettings.update({
'lens:useFieldExistenceSampling': true,
});
});
after(async () => {
await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional');
await esArchiver.unload('x-pack/test/functional/es_archives/visualize/default');
await kibanaServer.uiSettings.update({
'lens:useFieldExistenceSampling': false,
});
});
describe('existence', () => {
it('should find which fields exist in the sample documents', async () => {
const { body } = await supertest
.post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`)
.set(COMMON_HEADERS)
.send({
dslQuery: {
bool: {
filter: [{ match_all: {} }],
},
},
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
})
.expect(200);
expect(body.indexPatternTitle).to.eql('logstash-*');
expect(body.existingFieldNames.sort()).to.eql(fieldsWithData.sort());
});
it('should succeed for thousands of fields', async () => {
const { body } = await supertest
.post(`/api/lens/existing_fields/${encodeURIComponent('metricbeat-*')}`)
.set(COMMON_HEADERS)
.send({
dslQuery: { match_all: {} },
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
})
.expect(200);
expect(body.indexPatternTitle).to.eql('metricbeat-*');
expect(body.existingFieldNames.sort()).to.eql(metricBeatData.sort());
});
it('should return fields filtered by query and filters', async () => {
const expectedFieldNames = [
'@message',
'@message.raw',
'@tags',
'@tags.raw',
'@timestamp',
'_id',
'_index',
'agent',
'agent.raw',
'bytes',
'clientip',
'extension',
'extension.raw',
'headings',
'headings.raw',
'host',
'host.raw',
'index',
'index.raw',
'referer',
'request',
'request.raw',
'response',
'response.raw',
'runtime_number',
'spaces',
'spaces.raw',
'type',
'url',
'url.raw',
'utc_time',
'xss',
'xss.raw',
];
const { body } = await supertest
.post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`)
.set(COMMON_HEADERS)
.send({
dslQuery: {
bool: {
filter: [{ match: { referer: 'https://www.taylorswift.com/' } }],
},
},
fromDate: TEST_START_TIME,
toDate: TEST_END_TIME,
})
.expect(200);
expect(body.existingFieldNames.sort()).to.eql(expectedFieldNames.sort());
});
});
});
};

25
x-pack/test/api_integration/es_archives/lens/constant_keyword/data.json Normal file
View file

@ -0,0 +1,25 @@
{
"type": "doc",
"value": {
"id": "1",
"index": "existence_index_1",
"source": {
"filter_field": "a",
"textfield1": "test",
"ts": "2021-01-02"
}
}
}
{
"type": "doc",
"value": {
"id": "1",
"index": "existence_index_2",
"source": {
"filter_field": "b",
"textfield2": "test",
"ts": "2022-01-02"
}
}
}

59
x-pack/test/api_integration/es_archives/lens/constant_keyword/mappings.json Normal file
View file

@ -0,0 +1,59 @@
{
"type": "index",
"value": {
"index": "existence_index_1",
"mappings": {
"properties": {
"filter_field": {
"type": "constant_keyword",
"value": "a"
},
"textfield1": {
"type": "keyword"
},
"ts": {
"type": "date"
}
}
},
"settings": {
"index": {
"number_of_replicas": "0",
"number_of_shards": "1"
}
}
}
}
{
"type": "index",
"value": {
"index": "existence_index_2",
"mappings": {
"runtime": {
"mapping_runtime_field": {
"type": "keyword",
"script" : { "source" : "emit('abc')" }
}
},
"properties": {
"filter_field": {
"type": "constant_keyword",
"value": "b"
},
"textfield2": {
"type": "keyword"
},
"ts": {
"type": "date"
}
}
},
"settings": {
"index": {
"number_of_replicas": "0",
"number_of_shards": "1"
}
}
}
}

16
x-pack/test/api_integration/fixtures/kbn_archiver/lens/constant_keyword.json Normal file
View file

@ -0,0 +1,16 @@
{
"attributes": {
"timeFieldName": "ts",
"title": "existence_index_*",
"runtimeFieldMap":"{\"data_view_runtime_field\":{\"type\":\"keyword\",\"script\":{\"source\":\"emit('a')\"}}}"
},
"coreMigrationVersion": "8.0.0",
"id": "existence_index",
"migrationVersion": {
"index-pattern": "7.11.0"
},
"references": [],
"type": "index-pattern",
"updated_at": "2018-12-21T00:43:07.096Z",
"version": "WzEzLDJd"
}

2
x-pack/test/functional/apps/lens/drag_and_drop.ts
View file

@ -328,8 +328,10 @@ export default function ({ getPageObjects }: FtrProviderContext) {
});
it('overwrite existing time dimension if one exists already', async () => {
await PageObjects.lens.searchField('utc');
await PageObjects.lens.dragFieldToWorkspace('utc_time');
await PageObjects.lens.waitForVisualization();
await PageObjects.lens.searchField('client');
await PageObjects.lens.dragFieldToWorkspace('clientip');
await PageObjects.lens.waitForVisualization();
expect(await PageObjects.lens.getDimensionTriggersTexts('lnsXY_xDimensionPanel')).to.eql([

4
x-pack/test/functional/apps/lens/epoch_millis.ts
View file

@ -30,13 +30,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
it('should show field list', async () => {
await PageObjects.visualize.navigateToNewVisualization();
await PageObjects.visualize.clickVisType('lens');
await PageObjects.lens.switchDataPanelIndexPattern('epoch-millis');
await PageObjects.lens.switchDataPanelIndexPattern('epoch-millis*');
await PageObjects.lens.goToTimeRange();
await PageObjects.lens.switchToVisualization('lnsDatatable');
const fieldList = await PageObjects.lens.findAllFields();
expect(fieldList).to.contain('@timestamp');
// not defined for document in time range, only out of time range
expect(fieldList).not.to.contain('agent.raw');
});
it('should able to configure a regular metric', async () => {

4
x-pack/test/functional/es_archives/lens/epoch_millis/data.json
View file

@ -2,7 +2,7 @@
"type": "doc",
"value": {
"id": "AU_x4-TaGFA8no6QjiSJ",
"index": "epoch-millis",
"index": "epoch-millis1",
"source": {
"@message": "212.113.62.183 - - [2015-09-21T06:09:20.045Z] \"GET /uploads/dafydd-williams.jpg HTTP/1.1\" 200 3182 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1\"",
"@tags": [
@ -75,7 +75,7 @@
"type": "doc",
"value": {
"id": "AU_x4-TaGFA8no6QjiSL",
"index": "epoch-millis",
"index": "epoch-millis2",
"source": {
"@message": "156.252.112.76 - - [2015-09-21T21:13:02.070Z] \"GET /uploads/aleksandr-samokutyayev.jpg HTTP/1.1\" 200 6176 \"-\" \"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24\"",
"@tags": [

377
x-pack/test/functional/es_archives/lens/epoch_millis/mappings.json
View file

@ -1,7 +1,382 @@
{
"type": "index",
"value": {
"index": "epoch-millis",
"index": "epoch-millis1",
"mappings": {
"dynamic_templates": [
{
"string_fields": {
"mapping": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"match": "*",
"match_mapping_type": "string"
}
}
],
"runtime": {
"runtime_number": {
"type": "long",
"script" : { "source" : "emit(doc['bytes'].value)" }
}
},
"properties": {
"@message": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"@tags": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"@timestamp": {
"type": "date",
"format": "epoch_millis"
},
"agent": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"bytes": {
"type": "long"
},
"clientip": {
"type": "ip"
},
"extension": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"geo": {
"properties": {
"coordinates": {
"type": "geo_point"
},
"dest": {
"type": "keyword"
},
"src": {
"type": "keyword"
},
"srcdest": {
"type": "keyword"
}
}
},
"headings": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"host": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"id": {
"type": "integer"
},
"index": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"ip": {
"type": "ip"
},
"links": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"machine": {
"properties": {
"os": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"ram": {
"type": "long"
},
"ram_range": {
"type": "long_range"
}
}
},
"memory": {
"type": "double"
},
"meta": {
"properties": {
"char": {
"type": "keyword"
},
"related": {
"type": "text"
},
"user": {
"properties": {
"firstname": {
"type": "text"
},
"lastname": {
"type": "integer"
}
}
}
}
},
"phpmemory": {
"type": "long"
},
"referer": {
"type": "keyword"
},
"relatedContent": {
"properties": {
"article:modified_time": {
"type": "date"
},
"article:published_time": {
"type": "date"
},
"article:section": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"article:tag": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"og:description": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"og:image": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"og:image:height": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"og:image:width": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"og:site_name": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"og:title": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"og:type": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"og:url": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"twitter:card": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"twitter:description": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"twitter:image": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"twitter:site": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"twitter:title": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"url": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
}
}
},
"request": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"response": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"spaces": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"type": {
"type": "keyword"
},
"url": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"utc_time": {
"type": "date"
},
"xss": {
"fields": {
"raw": {
"type": "keyword"
}
},
"type": "text"
}
}
},
"settings": {
"index": {
"analysis": {
"analyzer": {
"url": {
"max_token_length": "1000",
"tokenizer": "uax_url_email",
"type": "standard"
}
}
},
"number_of_replicas": "0",
"number_of_shards": "1"
}
}
}
}
{
"type": "index",
"value": {
"index": "epoch-millis2",
"mappings": {
"dynamic_templates": [
{

2
x-pack/test/functional/fixtures/kbn_archiver/lens/epoch_millis.json
View file

@ -1,7 +1,7 @@
{
"attributes": {
"timeFieldName": "@timestamp",
"title": "epoch-millis"
"title": "epoch-millis*"
},
"coreMigrationVersion": "8.0.0",
"id": "epoch-millis",