github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Logs UI] Display and query runtime fields from KIPs in the log stream and entry flyout (#97467)

Browse source 
This enhances the queries such that they pass runtime fields defined on Kibana index patterns as `runtime_mappings` in the log entry search strategies.
This commit is contained in:
Felix Stürmer 2021-04-20 11:22:14 +02:00 • committed by GitHub
parent 20b585d122
commit 9ae605af93
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 291 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

100
x-pack/plugins/infra/common/dependency_mocks/index_patterns.ts Normal file
View file

@ -0,0 +1,100 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { from, of } from 'rxjs';
import { delay } from 'rxjs/operators';
import {
fieldList,
FieldSpec,
IIndexPattern,
IndexPattern,
IndexPatternsContract,
RuntimeField,
} from 'src/plugins/data/common';
type IndexPatternMock = Pick<
IndexPattern,
| 'fields'
| 'getComputedFields'
| 'getFieldByName'
| 'getTimeField'
| 'id'
| 'isTimeBased'
| 'title'
| 'type'
>;
type IndexPatternMockSpec = Pick<IIndexPattern, 'id' | 'title' | 'type' | 'timeFieldName'> & {
fields: FieldSpec[];
};
export const createIndexPatternMock = ({
id,
title,
type = undefined,
fields,
timeFieldName,
}: IndexPatternMockSpec): IndexPatternMock => {
const indexPatternFieldList = fieldList(fields);
return {
id,
title,
type,
fields: indexPatternFieldList,
getTimeField: () => indexPatternFieldList.find(({ name }) => name === timeFieldName),
isTimeBased: () => timeFieldName != null,
getFieldByName: (fieldName) => indexPatternFieldList.find(({ name }) => name === fieldName),
getComputedFields: () => ({
runtimeFields: indexPatternFieldList.reduce<Record<string, RuntimeField>>(
(accumulatedFields, { name, runtimeField }) => ({
...accumulatedFields,
...(runtimeField != null
? {
[name]: runtimeField,
}
: {}),
}),
{}
),
scriptFields: {},
storedFields: [],
docvalueFields: [],
}),
};
};
export const createIndexPatternsMock = (
asyncDelay: number,
indexPatterns: IndexPatternMock[]
): {
getIdsWithTitle: IndexPatternsContract['getIdsWithTitle'];
get: (...args: Parameters<IndexPatternsContract['get']>) => Promise<IndexPatternMock>;
} => {
return {
async getIdsWithTitle(_refresh?: boolean) {
const indexPatterns$ = of(
indexPatterns.map(({ id = 'unknown_id', title }) => ({ id, title }))
);
return await indexPatterns$.pipe(delay(asyncDelay)).toPromise();
},
async get(indexPatternId: string) {
const indexPatterns$ = from(
indexPatterns.filter((indexPattern) => indexPattern.id === indexPatternId)
);
return await indexPatterns$.pipe(delay(asyncDelay)).toPromise();
},
};
};
export const createIndexPatternsStartMock = (
asyncDelay: number,
indexPatterns: IndexPatternMock[]
): any => {
return {
indexPatternsServiceFactory: async () => createIndexPatternsMock(asyncDelay, indexPatterns),
};
};

36
x-pack/plugins/infra/common/log_sources/resolved_log_source_configuration.ts
View file

@ -5,11 +5,13 @@
* 2.0.
*/
import { estypes } from '@elastic/elasticsearch';
import { IndexPattern, IndexPatternsContract } from '../../../../../src/plugins/data/common';
import { ObjectEntries } from '../utility_types';
import {
LogSourceConfigurationProperties,
LogSourceColumnConfiguration,
LogSourceConfigurationProperties,
} from './log_source_configuration';
import { IndexPatternsContract, IndexPattern } from '../../../../../src/plugins/data/common';
export interface ResolvedLogSourceConfiguration {
name: string;
@ -19,6 +21,7 @@ export interface ResolvedLogSourceConfiguration {
tiebreakerField: string;
messageField: string[];
fields: IndexPattern['fields'];
runtimeMappings: estypes.RuntimeFields;
columns: LogSourceColumnConfiguration[];
}
@ -52,6 +55,7 @@ const resolveLegacyReference = async (
tiebreakerField: sourceConfiguration.fields.tiebreaker,
messageField: sourceConfiguration.fields.message,
fields,
runtimeMappings: {},
columns: sourceConfiguration.logColumns,
name: sourceConfiguration.name,
description: sourceConfiguration.description,
@ -76,8 +80,36 @@ const resolveKibanaIndexPatternReference = async (
tiebreakerField: '_doc',
messageField: ['message'],
fields: indexPattern.fields,
runtimeMappings: resolveRuntimeMappings(indexPattern),
columns: sourceConfiguration.logColumns,
name: sourceConfiguration.name,
description: sourceConfiguration.description,
};
};
// this might take other sources of runtime fields into account in the future
const resolveRuntimeMappings = (indexPattern: IndexPattern): estypes.RuntimeFields => {
const { runtimeFields } = indexPattern.getComputedFields();
const runtimeMappingsFromIndexPattern = (Object.entries(runtimeFields) as ObjectEntries<
typeof runtimeFields
>).reduce<estypes.RuntimeFields>(
(accumulatedMappings, [runtimeFieldName, runtimeFieldSpec]) => ({
...accumulatedMappings,
[runtimeFieldName]: {
type: runtimeFieldSpec.type,
...(runtimeFieldSpec.script != null
? {
script: {
lang: 'painless', // required in the es types
source: runtimeFieldSpec.script.source,
},
}
: {}),
},
}),
{}
);
return runtimeMappingsFromIndexPattern;
};

37
x-pack/plugins/infra/public/utils/logs_overview_fetches.test.ts
View file

@ -8,11 +8,12 @@
import { CoreStart } from 'kibana/public';
import { coreMock } from 'src/core/public/mocks';
import { dataPluginMock } from 'src/plugins/data/public/mocks';
import { callFetchLogSourceStatusAPI } from '../containers/logs/log_source/api/fetch_log_source_status';
import { createIndexPatternMock } from '../../common/dependency_mocks/index_patterns';
import { GetLogSourceConfigurationSuccessResponsePayload } from '../../common/http_api/log_sources/get_log_source_configuration';
import { callFetchLogSourceConfigurationAPI } from '../containers/logs/log_source/api/fetch_log_source_configuration';
import { callFetchLogSourceStatusAPI } from '../containers/logs/log_source/api/fetch_log_source_status';
import { InfraClientStartDeps, InfraClientStartExports } from '../types';
import { getLogsHasDataFetcher, getLogsOverviewDataFetcher } from './logs_overview_fetchers';
import { GetLogSourceConfigurationSuccessResponsePayload } from '../../common/http_api/log_sources/get_log_source_configuration';
jest.mock('../containers/logs/log_source/api/fetch_log_source_status');
const mockedCallFetchLogSourceStatusAPI = callFetchLogSourceStatusAPI as jest.MockedFunction<
@ -41,6 +42,36 @@ function setup() {
//
const dataResponder = jest.fn();
(data.indexPatterns.get as jest.Mock).mockResolvedValue(
createIndexPatternMock({
id: 'test-index-pattern',
title: 'log-indices-*',
timeFieldName: '@timestamp',
fields: [
{
name: 'event.dataset',
type: 'string',
esTypes: ['keyword'],
aggregatable: true,
searchable: true,
},
{
name: 'runtime_field',
type: 'string',
runtimeField: {
type: 'keyword',
script: {
source: 'emit("runtime value")',
},
},
esTypes: ['keyword'],
aggregatable: true,
searchable: true,
},
],
})
);
(data.search.search as jest.Mock).mockReturnValue({
subscribe: (progress: Function, error: Function, finish: Function) => {
progress(dataResponder());
@ -114,7 +145,7 @@ describe('Logs UI Observability Homepage Functions', () => {
configuration: {
logIndices: {
type: 'index_pattern',
indexPatternId: 'some-test-id',
indexPatternId: 'test-index-pattern',
},
fields: { timestamp: '@timestamp', tiebreaker: '_doc' },
},

2
x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts
View file

@ -93,6 +93,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter {
],
},
},
runtime_mappings: resolvedLogSourceConfiguration.runtimeMappings,
sort,
...highlightClause,
...searchAfterClause,
@ -182,6 +183,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter {
],
},
},
runtime_mappings: resolvedLogSourceConfiguration.runtimeMappings,
size: 0,
track_total_hits: false,
},

46
x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts
View file

@ -19,13 +19,16 @@ import {
SearchStrategyDependencies,
} from 'src/plugins/data/server';
import { createSearchSessionsClientMock } from '../../../../../../src/plugins/data/server/search/mocks';
import {
createIndexPatternMock,
createIndexPatternsStartMock,
} from '../../../common/dependency_mocks/index_patterns';
import { InfraSource } from '../../lib/sources';
import { createInfraSourcesMock } from '../../lib/sources/mocks';
import {
logEntriesSearchRequestStateRT,
logEntriesSearchStrategyProvider,
} from './log_entries_search_strategy';
import { getIndexPatternsMock } from './mocks';
describe('LogEntries search strategy', () => {
it('handles initial search requests', async () => {
@ -72,6 +75,15 @@ describe('LogEntries search strategy', () => {
index: 'log-indices-*',
body: expect.objectContaining({
fields: expect.arrayContaining(['event.dataset', 'message']),
runtime_mappings: {
runtime_field: {
type: 'keyword',
script: {
lang: 'painless',
source: 'emit("runtime value")',
},
},
},
}),
}),
}),
@ -258,7 +270,7 @@ const createSourceConfigurationMock = (): InfraSource => ({
description: 'SOURCE_DESCRIPTION',
logIndices: {
type: 'index_pattern',
indexPatternId: 'some-test-id',
indexPatternId: 'test-index-pattern',
},
metricAlias: 'metric-indices-*',
inventoryDefaultView: 'DEFAULT_VIEW',
@ -323,5 +335,33 @@ const createDataPluginMock = (esSearchStrategyMock: ISearchStrategy): any => ({
search: {
getSearchStrategy: jest.fn().mockReturnValue(esSearchStrategyMock),
},
indexPatterns: getIndexPatternsMock(),
indexPatterns: createIndexPatternsStartMock(0, [
createIndexPatternMock({
id: 'test-index-pattern',
title: 'log-indices-*',
timeFieldName: '@timestamp',
fields: [
{
name: 'event.dataset',
type: 'string',
esTypes: ['keyword'],
aggregatable: true,
searchable: true,
},
{
name: 'runtime_field',
type: 'string',
runtimeField: {
type: 'keyword',
script: {
source: 'emit("runtime value")',
},
},
esTypes: ['keyword'],
aggregatable: true,
searchable: true,
},
],
}),
]),
});

3
x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts
View file

@ -109,7 +109,7 @@ export const logEntriesSearchStrategyProvider = ({
forkJoin([resolvedSourceConfiguration$, messageFormattingRules$]).pipe(
map(
([
{ indices, timestampField, tiebreakerField, columns },
{ indices, timestampField, tiebreakerField, columns, runtimeMappings },
messageFormattingRules,
]): IEsSearchRequest => {
return {
@ -123,6 +123,7 @@ export const logEntriesSearchStrategyProvider = ({
timestampField,
tiebreakerField,
getRequiredFields(params.columns ?? columns, messageFormattingRules),
runtimeMappings,
params.query,
params.highlightPhrase
),

69
x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts
View file

@ -18,14 +18,17 @@ import {
ISearchStrategy,
SearchStrategyDependencies,
} from 'src/plugins/data/server';
import { getIndexPatternsMock } from './mocks';
import { createSearchSessionsClientMock } from '../../../../../../src/plugins/data/server/search/mocks';
import {
createIndexPatternMock,
createIndexPatternsStartMock,
} from '../../../common/dependency_mocks/index_patterns';
import { InfraSource } from '../../../common/source_configuration/source_configuration';
import { createInfraSourcesMock } from '../../lib/sources/mocks';
import {
logEntrySearchRequestStateRT,
logEntrySearchStrategyProvider,
} from './log_entry_search_strategy';
import { createSearchSessionsClientMock } from '../../../../../../src/plugins/data/server/search/mocks';
import { InfraSource } from '../../../common/source_configuration/source_configuration';
describe('LogEntry search strategy', () => {
it('handles initial search requests', async () => {
@ -61,7 +64,33 @@ describe('LogEntry search strategy', () => {
.toPromise();
expect(sourcesMock.getSourceConfiguration).toHaveBeenCalled();
expect(esSearchStrategyMock.search).toHaveBeenCalled();
expect(esSearchStrategyMock.search).toHaveBeenCalledWith(
{
params: expect.objectContaining({
index: 'log-indices-*',
body: expect.objectContaining({
query: {
ids: {
values: ['LOG_ENTRY_ID'],
},
},
runtime_mappings: {
runtime_field: {
type: 'keyword',
script: {
lang: 'painless',
source: 'emit("runtime value")',
},
},
},
}),
terminate_after: 1,
track_total_hits: false,
}),
},
expect.anything(),
expect.anything()
);
expect(response.id).toEqual(expect.any(String));
expect(response.isRunning).toBe(true);
});
@ -207,7 +236,7 @@ const createSourceConfigurationMock = (): InfraSource => ({
description: 'SOURCE_DESCRIPTION',
logIndices: {
type: 'index_pattern',
indexPatternId: 'some-test-id',
indexPatternId: 'test-index-pattern',
},
metricAlias: 'metric-indices-*',
inventoryDefaultView: 'DEFAULT_VIEW',
@ -261,5 +290,33 @@ const createDataPluginMock = (esSearchStrategyMock: ISearchStrategy): any => ({
search: {
getSearchStrategy: jest.fn().mockReturnValue(esSearchStrategyMock),
},
indexPatterns: getIndexPatternsMock(),
indexPatterns: createIndexPatternsStartMock(0, [
createIndexPatternMock({
id: 'test-index-pattern',
title: 'log-indices-*',
timeFieldName: '@timestamp',
fields: [
{
name: 'event.dataset',
type: 'string',
esTypes: ['keyword'],
aggregatable: true,
searchable: true,
},
{
name: 'runtime_field',
type: 'string',
runtimeField: {
type: 'keyword',
script: {
source: 'emit("runtime value")',
},
},
esTypes: ['keyword'],
aggregatable: true,
searchable: true,
},
],
}),
]),
});

10
x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.ts
View file

@ -78,13 +78,19 @@ export const logEntrySearchStrategyProvider = ({
concatMap(({ params }) =>
resolvedSourceConfiguration$.pipe(
map(
({ indices, timestampField, tiebreakerField }): IEsSearchRequest => ({
({
indices,
timestampField,
tiebreakerField,
runtimeMappings,
}): IEsSearchRequest => ({
// @ts-expect-error @elastic/elasticsearch declares indices_boost as Record<string, number>
params: createGetLogEntryQuery(
indices,
params.logEntryId,
timestampField,
tiebreakerField
tiebreakerField,
runtimeMappings
),
})
)

37
x-pack/plugins/infra/server/services/log_entries/mocks.ts
View file

@ -1,37 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IIndexPattern, IFieldType, IndexPatternsContract } from 'src/plugins/data/common';
const indexPatternFields: IFieldType[] = [
{
name: 'event.dataset',
type: 'string',
esTypes: ['keyword'],
aggregatable: true,
filterable: true,
searchable: true,
},
];
const indexPattern: IIndexPattern = {
id: '1234',
title: 'log-indices-*',
timeFieldName: '@timestamp',
fields: indexPatternFields,
};
export const getIndexPatternsMock = (): any => {
return {
indexPatternsServiceFactory: async () => {
return {
get: async (id) => indexPattern,
getFieldsForWildcard: async (options) => indexPatternFields,
} as Pick<IndexPatternsContract, 'get' | 'getFieldsForWildcard'>;
},
};
};

2
x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts
View file

@ -29,6 +29,7 @@ export const createGetLogEntriesQuery = (
timestampField: string,
tiebreakerField: string,
fields: string[],
runtimeMappings?: estypes.RuntimeFields,
query?: JsonObject,
highlightTerm?: string
): estypes.AsyncSearchSubmitRequest => {
@ -53,6 +54,7 @@ export const createGetLogEntriesQuery = (
},
// @ts-expect-error @elastic/elasticsearch doesn't declare body.fields on AsyncSearchSubmitRequest
fields,
runtime_mappings: runtimeMappings,
_source: false,
...createSortClause(sortDirection, timestampField, tiebreakerField),
...createSearchAfterClause(cursor),

4
x-pack/plugins/infra/server/services/log_entries/queries/log_entry.ts
View file

@ -17,7 +17,8 @@ export const createGetLogEntryQuery = (
logEntryIndex: string,
logEntryId: string,
timestampField: string,
tiebreakerField: string
tiebreakerField: string,
runtimeMappings?: estypes.RuntimeFields
): estypes.AsyncSearchSubmitRequest => ({
index: logEntryIndex,
terminate_after: 1,
@ -32,6 +33,7 @@ export const createGetLogEntryQuery = (
},
// @ts-expect-error @elastic/elasticsearch doesn't declare body.fields on AsyncSearchSubmitRequest
fields: ['*'],
runtime_mappings: runtimeMappings,
sort: [{ [timestampField]: 'desc' }, { [tiebreakerField]: 'desc' }],
_source: false,
},