github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[DataViews] expose list of matched indices (#139067)

Browse source
This commit is contained in:
Peter Pisljar 2022-08-25 18:32:06 +02:00 • committed by GitHub
parent 135115d833
commit f68751054a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 100 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/kbn-optimizer/limits.yml
View file

@ -101,7 +101,7 @@ pageLoadAssetSize:
visTypeMetric: 23332
bfetch: 22837
kibanaUtils: 79713
dataViews: 43532
dataViews: 44532
expressions: 140958
fieldFormats: 65209
kibanaReact: 74422

5
src/plugins/data_views/common/data_views/data_view.ts
View file

@ -146,6 +146,11 @@ export class DataView implements DataViewBase {
*/
public name: string = '';
/*
* list of indices that the index pattern matched
*/
public matchedIndices: string[] = [];
/**
* constructor
* @param config - config data and dependencies

2
src/plugins/data_views/common/data_views/data_views.test.ts
View file

@ -21,7 +21,7 @@ import { stubbedSavedObjectIndexPattern } from '../data_view.stub';
const createFieldsFetcher = () =>
({
getFieldsForWildcard: jest.fn(async () => []),
getFieldsForWildcard: jest.fn(async () => ({ fields: [], indices: [] })),
} as any as IDataViewsApiClient);
const fieldFormats = fieldFormatsMock;

41
src/plugins/data_views/common/data_views/data_views.ts
View file

@ -478,7 +478,7 @@ export class DataViewsService {
*/
getFieldsForWildcard = async (options: GetFieldsOptions): Promise<FieldSpec[]> => {
const metaFields = await this.config.get<string[]>(META_FIELDS);
return this.apiClient.getFieldsForWildcard({
const { fields } = await this.apiClient.getFieldsForWildcard({
pattern: options.pattern,
metaFields,
type: options.type,
@ -486,6 +486,7 @@ export class DataViewsService {
allowNoIndex: options.allowNoIndex,
filter: options.filter,
});
return fields;
};
/**
@ -505,13 +506,36 @@ export class DataViewsService {
pattern: indexPattern.title as string,
});
private getFieldsAndIndicesForDataView = async (dataView: DataView) => {
const metaFields = await this.config.get<string[]>(META_FIELDS);
return this.apiClient.getFieldsForWildcard({
type: dataView.type,
rollupIndex: dataView?.typeMeta?.params?.rollup_index,
allowNoIndex: dataView.allowNoIndex,
pattern: dataView.title as string,
metaFields,
});
};
private getFieldsAndIndicesForWildcard = async (options: GetFieldsOptions) => {
const metaFields = await this.config.get<string[]>(META_FIELDS);
return await this.apiClient.getFieldsForWildcard({
pattern: options.pattern,
metaFields,
type: options.type,
rollupIndex: options.rollupIndex,
allowNoIndex: options.allowNoIndex,
filter: options.filter,
});
};
/**
* Refresh field list for a given index pattern.
* @param indexPattern
*/
refreshFields = async (indexPattern: DataView) => {
try {
const fields = (await this.getFieldsForIndexPattern(indexPattern)) as FieldSpec[];
const { fields, indices } = await this.getFieldsAndIndicesForDataView(indexPattern);
fields.forEach((field) => (field.isMapped = true));
const scripted = indexPattern.getScriptedFields().map((field) => field.spec);
const fieldAttrs = indexPattern.getFieldAttrs();
@ -527,6 +551,7 @@ export class DataViewsService {
!fieldsWithSavedAttrs.find((mappedField) => mappedField.name === runtimeField.name)
);
indexPattern.fields.replaceAll([...runtimeFieldsArray, ...fieldsWithSavedAttrs]);
indexPattern.matchedIndices = indices;
} catch (err) {
if (err instanceof DataViewMissingIndices) {
this.onNotification(
@ -567,7 +592,7 @@ export class DataViewsService {
const scriptedFields = fieldsAsArr.filter((field) => field.scripted);
try {
let updatedFieldList: FieldSpec[];
const newFields = (await this.getFieldsForWildcard(options)) as FieldSpec[];
const { fields: newFields, indices } = await this.getFieldsAndIndicesForWildcard(options);
newFields.forEach((field) => (field.isMapped = true));
// If allowNoIndex, only update field list if field caps finds fields. To support
@ -578,7 +603,7 @@ export class DataViewsService {
updatedFieldList = fieldsAsArr;
}
return this.fieldArrayToMap(updatedFieldList, fieldAttrs);
return { fields: this.fieldArrayToMap(updatedFieldList, fieldAttrs), indices };
} catch (err) {
if (err instanceof DataViewMissingIndices) {
this.onNotification(
@ -693,8 +718,10 @@ export class DataViewsService {
? JSON.parse(savedObject.attributes.fieldAttrs)
: {};
let matchedIndices: string[] = [];
try {
spec.fields = await this.refreshFieldSpecMap(
const { fields, indices } = await this.refreshFieldSpecMap(
spec.fields || {},
savedObject.id,
spec.title as string,
@ -708,6 +735,9 @@ export class DataViewsService {
spec.fieldAttrs
);
spec.fields = fields;
matchedIndices = indices || [];
const runtimeFieldSpecs = this.getRuntimeFields(runtimeFieldMap, spec.fieldAttrs);
// mapped fields overwrite runtime fields
spec.fields = { ...runtimeFieldSpecs, ...spec.fields };
@ -740,6 +770,7 @@ export class DataViewsService {
: {};
const indexPattern = await this.create(spec, true);
indexPattern.matchedIndices = matchedIndices;
indexPattern.resetOriginalSavedObjectBody();
return indexPattern;
};

10
src/plugins/data_views/common/types.ts
View file

@ -317,8 +317,16 @@ export interface GetFieldsOptions {
filter?: QueryDslQueryContainer;
}
/**
* FieldsForWildcard response
*/
export interface FieldsForWildcardResponse {
fields: FieldSpec[];
indices: string[];
}
export interface IDataViewsApiClient {
getFieldsForWildcard: (options: GetFieldsOptions) => Promise<FieldSpec[]>;
getFieldsForWildcard: (options: GetFieldsOptions) => Promise<FieldsForWildcardResponse>;
hasUserDataView: () => Promise<boolean>;
}

9
src/plugins/data_views/public/data_views/data_views_api_client.ts
View file

@ -8,7 +8,8 @@
import { HttpSetup } from '@kbn/core/public';
import { DataViewMissingIndices } from '../../common/lib';
import { FieldSpec, GetFieldsOptions, IDataViewsApiClient } from '../../common';
import { GetFieldsOptions, IDataViewsApiClient } from '../../common';
import { FieldsForWildcardResponse } from '../../common/types';
const API_BASE_URL: string = `/api/index_patterns/`;
@ -50,14 +51,16 @@ export class DataViewsApiClient implements IDataViewsApiClient {
*/
getFieldsForWildcard(options: GetFieldsOptions) {
const { pattern, metaFields, type, rollupIndex, allowNoIndex, filter } = options;
return this._request<{ fields: FieldSpec[] }>(this._getUrl(['_fields_for_wildcard']), {
return this._request<FieldsForWildcardResponse>(this._getUrl(['_fields_for_wildcard']), {
pattern,
meta_fields: metaFields,
type,
rollup_index: rollupIndex,
allow_no_index: allowNoIndex,
filter,
}).then((resp) => resp?.fields || []);
}).then((response) => {
return response || { fields: [], indices: [] };
});
}
/**

17
src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts
View file

@ -61,7 +61,7 @@ export class IndexPatternsFetcher {
type?: string;
rollupIndex?: string;
filter?: QueryDslQueryContainer;
}): Promise<FieldDescriptor[]> {
}): Promise<{ fields: FieldDescriptor[]; indices: string[] }> {
const { pattern, metaFields = [], fieldCapsOptions, type, rollupIndex, filter } = options;
const patternList = Array.isArray(pattern) ? pattern : pattern.split(',');
const allowNoIndices = fieldCapsOptions
@ -94,17 +94,20 @@ export class IndexPatternsFetcher {
}
const rollupIndexCapabilities = capabilityCheck.aggs;
const fieldCapsResponseObj = keyBy(fieldCapsResponse, 'name');
const fieldCapsResponseObj = keyBy(fieldCapsResponse.fields, 'name');
// Keep meta fields
metaFields!.forEach(
(field: string) =>
fieldCapsResponseObj[field] && rollupFields.push(fieldCapsResponseObj[field])
);
return mergeCapabilitiesWithFields(
rollupIndexCapabilities!,
fieldCapsResponseObj,
rollupFields
);
return {
fields: mergeCapabilitiesWithFields(
rollupIndexCapabilities!,
fieldCapsResponseObj,
rollupFields
),
indices: fieldCapsResponse.indices,
};
}
return fieldCapsResponse;
}

12
src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js
View file

@ -85,7 +85,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
fieldsFromFieldCaps: fields.map((name) => ({ name })),
});
const fieldNames = (await getFieldCapabilities(getArgsWithCallCluster())).map(
const fieldNames = (await getFieldCapabilities(getArgsWithCallCluster())).fields.map(
(field) => field.name
);
expect(fieldNames).toEqual(fields);
@ -99,7 +99,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
fieldsFromFieldCaps: shuffle(letters.map((name) => ({ name }))),
});
const fieldNames = (await getFieldCapabilities(getArgsWithCallCluster())).map(
const fieldNames = (await getFieldCapabilities(getArgsWithCallCluster())).fields.map(
(field) => field.name
);
expect(fieldNames).toEqual(sortedLetters);
@ -115,8 +115,8 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
const args = getArgsWithCallCluster({ metaFields: ['meta1', 'meta2'] });
const resp = await getFieldCapabilities(args);
expect(resp).toHaveLength(4);
expect(resp.map((field) => field.name)).toEqual(['bar', 'foo', 'meta1', 'meta2']);
expect(resp.fields).toHaveLength(4);
expect(resp.fields.map((field) => field.name)).toEqual(['bar', 'foo', 'meta1', 'meta2']);
});
});
@ -141,7 +141,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
fieldsFromFieldCaps: [field],
});
const resp = await getFieldCapabilities(getArgsWithCallCluster());
const { fields: resp } = await getFieldCapabilities(getArgsWithCallCluster());
expect(resp).toHaveLength(1);
expect(resp[0]).toHaveProperty(property);
expect(resp[0][property]).not.toBe(footballs[0]);
@ -185,7 +185,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
},
});
expect(await getFieldCapabilities(getArgsWithCallCluster())).toEqual([
expect((await getFieldCapabilities(getArgsWithCallCluster())).fields).toEqual([
{ notFieldAnymore: 1 },
{ notFieldAnymore: 1 },
]);

7
src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts
View file

@ -31,7 +31,7 @@ interface FieldCapabilitiesParams {
* @param {Array} [indices=[]] the list of indexes to check
* @param {Array} [metaFields=[]] the list of internal fields to include
* @param {Object} fieldCapsOptions
* @return {Promise<Array<FieldDescriptor>>}
* @return {Promise<{ fields: Array<FieldDescriptor>, indices: Array<string>>}>}
*/
export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
const { callCluster, indices = [], fieldCapsOptions, filter, metaFields = [] } = params;
@ -67,5 +67,8 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
)
.map(mergeOverrides);
return sortBy(allFieldsUnsorted, 'name');
return {
fields: sortBy(allFieldsUnsorted, 'name'),
indices: esFieldCaps.body.indices as string[],
};
}

4
src/plugins/data_views/server/routes/fields_for.ts
View file

@ -73,7 +73,7 @@ const handler: RequestHandler<{}, IQuery, IBody> = async (context, request, resp
}
try {
const fields = await indexPatterns.getFieldsForWildcard({
const { fields, indices } = await indexPatterns.getFieldsForWildcard({
pattern,
metaFields: parsedFields,
type,
@ -85,7 +85,7 @@ const handler: RequestHandler<{}, IQuery, IBody> = async (context, request, resp
});
return response.ok({
body: { fields },
body: { fields, indices },
headers: {
'content-type': 'application/json',
},

1
test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js
View file

@ -69,6 +69,7 @@ export default function ({ getService }) {
metadata_field: false,
},
],
indices: ['logs-2017.01.01', 'logs-2017.01.02'],
});
}));
});

3
test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js
View file

@ -85,6 +85,7 @@ export default function ({ getService }) {
.query({ pattern: 'basic_index' })
.expect(200, {
fields: testFields,
indices: ['basic_index'],
})
.then(ensureFieldsAreSorted);
});
@ -176,6 +177,7 @@ export default function ({ getService }) {
metadata_field: false,
},
],
indices: ['basic_index'],
})
.then(ensureFieldsAreSorted);
});
@ -186,6 +188,7 @@ export default function ({ getService }) {
.query({ pattern: 'bad_index,basic_index' })
.expect(200, {
fields: testFields,
indices: ['basic_index'],
});
});
it('returns 404 when no patterns exist', async () => {

2
x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.test.ts
View file

@ -75,7 +75,7 @@ describe('RuleDataClient', () => {
describe('getReader()', () => {
beforeEach(() => {
jest.resetAllMocks();
getFieldsForWildcardMock.mockResolvedValue(['foo']);
getFieldsForWildcardMock.mockResolvedValue({ fields: ['foo'] });
IndexPatternsFetcher.prototype.getFieldsForWildcard = getFieldsForWildcardMock;
});

2
x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts
View file

@ -131,7 +131,7 @@ export class RuleDataClient implements IRuleDataClient {
const indexPatternsFetcher = new IndexPatternsFetcher(clusterClient);
try {
const fields = await indexPatternsFetcher.getFieldsForWildcard({
const { fields } = await indexPatternsFetcher.getFieldsForWildcard({
pattern: indexPattern,
});

2
x-pack/plugins/synthetics/server/legacy_uptime/lib/requests/get_index_pattern.ts
View file

@ -29,7 +29,7 @@ export const getUptimeIndexPattern = async ({
// we have to catch errors here to avoid all endpoints returning 500 for users without APM data
// (would be a bad first time experience)
try {
const fields = await indexPatternsFetcher.getFieldsForWildcard({
const { fields } = await indexPatternsFetcher.getFieldsForWildcard({
pattern: dynamicSettings.heartbeatIndices,
});

22
x-pack/plugins/timelines/server/search_strategy/index_fields/index.ts
View file

@ -136,18 +136,20 @@ export const requestIndexFieldSearch = async (
[]
);
if (!request.onlyCheckIfIndicesExist) {
const fieldDescriptor = await Promise.all(
indicesExist.map(async (index, n) => {
if (index.startsWith('.alerts-observability')) {
return indexPatternsFetcherAsInternalUser.getFieldsForWildcard({
const fieldDescriptor = (
await Promise.all(
indicesExist.map(async (index, n) => {
if (index.startsWith('.alerts-observability')) {
return indexPatternsFetcherAsInternalUser.getFieldsForWildcard({
pattern: index,
});
}
return indexPatternsFetcherAsCurrentUser.getFieldsForWildcard({
pattern: index,
});
}
return indexPatternsFetcherAsCurrentUser.getFieldsForWildcard({
pattern: index,
});
})
);
})
)
).map((response) => response.fields || []);
indexFields = await formatIndexFields(beatFields, fieldDescriptor, patternList);
}
}