github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ML] Field Stats: Use field caps option include_empty_fields=false to identify populated fields. (#205417)

Browse source 
## Summary

Part of #178606.

Uses `dataViews.getFieldsForIndexPattern()` instead of custom code to
identify populated fields for field stats and the data grid used in the
Data Frame Analytics wizard.

- The previous custom code supported abort signals to cancel requests as
well as runtime fields. This was not yet supported by
`getFieldsForIndexPattern/getFieldsForWildcard`, so this PR adds that
capability.
- This also tweaks the options interface for `getFieldsForIndexPattern`
so you no longer have to pass in the empty `pattern: ''`.

This GIF demonstrates cancelling the request by navigating away from the
Data Frame Analytics wizard while the page is still loading (done with
3G throttling in dev tools):

![field-caps-cancel-0001](https://github.com/user-attachments/assets/8865ef08-76f0-4c84-a459-211230b2608e)

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Walter Rafelsberger 2025-01-15 15:16:17 +01:00 • committed by GitHub
parent 571ee960ad
commit abbdd0f826
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 365 additions and 327 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/platform/packages/shared/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts
View file

@ -51,8 +51,6 @@ export async function fetchFieldExistence({
dataViewsService: DataViewsContract;
}) {
const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, {
// filled in by data views service
pattern: '',
indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery),
includeEmptyFields: false,
});

32
src/platform/plugins/shared/data_views/common/data_views/data_views.test.ts
View file

@ -219,9 +219,7 @@ describe('IndexPatterns', () => {
});
test('getFieldsForIndexPattern called with allowHidden set to undefined as default', async () => {
await indexPatterns.getFieldsForIndexPattern({ id: '1' } as DataViewSpec, {
pattern: 'something',
});
await indexPatterns.getFieldsForIndexPattern({ id: '1' } as DataViewSpec);
expect(apiClient.getFieldsForWildcard).toBeCalledWith({
allowHidden: undefined,
allowNoIndex: true,
@ -233,9 +231,7 @@ describe('IndexPatterns', () => {
});
test('getFieldsForIndexPattern called with allowHidden set to true', async () => {
await indexPatterns.getFieldsForIndexPattern({ id: '1', allowHidden: true } as DataViewSpec, {
pattern: 'something',
});
await indexPatterns.getFieldsForIndexPattern({ id: '1', allowHidden: true } as DataViewSpec);
expect(apiClient.getFieldsForWildcard).toBeCalledWith({
allowHidden: true,
allowNoIndex: true,
@ -247,9 +243,7 @@ describe('IndexPatterns', () => {
});
test('getFieldsForIndexPattern called with allowHidden set to false', async () => {
await indexPatterns.getFieldsForIndexPattern({ id: '1', allowHidden: false } as DataViewSpec, {
pattern: 'something',
});
await indexPatterns.getFieldsForIndexPattern({ id: '1', allowHidden: false } as DataViewSpec);
expect(apiClient.getFieldsForWildcard).toBeCalledWith({
allowHidden: false,
allowNoIndex: true,
@ -261,12 +255,10 @@ describe('IndexPatterns', () => {
});
test('getFieldsForIndexPattern called with getAllowHidden returning true', async () => {
await indexPatterns.getFieldsForIndexPattern(
{ id: '1', getAllowHidden: () => true } as DataView,
{
pattern: 'something',
}
);
await indexPatterns.getFieldsForIndexPattern({
id: '1',
getAllowHidden: () => true,
} as DataView);
expect(apiClient.getFieldsForWildcard).toBeCalledWith({
allowHidden: true,
allowNoIndex: true,
@ -278,12 +270,10 @@ describe('IndexPatterns', () => {
});
test('getFieldsForIndexPattern called with getAllowHidden returning false', async () => {
await indexPatterns.getFieldsForIndexPattern(
{ id: '1', getAllowHidden: () => false } as DataView,
{
pattern: 'something',
}
);
await indexPatterns.getFieldsForIndexPattern({
id: '1',
getAllowHidden: () => false,
} as DataView);
expect(apiClient.getFieldsForWildcard).toBeCalledWith({
allowHidden: false,
allowNoIndex: true,

6
src/platform/plugins/shared/data_views/common/data_views/data_views.ts
View file

@ -242,7 +242,7 @@ export interface DataViewsServicePublicMethods {
*/
getFieldsForIndexPattern: (
indexPattern: DataView | DataViewSpec,
options?: GetFieldsOptions | undefined
options?: Omit<GetFieldsOptions, 'allowNoIndex' | 'pattern'>
) => Promise<FieldSpec[]>;
/**
* Get fields for index pattern string
@ -593,13 +593,13 @@ export class DataViewsService {
};
/**
* Get field list by providing an index patttern (or spec).
* Get field list by providing an index pattern (or spec).
* @param options options for getting field list
* @returns FieldSpec[]
*/
getFieldsForIndexPattern = async (
indexPattern: DataView | DataViewSpec,
options?: Omit<GetFieldsOptions, 'allowNoIndex'>
options?: Omit<GetFieldsOptions, 'allowNoIndex' | 'pattern'>
) =>
this.getFieldsForWildcard({
type: indexPattern.type,

2
src/platform/plugins/shared/data_views/common/types.ts
View file

@ -324,6 +324,8 @@ export interface GetFieldsOptions {
forceRefresh?: boolean;
fieldTypes?: string[];
includeEmptyFields?: boolean;
abortSignal?: AbortSignal;
runtimeMappings?: estypes.MappingRuntimeFields;
}
/**

3
src/platform/plugins/shared/data_views/public/data_views/data_views_api_client.test.mock.ts
View file

@ -12,3 +12,6 @@ import { setup } from '@kbn/core-test-helpers-http-setup-browser';
export const { http } = setup((injectedMetadata) => {
injectedMetadata.getBasePath.mockReturnValue('/hola/daro/');
});
export const indexFilterMock = { bool: { must: [{ match_all: {} }] } };
export const runtimeMappingsMock = { myField: { type: 'keyword' } };

38
src/platform/plugins/shared/data_views/public/data_views/data_views_api_client.test.ts
View file

@ -8,9 +8,10 @@
*/
import type { HttpSetup } from '@kbn/core/public';
import { http } from './data_views_api_client.test.mock';
import { DataViewsApiClient } from './data_views_api_client';
import { http, indexFilterMock, runtimeMappingsMock } from './data_views_api_client.test.mock';
import { getFieldsForWildcardRequestBody, DataViewsApiClient } from './data_views_api_client';
import { FIELDS_PATH as expectedPath } from '../../common/constants';
import type { GetFieldsOptions } from '../../common';
describe('IndexPatternsApiClient', () => {
let fetchSpy: jest.SpyInstance;
@ -56,3 +57,36 @@ describe('IndexPatternsApiClient', () => {
expect(fetchSpy.mock.calls[0][1].query.field_types).toEqual(fieldTypes);
});
});
describe('getFieldsForWildcardRequestBody', () => {
test('returns undefined if no indexFilter or runtimeMappings', () => {
expect(getFieldsForWildcardRequestBody({} as unknown as GetFieldsOptions)).toBeUndefined();
});
test('returns just indexFilter if no runtimeMappings', () => {
expect(
getFieldsForWildcardRequestBody({
indexFilter: indexFilterMock,
} as unknown as GetFieldsOptions)
).toEqual(JSON.stringify({ index_filter: indexFilterMock }));
});
test('returns just runtimeMappings if no indexFilter', () => {
expect(
getFieldsForWildcardRequestBody({
runtimeMappings: runtimeMappingsMock,
} as unknown as GetFieldsOptions)
).toEqual(JSON.stringify({ runtime_mappings: runtimeMappingsMock }));
});
test('returns both indexFilter and runtimeMappings', () => {
expect(
getFieldsForWildcardRequestBody({
indexFilter: indexFilterMock,
runtimeMappings: runtimeMappingsMock,
} as unknown as GetFieldsOptions)
).toEqual(
JSON.stringify({ index_filter: indexFilterMock, runtime_mappings: runtimeMappingsMock })
);
});
});

47
src/platform/plugins/shared/data_views/public/data_views/data_views_api_client.ts
View file

@ -29,6 +29,24 @@ async function sha1(str: string) {
}
}
/**
* Helper function to get the request body for the getFieldsForWildcard request
* @param options options for fields request
* @returns string | undefined
*/
export function getFieldsForWildcardRequestBody(options: GetFieldsOptions): string | undefined {
const { indexFilter, runtimeMappings } = options;
if (!indexFilter && !runtimeMappings) {
return;
}
return JSON.stringify({
...(indexFilter && { index_filter: indexFilter }),
...(runtimeMappings && { runtime_mappings: runtimeMappings }),
});
}
/**
* Data Views API Client - client implementation
*/
@ -49,7 +67,8 @@ export class DataViewsApiClient implements IDataViewsApiClient {
url: string,
query?: {},
body?: string,
forceRefresh?: boolean
forceRefresh?: boolean,
abortSignal?: AbortSignal
): Promise<HttpResponse<T> | undefined> {
const asResponse = true;
const cacheOptions: { cache?: RequestCache } = forceRefresh ? { cache: 'no-cache' } : {};
@ -59,21 +78,33 @@ export class DataViewsApiClient implements IDataViewsApiClient {
const headers = userHash ? { 'user-hash': userHash } : undefined;
const request = body
? this.http.post<T>(url, { query, body, version, asResponse })
? this.http.post<T>(url, { query, body, version, asResponse, signal: abortSignal })
: this.http.fetch<T>(url, {
query,
version,
...cacheOptions,
asResponse,
headers,
signal: abortSignal,
});
return request.catch((resp) => {
if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') {
throw new DataViewMissingIndices(resp.body.message);
// Custom errors with a body
if (resp?.body) {
if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') {
throw new DataViewMissingIndices(resp.body.message);
}
throw new Error(resp.body.message || resp.body.error || `${resp.body.statusCode} Response`);
}
throw new Error(resp.body.message || resp.body.error || `${resp.body.statusCode} Response`);
// Regular errors including AbortError
if (typeof resp?.name === 'string' && typeof resp?.message === 'string') {
throw resp;
}
// Other unknown errors
throw new Error('Unknown error');
});
}
@ -99,6 +130,7 @@ export class DataViewsApiClient implements IDataViewsApiClient {
allowHidden,
fieldTypes,
includeEmptyFields,
abortSignal,
} = options;
const path = indexFilter ? FIELDS_FOR_WILDCARD_PATH : FIELDS_PATH;
const versionQueryParam = indexFilter ? {} : { apiVersion: version };
@ -119,8 +151,9 @@ export class DataViewsApiClient implements IDataViewsApiClient {
include_empty_fields: includeEmptyFields,
...versionQueryParam,
},
indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined,
forceRefresh
getFieldsForWildcardRequestBody(options),
forceRefresh,
abortSignal
).then((response) => {
return {
indices: response?.body?.indices || [],

6
src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts
View file

@ -82,6 +82,8 @@ export class IndexPatternsFetcher {
allowHidden?: boolean;
fieldTypes?: string[];
includeEmptyFields?: boolean;
abortSignal?: AbortSignal;
runtimeMappings?: estypes.MappingRuntimeFields;
}): Promise<{ fields: FieldDescriptor[]; indices: string[] }> {
const {
pattern,
@ -93,6 +95,8 @@ export class IndexPatternsFetcher {
allowHidden,
fieldTypes,
includeEmptyFields,
abortSignal,
runtimeMappings,
} = options;
const allowNoIndices = fieldCapsOptions?.allow_no_indices || this.allowNoIndices;
@ -112,6 +116,8 @@ export class IndexPatternsFetcher {
expandWildcards,
fieldTypes,
includeEmptyFields,
runtimeMappings,
abortSignal,
});
if (this.rollupsEnabled && type === DataViewType.ROLLUP && rollupIndex) {

12
src/platform/plugins/shared/data_views/server/fetcher/lib/es_api.ts
View file

@ -8,7 +8,10 @@
*/
import { ElasticsearchClient } from '@kbn/core/server';
import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type {
ExpandWildcard,
MappingRuntimeFields,
} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { QueryDslQueryContainer } from '../../../common/types';
import { convertEsError } from './errors';
@ -50,6 +53,8 @@ interface FieldCapsApiParams {
expandWildcards?: ExpandWildcard;
fieldTypes?: string[];
includeEmptyFields?: boolean;
runtimeMappings?: MappingRuntimeFields;
abortSignal?: AbortSignal;
}
/**
@ -77,6 +82,8 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
expandWildcards,
fieldTypes,
includeEmptyFields,
runtimeMappings,
abortSignal,
} = params;
try {
return await callCluster.fieldCaps(
@ -88,9 +95,10 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
expand_wildcards: expandWildcards,
types: fieldTypes,
include_empty_fields: includeEmptyFields ?? true,
runtime_mappings: runtimeMappings,
...fieldCapsOptions,
},
{ meta: true }
{ meta: true, signal: abortSignal }
);
} catch (error) {
// return an empty set for closed indices

2
src/platform/plugins/shared/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js
View file

@ -34,6 +34,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
const fillUndefinedParams = (args) => ({
callCluster: undefined,
abortSignal: undefined,
indices: undefined,
expandWildcards: undefined,
fieldCapsOptions: undefined,
@ -41,6 +42,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
indexFilter: undefined,
fields: undefined,
includeEmptyFields: undefined,
runtimeMappings: undefined,
...args,
});

11
src/platform/plugins/shared/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts
View file

@ -9,7 +9,10 @@
import { defaults, keyBy, sortBy } from 'lodash';
import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type {
ExpandWildcard,
MappingRuntimeFields,
} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server';
import { callFieldCapsApi } from '../es_api';
import { readFieldCapsResponse } from './field_caps_response';
@ -30,6 +33,8 @@ interface FieldCapabilitiesParams {
expandWildcards?: ExpandWildcard;
fieldTypes?: string[];
includeEmptyFields?: boolean;
runtimeMappings?: MappingRuntimeFields;
abortSignal?: AbortSignal;
}
/**
@ -54,6 +59,8 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
expandWildcards,
fieldTypes,
includeEmptyFields,
runtimeMappings,
abortSignal,
} = params;
const excludedTiers = await uiSettingsClient?.get<string>(DATA_VIEWS_FIELDS_EXCLUDED_TIERS);
@ -66,6 +73,8 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
expandWildcards,
fieldTypes,
includeEmptyFields,
runtimeMappings,
abortSignal,
});
const fieldCapsArr = readFieldCapsResponse(esFieldCaps.body);
const fieldsFromFieldCapsByName = keyBy(fieldCapsArr, 'name');

4
src/platform/plugins/shared/data_views/server/index_patterns_api_client.ts
View file

@ -33,6 +33,8 @@ export class IndexPatternsApiServer implements IDataViewsApiClient {
indexFilter,
fields,
includeEmptyFields,
abortSignal,
runtimeMappings,
}: GetFieldsOptions) {
const indexPatterns = new IndexPatternsFetcher(this.esClient, {
uiSettingsClient: this.uiSettingsClient,
@ -48,6 +50,8 @@ export class IndexPatternsApiServer implements IDataViewsApiClient {
indexFilter,
fields,
includeEmptyFields,
abortSignal,
runtimeMappings,
})
.catch((err) => {
if (

34
src/platform/plugins/shared/data_views/server/rest_api_routes/internal/fields_for.ts
View file

@ -7,6 +7,7 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import type { Observable } from 'rxjs';
import { estypes } from '@elastic/elasticsearch';
import { schema } from '@kbn/config-schema';
import type { IRouter, RequestHandler, RouteAuthz, StartServicesAccessor } from '@kbn/core/server';
@ -20,6 +21,19 @@ import type {
import type { FieldDescriptorRestResponse } from '../route_types';
import { FIELDS_FOR_WILDCARD_PATH as path } from '../../../common/constants';
/**
* Copied from `@kbn/data-plugin/server` to avoid a cyclic dependency.
*
* A simple utility function that returns an `AbortSignal` corresponding to an `AbortController`
* which aborts when the given request is aborted.
* @param aborted$ The observable of abort events (usually `request.events.aborted$`)
*/
function getRequestAbortedSignal(aborted$: Observable<void>): AbortSignal {
const controller = new AbortController();
aborted$.subscribe(() => controller.abort());
return controller.signal;
}
/**
* Accepts one of the following:
* 1. An array of field names
@ -42,7 +56,19 @@ export const parseFields = (fields: string | string[], fldName: string): string[
const access = 'internal';
export type IBody = { index_filter?: estypes.QueryDslQueryContainer } | undefined;
export type IBody =
| {
index_filter?: estypes.QueryDslQueryContainer;
runtime_mappings?: estypes.MappingRuntimeFields;
}
| undefined;
export const bodySchema = schema.maybe(
schema.object({
index_filter: schema.maybe(schema.any()),
runtime_mappings: schema.maybe(schema.any()),
})
);
export interface IQuery {
pattern: string;
meta_fields: string | string[];
@ -111,7 +137,7 @@ export const validate: VersionedRouteValidation<any, any, any> = {
request: {
query: querySchema,
// not available to get request
body: schema.maybe(schema.object({ index_filter: schema.any() })),
body: bodySchema,
},
response: {
200: {
@ -126,6 +152,7 @@ export const validate: VersionedRouteValidation<any, any, any> = {
const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> =
(isRollupsEnabled) => async (context, request, response) => {
const abortSignal = getRequestAbortedSignal(request.events.aborted$);
const core = await context.core;
const { asCurrentUser } = core.elasticsearch.client;
const uiSettings = core.uiSettings.client;
@ -148,6 +175,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
// not available to get request
const indexFilter = request.body?.index_filter;
const runtimeMappings = request.body?.runtime_mappings;
let parsedFields: string[] = [];
let parsedMetaFields: string[] = [];
@ -174,7 +202,9 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
indexFilter,
allowHidden,
includeEmptyFields,
runtimeMappings,
...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
abortSignal,
});
const body: { fields: FieldDescriptorRestResponse[]; indices: string[] } = {

1
x-pack/platform/packages/private/ml/data_grid/index.ts
View file

@ -12,6 +12,7 @@ export {
getFeatureImportance,
getFieldsFromKibanaDataView,
getNestedOrEscapedVal,
getPopulatedFieldsFromKibanaDataView,
getProcessedFields,
getTopClasses,
multiColumnSortFactory,

26
x-pack/platform/packages/private/ml/data_grid/lib/common.test.ts
View file

@ -5,10 +5,14 @@
* 2.0.
*/
import type { DataViewField } from '@kbn/data-views-plugin/common';
import type { DataView, DataViewField } from '@kbn/data-views-plugin/common';
import type { MultiColumnSorter } from './common';
import { getDataGridSchemaFromKibanaFieldType, multiColumnSortFactory } from './common';
import {
getDataGridSchemaFromKibanaFieldType,
getPopulatedFieldsFromKibanaDataView,
multiColumnSortFactory,
} from './common';
const data = [
{ s: 'a', n: 1 },
@ -18,6 +22,24 @@ const data = [
];
describe('Data Grid Common', () => {
describe('getPopulatedFieldsFromKibanaDataView', () => {
it('returns populated fields from a kibana data view', () => {
const populatedFields = getPopulatedFieldsFromKibanaDataView(
{
fields: [
{ name: '_source', type: 'json' },
{ name: 'airline', type: 'string' },
{ name: 'response', type: 'number' },
],
metaFields: ['_source'],
} as DataView,
['airline', 'response', 'does_not_exist']
);
expect(populatedFields).toStrictEqual(['airline', 'response']);
});
});
describe('multiColumnSortFactory', () => {
it('returns desc sorted by one column', () => {
const sortingColumns1: MultiColumnSorter[] = [{ id: 's', direction: 'desc', type: 'number' }];

15
x-pack/platform/packages/private/ml/data_grid/lib/common.ts
View file

@ -102,6 +102,21 @@ export const getFieldsFromKibanaDataView = (dataView: DataView): string[] => {
return dataViewFields;
};
/**
* Retrieves just the populated fields from a Kibana data view.
* @param {DataView} dataView - The Kibana data view.
* @param {string[]} [populatedFields] - The populated fields.
* returns {string[]} - The array of populated fields from the data view.
*/
export const getPopulatedFieldsFromKibanaDataView = (
dataView: DataView,
populatedFields?: string[]
): string[] => {
const allPopulatedFields = Array.isArray(populatedFields) ? populatedFields : [];
const allDataViewFields = getFieldsFromKibanaDataView(dataView);
return allPopulatedFields.filter((d) => allDataViewFields.includes(d)).sort();
};
/**
* Record of ES field types.
*/

77
x-pack/platform/packages/private/ml/field_stats_flyout/field_stats_flyout_provider.tsx
View file

@ -5,18 +5,23 @@
* 2.0.
*/
import type { PropsWithChildren, FC } from 'react';
import React, { useCallback, useState } from 'react';
import React, {
useCallback,
useEffect,
useRef,
useState,
type FC,
type PropsWithChildren,
} from 'react';
import useMountedState from 'react-use/lib/useMountedState';
import type { DataView } from '@kbn/data-plugin/common';
import type { FieldStatsServices } from '@kbn/unified-field-list/src/components/field_stats';
import type { TimeRange as TimeRangeMs } from '@kbn/ml-date-picker';
import type { FieldStatsProps } from '@kbn/unified-field-list/src/components/field_stats';
import { useEffect } from 'react';
import { getProcessedFields } from '@kbn/ml-data-grid';
import { stringHash } from '@kbn/ml-string-hash';
import { lastValueFrom } from 'rxjs';
import { useRef } from 'react';
import { getMergedSampleDocsForPopulatedFieldsQuery } from './populated_fields/get_merged_populated_fields_query';
import { getRangeFilter } from './populated_fields/get_range_filter';
import { FieldStatsFlyout } from './field_stats_flyout';
import { MLFieldStatsFlyoutContext } from './use_field_stats_flyout_context';
import { PopulatedFieldsCacheManager } from './populated_fields/populated_fields_cache_manager';
@ -68,7 +73,6 @@ export const FieldStatsFlyoutProvider: FC<FieldStatsFlyoutProviderProps> = (prop
disablePopulatedFields = false,
children,
} = props;
const { search } = fieldStatsServices.data;
const [isFieldStatsFlyoutVisible, setFieldStatsIsFlyoutVisible] = useState(false);
const [fieldName, setFieldName] = useState<string | undefined>();
const [fieldValue, setFieldValue] = useState<string | number | undefined>();
@ -80,63 +84,43 @@ export const FieldStatsFlyoutProvider: FC<FieldStatsFlyoutProviderProps> = (prop
const [manager] = useState(new PopulatedFieldsCacheManager());
const [populatedFields, setPopulatedFields] = useState<Set<string> | undefined>();
const abortController = useRef(new AbortController());
const isMounted = useMountedState();
useEffect(
function fetchSampleDocsEffect() {
function fetchPopulatedFieldsEffect() {
if (disablePopulatedFields) return;
let unmounted = false;
if (abortController.current) {
abortController.current.abort();
abortController.current = new AbortController();
}
const queryAndRunTimeMappings = getMergedSampleDocsForPopulatedFieldsQuery({
searchQuery: dslQuery,
runtimeFields: dataView.getRuntimeMappings(),
datetimeField: dataView.getTimeField()?.name,
timeRange: timeRangeMs,
});
const indexPattern = dataView.getIndexPattern();
const esSearchRequestParams = {
index: indexPattern,
body: {
fields: ['*'],
_source: false,
...queryAndRunTimeMappings,
size: 500,
},
};
const cacheKey = stringHash(JSON.stringify(esSearchRequestParams)).toString();
const indexFilter = getRangeFilter(dataView.getTimeField()?.name, timeRangeMs);
const cacheKey = stringHash(JSON.stringify(indexFilter)).toString();
const fetchSampleDocuments = async function () {
const fetchPopulatedFields = async function () {
try {
const resp = await lastValueFrom(
search.search(
{
params: esSearchRequestParams,
},
{ abortSignal: abortController.current.signal }
)
const nonEmptyFields = await fieldStatsServices.dataViews.getFieldsForIndexPattern(
dataView,
{
includeEmptyFields: false,
indexFilter,
runtimeMappings: dataView.getRuntimeMappings(),
abortSignal: abortController.current.signal,
}
);
const docs = resp.rawResponse.hits.hits.map((d) => getProcessedFields(d.fields ?? {}));
// Get all field names for each returned doc and flatten it
// to a list of unique field names used across all docs.
const fieldsWithData = new Set(docs.map(Object.keys).flat(1));
const fieldsWithData = new Set([...nonEmptyFields.map((field) => field.name)]);
manager.set(cacheKey, fieldsWithData);
if (!unmounted) {
if (isMounted()) {
setPopulatedFields(fieldsWithData);
}
} catch (e) {
if (e.name !== 'AbortError') {
if (e?.name !== 'AbortError') {
// eslint-disable-next-line no-console
console.error(
`An error occurred fetching sample documents to determine populated field stats.
\nQuery:\n${JSON.stringify(esSearchRequestParams)}
`An error occurred fetching field caps to determine populated fields.
\nError:${e}`
);
}
@ -147,11 +131,10 @@ export const FieldStatsFlyoutProvider: FC<FieldStatsFlyoutProviderProps> = (prop
if (cachedResult) {
return cachedResult;
} else {
fetchSampleDocuments();
fetchPopulatedFields();
}
return () => {
unmounted = true;
abortController.current.abort();
};
},

106
x-pack/platform/packages/private/ml/field_stats_flyout/populated_fields/get_merged_populated_fields_query.test.ts
View file

@ -1,106 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getMergedSampleDocsForPopulatedFieldsQuery } from './get_merged_populated_fields_query';
describe('getMergedSampleDocsForPopulatedFieldsQuery()', () => {
it('should wrap the original query in function_score', () => {
expect(
getMergedSampleDocsForPopulatedFieldsQuery({
searchQuery: { match_all: {} },
runtimeFields: {},
})
).toStrictEqual({
query: {
function_score: { query: { bool: { must: [{ match_all: {} }] } }, random_score: {} },
},
runtime_mappings: {},
});
});
it('should append the time range to the query if timeRange and datetimeField are provided', () => {
expect(
getMergedSampleDocsForPopulatedFieldsQuery({
searchQuery: {
bool: {
should: [{ match_phrase: { version: '1' } }],
minimum_should_match: 1,
filter: [
{
terms: {
cluster_uuid: '',
},
},
],
must_not: [],
},
},
runtimeFields: {},
timeRange: { from: 1613995874349, to: 1614082617000 },
datetimeField: '@timestamp',
})
).toStrictEqual({
query: {
function_score: {
query: {
bool: {
filter: [
{ terms: { cluster_uuid: '' } },
{
range: {
'@timestamp': {
format: 'epoch_millis',
gte: 1613995874349,
lte: 1614082617000,
},
},
},
],
minimum_should_match: 1,
must_not: [],
should: [{ match_phrase: { version: '1' } }],
},
},
random_score: {},
},
},
runtime_mappings: {},
});
});
it('should not append the time range to the query if datetimeField is undefined', () => {
expect(
getMergedSampleDocsForPopulatedFieldsQuery({
searchQuery: {
bool: {
should: [{ match_phrase: { airline: 'AAL' } }],
minimum_should_match: 1,
filter: [],
must_not: [],
},
},
runtimeFields: {},
timeRange: { from: 1613995874349, to: 1614082617000 },
})
).toStrictEqual({
query: {
function_score: {
query: {
bool: {
filter: [],
minimum_should_match: 1,
must_not: [],
should: [{ match_phrase: { airline: 'AAL' } }],
},
},
random_score: {},
},
},
runtime_mappings: {},
});
});
});

74
x-pack/platform/packages/private/ml/field_stats_flyout/populated_fields/get_merged_populated_fields_query.ts
View file

@ -1,74 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { TimeRange as TimeRangeMs } from '@kbn/ml-date-picker';
import { isPopulatedObject } from '@kbn/ml-is-populated-object';
import { cloneDeep } from 'lodash';
import { getDefaultDSLQuery } from '@kbn/ml-query-utils';
export const getMergedSampleDocsForPopulatedFieldsQuery = ({
runtimeFields,
searchQuery,
datetimeField,
timeRange,
}: {
runtimeFields: estypes.MappingRuntimeFields;
searchQuery?: estypes.QueryDslQueryContainer;
datetimeField?: string;
timeRange?: TimeRangeMs;
}): {
query: estypes.QueryDslQueryContainer;
runtime_mappings?: estypes.MappingRuntimeFields;
} => {
let rangeFilter;
if (timeRange && datetimeField !== undefined) {
if (isPopulatedObject(timeRange, ['from', 'to']) && timeRange.to > timeRange.from) {
rangeFilter = {
range: {
[datetimeField]: {
gte: timeRange.from,
lte: timeRange.to,
format: 'epoch_millis',
},
},
};
}
}
const query = cloneDeep(
!searchQuery || isPopulatedObject(searchQuery, ['match_all'])
? getDefaultDSLQuery()
: searchQuery
);
if (rangeFilter && isPopulatedObject<string, estypes.QueryDslBoolQuery>(query, ['bool'])) {
if (Array.isArray(query.bool.filter)) {
query.bool.filter.push(rangeFilter);
} else {
query.bool.filter = [rangeFilter];
}
}
const queryAndRuntimeFields: {
query: estypes.QueryDslQueryContainer;
runtime_mappings?: estypes.MappingRuntimeFields;
} = {
query: {
function_score: {
query,
// @ts-expect-error random_score is valid dsl query
random_score: {},
},
},
};
if (runtimeFields) {
queryAndRuntimeFields.runtime_mappings = runtimeFields;
}
return queryAndRuntimeFields;
};

28
x-pack/platform/packages/private/ml/field_stats_flyout/populated_fields/get_range_filter.test.ts Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getRangeFilter } from './get_range_filter';
describe('getRangeFilter()', () => {
it('should return a dummy match_all filter when all arguments are undefined', () => {
expect(getRangeFilter()).toStrictEqual({
bool: { must: { match_all: {} } },
});
});
it('should return the range filter to the query if timeRange and datetimeField are provided', () => {
expect(getRangeFilter('@timestamp', { from: 1613995874349, to: 1614082617000 })).toStrictEqual({
range: {
'@timestamp': {
format: 'epoch_millis',
gte: 1613995874349,
lte: 1614082617000,
},
},
});
});
});

50
x-pack/platform/packages/private/ml/field_stats_flyout/populated_fields/get_range_filter.ts Normal file
View file

@ -0,0 +1,50 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { TimeRange as TimeRangeMs } from '@kbn/ml-date-picker';
import { isPopulatedObject } from '@kbn/ml-is-populated-object';
interface RangeFilter {
range: Record<string, estypes.QueryDslRangeQuery>;
}
interface MatchAllFilter {
bool: { must: { match_all: {} } };
}
/**
* Get range filter for datetime field. Both arguments are optional.
* @param datetimeField
* @param timeRange
* @returns range filter
*/
export const getRangeFilter = (
datetimeField?: string,
timeRange?: TimeRangeMs
): RangeFilter | MatchAllFilter => {
if (
datetimeField !== undefined &&
isPopulatedObject(timeRange, ['from', 'to']) &&
timeRange.to > timeRange.from
) {
return {
range: {
[datetimeField]: {
gte: timeRange.from,
lte: timeRange.to,
format: 'epoch_millis',
},
},
};
}
return {
bool: { must: { match_all: {} } },
};
};

1
x-pack/platform/packages/private/ml/field_stats_flyout/tsconfig.json
View file

@ -23,7 +23,6 @@
"@kbn/i18n",
"@kbn/react-field",
"@kbn/ml-anomaly-utils",
"@kbn/ml-data-grid",
"@kbn/ml-string-hash",
"@kbn/ml-is-populated-object",
"@kbn/ml-query-utils",

11
x-pack/platform/plugins/private/transform/public/app/hooks/use_index_data.ts
View file

@ -17,7 +17,7 @@ import {
getFieldType,
getDataGridSchemaFromKibanaFieldType,
getDataGridSchemaFromESFieldType,
getFieldsFromKibanaDataView,
getPopulatedFieldsFromKibanaDataView,
showDataGridColumnChartErrorMessageToast,
useDataGrid,
useRenderCellValue,
@ -83,12 +83,11 @@ export const useIndexData = (options: UseIndexDataOptions): UseIndexDataReturnTy
[baseFilterCriteria]
);
const dataViewFields = useMemo(() => {
const allPopulatedFields = Array.isArray(populatedFields) ? populatedFields : [];
const allDataViewFields = getFieldsFromKibanaDataView(dataView);
return allPopulatedFields.filter((d) => allDataViewFields.includes(d)).sort();
const dataViewFields = useMemo(
() => getPopulatedFieldsFromKibanaDataView(dataView, populatedFields),
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [populatedFields]);
[populatedFields]
);
const columns: EuiDataGridColumn[] = useMemo(() => {
let result: Array<{ id: string; schema: string | undefined }> = [];

103
x-pack/platform/plugins/shared/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts
View file

@ -5,7 +5,8 @@
* 2.0.
*/
import { useEffect, useMemo, useState } from 'react';
import { useEffect, useMemo, useRef, useState } from 'react';
import useMountedState from 'react-use/lib/useMountedState';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { EuiDataGridColumn } from '@elastic/eui';
@ -26,7 +27,7 @@ import {
getFieldType,
getDataGridSchemaFromKibanaFieldType,
getDataGridSchemaFromESFieldType,
getFieldsFromKibanaDataView,
getPopulatedFieldsFromKibanaDataView,
showDataGridColumnChartErrorMessageToast,
useDataGrid,
useRenderCellValue,
@ -34,7 +35,7 @@ import {
INDEX_STATUS,
} from '@kbn/ml-data-grid';
import { useMlApi } from '../../../../contexts/kibana';
import { useMlApi, useMlKibana } from '../../../../contexts/kibana';
import { DataLoader } from '../../../../datavisualizer/index_based/data_loader';
type IndexSearchResponse = estypes.SearchResponse;
@ -81,56 +82,16 @@ export const useIndexData = (
toastNotifications: CoreSetup['notifications']['toasts'],
runtimeMappings?: RuntimeMappings
): UseIndexDataReturnType => {
const isMounted = useMountedState();
const {
services: {
data: { dataViews: dataViewsService },
},
} = useMlKibana();
const mlApi = useMlApi();
// Fetch 500 random documents to determine populated fields.
// This is a workaround to avoid passing potentially thousands of unpopulated fields
// (for example, as part of filebeat/metricbeat/ECS based indices)
// to the data grid component which would significantly slow down the page.
const [dataViewFields, setDataViewFields] = useState<string[]>();
const [timeRangeMs, setTimeRangeMs] = useState<TimeRangeMs | undefined>();
useEffect(() => {
async function fetchDataGridSampleDocuments() {
setErrorMessage('');
setStatus(INDEX_STATUS.LOADING);
const esSearchRequest = {
index: dataView.title,
body: {
fields: ['*'],
_source: false,
query: {
function_score: {
query: { match_all: {} },
random_score: {},
},
},
size: 500,
},
};
try {
const resp: IndexSearchResponse = await mlApi.esSearch(esSearchRequest);
const docs = resp.hits.hits.map((d) => getProcessedFields(d.fields ?? {}));
// Get all field names for each returned doc and flatten it
// to a list of unique field names used across all docs.
const allDataViewFields = getFieldsFromKibanaDataView(dataView);
const populatedFields = [...new Set(docs.map(Object.keys).flat(1))]
.filter((d) => allDataViewFields.includes(d))
.sort();
setStatus(INDEX_STATUS.LOADED);
setDataViewFields(populatedFields);
} catch (e) {
setErrorMessage(extractErrorMessage(e));
setStatus(INDEX_STATUS.ERROR);
}
}
fetchDataGridSampleDocuments();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
const abortController = useRef(new AbortController());
// To be used for data grid column selection
// and will be applied to doc and chart queries.
@ -139,6 +100,48 @@ export const useIndexData = (
[dataView, runtimeMappings]
);
useEffect(() => {
async function fetchPopulatedFields() {
if (abortController.current) {
abortController.current.abort();
abortController.current = new AbortController();
}
setErrorMessage('');
setStatus(INDEX_STATUS.LOADING);
try {
const nonEmptyFields = await dataViewsService.getFieldsForIndexPattern(dataView, {
includeEmptyFields: false,
// dummy filter, if no filter was provided the function would return all fields.
indexFilter: {
bool: { must: { match_all: {} } },
},
runtimeMappings: combinedRuntimeMappings,
abortSignal: abortController.current.signal,
});
const populatedFields = nonEmptyFields.map((field) => field.name);
if (isMounted()) {
setDataViewFields(getPopulatedFieldsFromKibanaDataView(dataView, populatedFields));
}
} catch (e) {
if (e?.name !== 'AbortError') {
setErrorMessage(extractErrorMessage(e));
setStatus(INDEX_STATUS.ERROR);
}
}
}
fetchPopulatedFields();
return () => {
abortController.current.abort();
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
// Available data grid columns, will be a combination of index pattern and runtime fields.
const [columns, setColumns] = useState<MLEuiDataGridColumn[]>([]);
useEffect(() => {

1
x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_exceptions/logic/use_exception_flyout_data.tsx
View file

@ -130,7 +130,6 @@ export const useFetchIndexPatterns = (rules: Rule[] | null): ReturnUseFetchExcep
}
try {
extendedFields = await data.dataViews.getFieldsForIndexPattern(dv, {
pattern: '',
includeUnmapped: true,
fields,
});