github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[data views] Allow data views created on hidden and system indices - second attempt (#168882)

Browse source 
## Summary

Previously, the 'Allow hidden and system indices' advanced option when
creating a data view was only a UI convenience. It allowed you to see
which hidden and system indices you were matching but they would be
would be selected just the same once the data view was loaded. At some
point something changed and now there are system and hidden indices that
require `expandWildcards: hidden` to be passed to field caps in order to
see anything. `allowHidden: boolean` is added to the DataView and
DataViewSpec and passed through when field caps requests are made.

This is primarily a tool for troubleshooting. For instance, instead of
hitting a full data stream across a number of data tiers you can select
a specific index to compare its performance.

NOTE: This is a second attempt. What I learned - the whole
`expand_wildcards` param is literal - you can directly query a hidden
index without `expandWildcards: hidden` since its not using a wildcard.
Tests now use a wildcard.

Closes: https://github.com/elastic/kibana/issues/164652

---------

Co-authored-by: Lukas Olson <lukas@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Matthew Kime 2023-11-14 14:31:40 -06:00 • committed by GitHub
parent 11b47c461f
commit 6c926c77f2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 174 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

47
src/plugins/data/common/search/search_source/fetch/get_search_params.test.ts
View file

@ -9,6 +9,7 @@
import { UI_SETTINGS } from '../../../constants';
import { GetConfigFn } from '../../../types';
import { getSearchParams, getSearchParamsFromRequest } from './get_search_params';
import { createStubDataView } from '@kbn/data-views-plugin/common/data_views/data_view.stub';
function getConfigStub(config: any = {}): GetConfigFn {
return (key) => config[key];
@ -46,4 +47,50 @@ describe('getSearchParams', () => {
query: 123,
});
});
test('sets expand_wildcards=all if data view has allowHidden=true', () => {
const getConfig = getConfigStub({
[UI_SETTINGS.COURIER_SET_REQUEST_PREFERENCE]: 'custom',
[UI_SETTINGS.COURIER_CUSTOM_REQUEST_PREFERENCE]: 'aaa',
});
const index = createStubDataView({
spec: {
allowHidden: true,
},
});
const searchParams = getSearchParamsFromRequest(
{
index,
body: {
query: 123,
track_total_hits: true,
},
},
{ getConfig }
);
expect(searchParams).toHaveProperty('expand_wildcards', 'all');
});
test('does not set expand_wildcards if data view has allowHidden=false', () => {
const getConfig = getConfigStub({
[UI_SETTINGS.COURIER_SET_REQUEST_PREFERENCE]: 'custom',
[UI_SETTINGS.COURIER_CUSTOM_REQUEST_PREFERENCE]: 'aaa',
});
const index = createStubDataView({
spec: {
allowHidden: false,
},
});
const searchParams = getSearchParamsFromRequest(
{
index,
body: {
query: 123,
track_total_hits: true,
},
},
{ getConfig }
);
expect(searchParams).not.toHaveProperty('expand_wildcards', 'all');
});
});

2
src/plugins/data/common/search/search_source/fetch/get_search_params.ts
View file

@ -42,8 +42,8 @@ export function getSearchParamsFromRequest(
return {
index: searchRequest.index.title || searchRequest.index,
body,
// @ts-expect-error `track_total_hits` not allowed at top level for `typesWithBodyKey`
track_total_hits,
...(searchRequest.index?.allowHidden && { expand_wildcards: 'all' }),
...searchParams,
};
}

4
src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_content.tsx
View file

@ -30,14 +30,16 @@ interface AdvancedParamsContentProps {
disableAllowHidden: boolean;
disableId: boolean;
onAllowHiddenChange?: (value: boolean) => void;
defaultVisible?: boolean;
}
export const AdvancedParamsContent = ({
disableAllowHidden,
disableId,
onAllowHiddenChange,
defaultVisible = false,
}: AdvancedParamsContentProps) => (
<AdvancedParamsSection>
<AdvancedParamsSection defaultVisible={defaultVisible}>
<EuiFlexGroup>
<EuiFlexItem>
<UseField<boolean, IndexPatternConfig>

5
src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_section.tsx
View file

@ -13,10 +13,11 @@ import { EuiButtonEmpty, EuiSpacer } from '@elastic/eui';
interface Props {
children: React.ReactNode;
defaultVisible: boolean;
}
export const AdvancedParamsSection = ({ children }: Props) => {
const [isVisible, setIsVisible] = useState<boolean>(false);
export const AdvancedParamsSection = ({ children, defaultVisible = false }: Props) => {
const [isVisible, setIsVisible] = useState<boolean>(defaultVisible);
const toggleIsVisible = useCallback(() => {
setIsVisible(!isVisible);

3
src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx
View file

@ -105,6 +105,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
title: editData.getIndexPattern(),
id: editData.id,
name: editData.name,
allowHidden: editData.getAllowHidden(),
...(editData.timeFieldName
? {
timestampField: { label: editData.timeFieldName, value: editData.timeFieldName },
@ -124,6 +125,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
timeFieldName: formData.timestampField?.value,
id: formData.id,
name: formData.name,
allowHidden: formData.allowHidden,
};
if (type === INDEX_PATTERN_TYPE.ROLLUP && rollupIndex) {
@ -293,6 +295,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
onAllowHiddenChange={() => {
form.getFields().title.validate();
}}
defaultVisible={editData?.getAllowHidden()}
/>
</Form>
<Footer

3
src/plugins/data_view_editor/public/components/data_view_flyout_content_container.tsx
View file

@ -50,10 +50,11 @@ const DataViewFlyoutContentContainer = ({
try {
let saveResponse;
if (editData) {
const { name = '', timeFieldName, title = '' } = dataViewSpec;
const { name = '', timeFieldName, title = '', allowHidden = false } = dataViewSpec;
editData.setIndexPattern(title);
editData.name = name;
editData.timeFieldName = timeFieldName;
editData.setAllowHidden(allowHidden);
saveResponse = editData.isPersisted()
? await dataViews.updateSavedObject(editData)
: editData;

1
src/plugins/data_view_editor/public/data_view_editor_service.ts
View file

@ -342,6 +342,7 @@ export class DataViewEditorService {
const getFieldsOptions: GetFieldsOptions = {
pattern: this.indexPattern,
allowHidden: this.allowHidden,
};
if (this.type === INDEX_PATTERN_TYPE.ROLLUP) {
getFieldsOptions.type = INDEX_PATTERN_TYPE.ROLLUP;

1
src/plugins/data_views/common/content_management/v1/cm_services.ts
View file

@ -46,6 +46,7 @@ const dataViewAttributesSchema = schema.object(
allowNoIndex: schema.maybe(schema.boolean()),
runtimeFieldMap: schema.maybe(schema.any()),
name: schema.maybe(schema.string()),
allowHidden: schema.maybe(schema.boolean()),
},
{ unknowns: 'forbid' }
);

3
src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap generated
View file

@ -28,6 +28,7 @@ Object {
exports[`IndexPattern toMinimalSpec can exclude fields 1`] = `
Object {
"allowHidden": false,
"allowNoIndex": false,
"fieldAttrs": undefined,
"fieldFormats": Object {},
@ -51,6 +52,7 @@ Object {
exports[`IndexPattern toSpec can optionally exclude fields 1`] = `
Object {
"allowHidden": false,
"allowNoIndex": false,
"fieldAttrs": Object {},
"fieldFormats": Object {},
@ -74,6 +76,7 @@ Object {
exports[`IndexPattern toSpec should match snapshot 1`] = `
Object {
"allowHidden": false,
"allowNoIndex": false,
"fieldAttrs": Object {
"@timestamp": Object {

1
src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap generated
View file

@ -28,6 +28,7 @@ exports[`IndexPatterns delete will throw if insufficient access 1`] = `[DataView
exports[`IndexPatterns savedObjectToSpec 1`] = `
Object {
"allowHidden": undefined,
"allowNoIndex": undefined,
"fieldAttrs": Object {
"aRuntimeField": Object {

8
src/plugins/data_views/common/data_views/abstract_data_views.ts
View file

@ -126,6 +126,8 @@ export abstract class AbstractDataView {
protected scriptedFields: DataViewFieldBase[];
private allowHidden: boolean = false;
constructor(config: AbstractDataViewDeps) {
const { spec = {}, fieldFormats, shortDotsEnable = false, metaFields = [] } = config;
@ -178,8 +180,13 @@ export abstract class AbstractDataView {
this.runtimeFieldMap = cloneDeep(spec.runtimeFieldMap) || {};
this.namespaces = spec.namespaces || [];
this.name = spec.name || '';
this.allowHidden = spec.allowHidden || false;
}
getAllowHidden = () => this.allowHidden;
setAllowHidden = (allowHidden: boolean) => (this.allowHidden = allowHidden);
/**
* Get name of Data View
*/
@ -325,6 +332,7 @@ export abstract class AbstractDataView {
allowNoIndex: this.allowNoIndex ? this.allowNoIndex : undefined,
runtimeFieldMap: stringifyOrUndefined(this.runtimeFieldMap),
name: this.name,
allowHidden: this.allowHidden,
};
}

1
src/plugins/data_views/common/data_views/data_view.ts
View file

@ -160,6 +160,7 @@ export class DataView extends AbstractDataView implements DataViewBase {
fieldAttrs,
allowNoIndex: this.allowNoIndex,
name: this.name,
allowHidden: this.getAllowHidden(),
};
// Filter undefined values from the spec

7
src/plugins/data_views/common/data_views/data_views.ts
View file

@ -531,6 +531,8 @@ export class DataViewsService {
allowNoIndex: true,
...options,
pattern: indexPattern.title as string,
allowHidden:
(indexPattern as DataViewSpec).allowHidden || (indexPattern as DataView)?.getAllowHidden(),
});
private getFieldsAndIndicesForDataView = async (dataView: DataView) => {
@ -541,6 +543,7 @@ export class DataViewsService {
allowNoIndex: true,
pattern: dataView.getIndexPattern(),
metaFields,
allowHidden: dataView.getAllowHidden(),
});
};
@ -553,6 +556,7 @@ export class DataViewsService {
rollupIndex: options.rollupIndex,
allowNoIndex: true,
indexFilter: options.indexFilter,
allowHidden: options.allowHidden,
});
};
@ -704,6 +708,7 @@ export class DataViewsService {
fieldAttrs,
allowNoIndex,
name,
allowHidden,
},
} = savedObject;
@ -731,6 +736,7 @@ export class DataViewsService {
allowNoIndex,
runtimeFieldMap: parsedRuntimeFieldMap,
name,
allowHidden,
};
};
@ -763,6 +769,7 @@ export class DataViewsService {
type,
rollupIndex: typeMeta?.params?.rollup_index,
allowNoIndex: spec.allowNoIndex,
allowHidden: spec.allowHidden,
},
spec.fieldAttrs,
displayErrors

9
src/plugins/data_views/common/types.ts
View file

@ -157,6 +157,10 @@ export interface DataViewAttributes {
* Name of the data view. Human readable name used to differentiate data view.
*/
name?: string;
/**
* Allow hidden and system indices when loading field list
*/
allowHidden?: boolean;
}
/**
@ -309,6 +313,7 @@ export interface GetFieldsOptions {
indexFilter?: QueryDslQueryContainer;
includeUnmapped?: boolean;
fields?: string[];
allowHidden?: boolean;
}
/**
@ -517,6 +522,10 @@ export type DataViewSpec = {
* Name of the data view. Human readable name used to differentiate data view.
*/
name?: string;
/**
* Allow hidden and system indices when loading field list
*/
allowHidden?: boolean;
};
// eslint-disable-next-line @typescript-eslint/consistent-type-definitions

2
src/plugins/data_views/public/data_views/data_views_api_client.ts
View file

@ -60,6 +60,7 @@ export class DataViewsApiClient implements IDataViewsApiClient {
indexFilter,
includeUnmapped,
fields,
allowHidden,
} = options;
return this._request<FieldsForWildcardResponse>(
FIELDS_FOR_WILDCARD_PATH,
@ -71,6 +72,7 @@ export class DataViewsApiClient implements IDataViewsApiClient {
allow_no_index: allowNoIndex,
include_unmapped: includeUnmapped,
fields,
allow_hidden: allowHidden,
},
indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined
).then((response) => {

1
src/plugins/data_views/server/content_management/data_views_storage.ts
View file

@ -37,6 +37,7 @@ export class DataViewsStorage extends SOContentStorage<DataViewCrudTypes> {
'runtimeFieldMap',
'allowNoIndex',
'name',
'allowHidden',
],
mSearchAdditionalSearchFields: ['name'],
logger,

14
src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts
View file

@ -71,12 +71,23 @@ export class IndexPatternsFetcher {
rollupIndex?: string;
indexFilter?: QueryDslQueryContainer;
fields?: string[];
allowHidden?: boolean;
}): Promise<{ fields: FieldDescriptor[]; indices: string[] }> {
const { pattern, metaFields = [], fieldCapsOptions, type, rollupIndex, indexFilter } = options;
const {
pattern,
metaFields = [],
fieldCapsOptions,
type,
rollupIndex,
indexFilter,
allowHidden,
} = options;
const allowNoIndices = fieldCapsOptions
? fieldCapsOptions.allow_no_indices
: this.allowNoIndices;
const expandWildcards = allowHidden ? 'all' : 'open';
const fieldCapsResponse = await getFieldCapabilities({
callCluster: this.elasticsearchClient,
indices: pattern,
@ -87,6 +98,7 @@ export class IndexPatternsFetcher {
},
indexFilter,
fields: options.fields || ['*'],
expandWildcards,
});
if (this.rollupsEnabled && type === 'rollup' && rollupIndex) {

4
src/plugins/data_views/server/fetcher/lib/es_api.ts
View file

@ -7,6 +7,7 @@
*/
import { ElasticsearchClient } from '@kbn/core/server';
import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { QueryDslQueryContainer } from '../../../common/types';
import { convertEsError } from './errors';
@ -45,6 +46,7 @@ interface FieldCapsApiParams {
fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
indexFilter?: QueryDslQueryContainer;
fields?: string[];
expandWildcards?: ExpandWildcard;
}
/**
@ -69,6 +71,7 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
include_unmapped: false,
},
fields = ['*'],
expandWildcards,
} = params;
try {
return await callCluster.fieldCaps(
@ -77,6 +80,7 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
fields,
ignore_unavailable: true,
index_filter: indexFilter,
expand_wildcards: expandWildcards,
...fieldCapsOptions,
},
{ meta: true }

1
src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js
View file

@ -34,6 +34,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
const fillUndefinedParams = (args) => ({
callCluster: undefined,
indices: undefined,
expandWildcards: undefined,
fieldCapsOptions: undefined,
indexFilter: undefined,
fields: undefined,

4
src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts
View file

@ -8,6 +8,7 @@
import { defaults, keyBy, sortBy } from 'lodash';
import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ElasticsearchClient } from '@kbn/core/server';
import { callFieldCapsApi } from '../es_api';
import { readFieldCapsResponse } from './field_caps_response';
@ -22,6 +23,7 @@ interface FieldCapabilitiesParams {
fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
indexFilter?: QueryDslQueryContainer;
fields?: string[];
expandWildcards?: ExpandWildcard;
}
/**
@ -42,6 +44,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
indexFilter,
metaFields = [],
fields,
expandWildcards,
} = params;
const esFieldCaps = await callFieldCapsApi({
callCluster,
@ -49,6 +52,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
fieldCapsOptions,
indexFilter,
fields,
expandWildcards,
});
const fieldCapsArr = readFieldCapsResponse(esFieldCaps.body);
const fieldsFromFieldCapsByName = keyBy(fieldCapsArr, 'name');

4
src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts
View file

@ -50,6 +50,7 @@ interface IQuery {
allow_no_index?: boolean;
include_unmapped?: boolean;
fields?: string[];
allow_hidden?: boolean;
}
const querySchema = schema.object({
@ -62,6 +63,7 @@ const querySchema = schema.object({
allow_no_index: schema.maybe(schema.boolean()),
include_unmapped: schema.maybe(schema.boolean()),
fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])),
allow_hidden: schema.maybe(schema.boolean()),
});
const fieldSubTypeSchema = schema.object({
@ -122,6 +124,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
rollup_index: rollupIndex,
allow_no_index: allowNoIndex,
include_unmapped: includeUnmapped,
allow_hidden: allowHidden,
} = request.query;
// not available to get request
@ -147,6 +150,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
includeUnmapped,
},
indexFilter,
allowHidden,
...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
});

1
src/plugins/data_views/server/rest_api_routes/route_types.ts
View file

@ -121,6 +121,7 @@ export type DataViewSpecRestResponse = {
allowNoIndex?: boolean;
namespaces?: string[];
name?: string;
allowHidden?: boolean;
};
export interface DataViewListItemRestResponse {

1
src/plugins/data_views/server/rest_api_routes/schema.ts
View file

@ -46,6 +46,7 @@ export const dataViewSpecSchema = schema.object({
runtimeFieldMap: schema.maybe(schema.recordOf(schema.string(), runtimeFieldSchema)),
name: schema.maybe(schema.string()),
namespaces: schema.maybe(schema.arrayOf(schema.string())),
allowHidden: schema.maybe(schema.boolean()),
});
export const dataViewsRuntimeResponseSchema = schema.object({

37
test/functional/apps/management/data_views/_data_view_create_delete.ts
View file

@ -17,6 +17,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const retry = getService('retry');
const testSubjects = getService('testSubjects');
const find = getService('find');
const es = getService('es');
const PageObjects = getPageObjects(['settings', 'common', 'header']);
describe('creating and deleting default data view', function describeIndexTests() {
@ -250,5 +251,41 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
});
});
});
describe('hidden index support', () => {
it('can create data view against hidden index', async () => {
const pattern = 'logstash-2015.09.2*';
await es.transport.request({
path: '/logstash-2015.09.2*/_settings',
method: 'PUT',
body: {
index: {
hidden: true,
},
},
});
await PageObjects.settings.createIndexPattern(
pattern,
'@timestamp',
undefined,
undefined,
undefined,
true
);
const patternName = await PageObjects.settings.getIndexPageHeading();
expect(patternName).to.be(pattern);
// verify that allow hidden persists through reload
await browser.refresh();
await testSubjects.click('editIndexPatternButton');
await testSubjects.click('toggleAdvancedSetting');
const allowHiddenField = await testSubjects.find('allowHiddenField');
const button = await allowHiddenField.findByTagName('button');
expect(await button.getAttribute('aria-checked')).to.be('true');
});
});
});
}

14
test/functional/page_objects/settings_page.ts
View file

@ -483,13 +483,20 @@ export class SettingsPageObject extends FtrService {
await customDataViewIdInput.type(value);
}
async allowHiddenClick() {
await this.testSubjects.click('toggleAdvancedSetting');
const allowHiddenField = await this.testSubjects.find('allowHiddenField');
(await allowHiddenField.findByTagName('button')).click();
}
async createIndexPattern(
indexPatternName: string,
// null to bypass default value
timefield: string | null = '@timestamp',
isStandardIndexPattern = true,
customDataViewId?: string,
dataViewName?: string
dataViewName?: string,
allowHidden?: boolean
) {
await this.retry.try(async () => {
await this.header.waitUntilLoadingHasFinished();
@ -502,6 +509,11 @@ export class SettingsPageObject extends FtrService {
} else {
await this.clickAddNewIndexPatternButton();
}
if (allowHidden) {
await this.allowHiddenClick();
}
await this.header.waitUntilLoadingHasFinished();
if (!isStandardIndexPattern) {
await this.selectRollupIndexPatternType();

3
x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts
View file

@ -249,6 +249,7 @@ describe('LogViewsClient class', () => {
},
],
"dataViewReference": DataView {
"allowHidden": false,
"allowNoIndex": false,
"deleteFieldFormat": [Function],
"deleteScriptedFieldInternal": [Function],
@ -274,6 +275,7 @@ describe('LogViewsClient class', () => {
},
"fields": FldList [],
"flattenHit": [Function],
"getAllowHidden": [Function],
"getFieldAttrs": [Function],
"getIndexPattern": [Function],
"getName": [Function],
@ -298,6 +300,7 @@ describe('LogViewsClient class', () => {
},
},
"scriptedFields": Array [],
"setAllowHidden": [Function],
"setFieldFormat": [Function],
"setIndexPattern": [Function],
"shortDotsEnable": false,