[data views] Allow data views created on hidden and system indices (#168174)

## Summary

Previously, the 'Allow hidden and system indices' advanced option when
creating a data view was only a UI convenience. It allowed you to see
which hidden and system indices you were matching but they would be
would be selected just the same once the data view was loaded. At some
point something changed and now there are system and hidden indices that
require `expandWildcards: hidden` to be passed to field caps in order to
see anything. `allowHidden: boolean` is added to the DataView and
DataViewSpec and passed through when field caps requests are made.

This is primarily a tool for troubleshooting. For instance, instead of
hitting a full data stream across a number of data tiers you can select
a specific index to compare its performance.

Closes: https://github.com/elastic/kibana/issues/164652

packages/kbn-optimizer/limits.yml

@@ -31,7 +31,7 @@ pageLoadAssetSize:
   dataViewEditor: 28082
   dataViewFieldEditor: 27000
   dataViewManagement: 5000
-  dataViews: 47300
+  dataViews: 48300
   dataVisualizer: 27530
   devTools: 38637
   discover: 99999

src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx

@@ -124,6 +124,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
         timeFieldName: formData.timestampField?.value,
         id: formData.id,
         name: formData.name,
+        allowHidden: formData.allowHidden,
       };
 
       if (type === INDEX_PATTERN_TYPE.ROLLUP && rollupIndex) {

src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap

@@ -28,6 +28,7 @@ exports[`IndexPatterns delete will throw if insufficient access 1`] = `[DataView
 
 exports[`IndexPatterns savedObjectToSpec 1`] = `
 Object {
+  "allowHidden": undefined,
   "allowNoIndex": undefined,
   "fieldAttrs": Object {
     "aRuntimeField": Object {

src/plugins/data_views/common/data_views/data_view.ts

@@ -152,6 +152,8 @@ export class DataView implements DataViewBase {
    */
   public matchedIndices: string[] = [];
 
+  private allowHidden: boolean = false;
+
   /**
    * constructor
    * @param config - config data and dependencies
@@ -187,6 +189,7 @@ export class DataView implements DataViewBase {
     this.runtimeFieldMap = cloneDeep(spec.runtimeFieldMap) || {};
     this.namespaces = spec.namespaces || [];
     this.name = spec.name || '';
+    this.allowHidden = spec.allowHidden || false;
   }
 
   /**
@@ -201,6 +204,8 @@ export class DataView implements DataViewBase {
 
   getIndexPattern = () => this.title;
 
+  getAllowHidden = () => this.allowHidden;
+
   /**
    * Set index pattern
    * @param string index pattern string

src/plugins/data_views/common/data_views/data_views.ts

@@ -584,6 +584,7 @@ export class DataViewsService {
       allowNoIndex: true,
       pattern: dataView.getIndexPattern(),
       metaFields,
+      allowHidden: dataView.getAllowHidden(),
     });
   };
 
@@ -596,6 +597,7 @@ export class DataViewsService {
       rollupIndex: options.rollupIndex,
       allowNoIndex: true,
       indexFilter: options.indexFilter,
+      allowHidden: options.allowHidden,
     });
   };
 
@@ -747,6 +749,7 @@ export class DataViewsService {
         fieldAttrs,
         allowNoIndex,
         name,
+        allowHidden,
       },
     } = savedObject;
 
@@ -774,6 +777,7 @@ export class DataViewsService {
       allowNoIndex,
       runtimeFieldMap: parsedRuntimeFieldMap,
       name,
+      allowHidden,
     };
   };
 

src/plugins/data_views/common/types.ts

@@ -157,6 +157,10 @@ export interface DataViewAttributes {
    * Name of the data view. Human readable name used to differentiate data view.
    */
   name?: string;
+  /**
+   * Allow hidden and system indices when loading field list
+   */
+  allowHidden?: boolean;
 }
 
 /**
@@ -309,6 +313,7 @@ export interface GetFieldsOptions {
   indexFilter?: QueryDslQueryContainer;
   includeUnmapped?: boolean;
   fields?: string[];
+  allowHidden?: boolean;
 }
 
 /**
@@ -512,6 +517,10 @@ export type DataViewSpec = {
    * Name of the data view. Human readable name used to differentiate data view.
    */
   name?: string;
+  /**
+   * Whether the data view is hidden from the user
+   */
+  allowHidden?: boolean;
 };
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-definitions

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -69,12 +69,23 @@ export class IndexPatternsFetcher {
     rollupIndex?: string;
     indexFilter?: QueryDslQueryContainer;
     fields?: string[];
+    allowHidden?: boolean;
   }): Promise<{ fields: FieldDescriptor[]; indices: string[] }> {
-    const { pattern, metaFields = [], fieldCapsOptions, type, rollupIndex, indexFilter } = options;
+    const {
+      pattern,
+      metaFields = [],
+      fieldCapsOptions,
+      type,
+      rollupIndex,
+      indexFilter,
+      allowHidden,
+    } = options;
     const allowNoIndices = fieldCapsOptions
       ? fieldCapsOptions.allow_no_indices
       : this.allowNoIndices;
 
+    const expandWildcards = allowHidden ? 'all' : 'open';
+
     const fieldCapsResponse = await getFieldCapabilities({
       callCluster: this.elasticsearchClient,
       indices: pattern,
@@ -85,6 +96,7 @@ export class IndexPatternsFetcher {
       },
       indexFilter,
       fields: options.fields || ['*'],
+      expandWildcards,
     });
 
     if (this.rollupsEnabled && type === 'rollup' && rollupIndex) {

src/plugins/data_views/server/fetcher/lib/es_api.ts

@@ -7,6 +7,7 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core/server';
+import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { QueryDslQueryContainer } from '../../../common/types';
 import { convertEsError } from './errors';
 
@@ -45,6 +46,7 @@ interface FieldCapsApiParams {
   fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
   indexFilter?: QueryDslQueryContainer;
   fields?: string[];
+  expandWildcard?: ExpandWildcard;
 }
 
 /**

src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js

@@ -34,6 +34,7 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
   const fillUndefinedParams = (args) => ({
     callCluster: undefined,
     indices: undefined,
+    expandWildcard: undefined,
     fieldCapsOptions: undefined,
     indexFilter: undefined,
     fields: undefined,

src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts

@@ -9,6 +9,7 @@
 import { defaults, keyBy, sortBy } from 'lodash';
 
 import { ElasticsearchClient } from '@kbn/core/server';
+import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { callFieldCapsApi } from '../es_api';
 import { readFieldCapsResponse } from './field_caps_response';
 import { mergeOverrides } from './overrides';
@@ -22,6 +23,7 @@ interface FieldCapabilitiesParams {
   fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
   indexFilter?: QueryDslQueryContainer;
   fields?: string[];
+  expandWildcards?: ExpandWildcard;
 }
 
 /**
@@ -42,6 +44,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
     indexFilter,
     metaFields = [],
     fields,
+    expandWildcards,
   } = params;
   const esFieldCaps = await callFieldCapsApi({
     callCluster,
@@ -49,6 +52,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
     fieldCapsOptions,
     indexFilter,
     fields,
+    expandWildcard: expandWildcards,
   });
   const fieldCapsArr = readFieldCapsResponse(esFieldCaps.body);
   const fieldsFromFieldCapsByName = keyBy(fieldCapsArr, 'name');

src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts

@@ -50,6 +50,7 @@ interface IQuery {
   allow_no_index?: boolean;
   include_unmapped?: boolean;
   fields?: string[];
+  allow_hidden?: boolean;
 }
 
 const querySchema = schema.object({
@@ -62,6 +63,7 @@ const querySchema = schema.object({
   allow_no_index: schema.maybe(schema.boolean()),
   include_unmapped: schema.maybe(schema.boolean()),
   fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])),
+  allow_hidden: schema.maybe(schema.boolean()),
 });
 
 const fieldSubTypeSchema = schema.object({
@@ -122,6 +124,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
       rollup_index: rollupIndex,
       allow_no_index: allowNoIndex,
       include_unmapped: includeUnmapped,
+      allow_hidden: allowHidden,
     } = request.query;
 
     // not available to get request
@@ -147,6 +150,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           includeUnmapped,
         },
         indexFilter,
+        allowHidden,
         ...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
       });
 

test/functional/apps/management/data_views/_data_view_create_delete.ts

@@ -17,6 +17,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const retry = getService('retry');
   const testSubjects = getService('testSubjects');
   const find = getService('find');
+  const es = getService('es');
   const PageObjects = getPageObjects(['settings', 'common', 'header']);
 
   describe('creating and deleting default data view', function describeIndexTests() {
@@ -250,5 +251,32 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         });
       });
     });
+
+    describe('hidden index support', () => {
+      it('can create data view against hidden index', async () => {
+        const pattern = 'logstash-2015.09.21';
+
+        await es.transport.request({
+          path: '/logstash-2015.09.21/_settings',
+          method: 'PUT',
+          body: {
+            index: {
+              hidden: true,
+            },
+          },
+        });
+
+        await PageObjects.settings.createIndexPattern(
+          pattern,
+          undefined,
+          undefined,
+          undefined,
+          undefined,
+          true
+        );
+        const patternName = await PageObjects.settings.getIndexPageHeading();
+        expect(patternName).to.be(pattern);
+      });
+    });
   });
 }

test/functional/page_objects/settings_page.ts

@@ -470,13 +470,19 @@ export class SettingsPageObject extends FtrService {
     await customDataViewIdInput.type(value);
   }
 
+  async allowHiddenClick() {
+    await this.testSubjects.click('toggleAdvancedSetting');
+    await this.testSubjects.click('allowHiddenField');
+  }
+
   async createIndexPattern(
     indexPatternName: string,
     // null to bypass default value
     timefield: string | null = '@timestamp',
     isStandardIndexPattern = true,
     customDataViewId?: string,
-    dataViewName?: string
+    dataViewName?: string,
+    allowHidden?: boolean
   ) {
     await this.retry.try(async () => {
       await this.header.waitUntilLoadingHasFinished();
@@ -489,6 +495,11 @@ export class SettingsPageObject extends FtrService {
       } else {
         await this.clickAddNewIndexPatternButton();
       }
+
+      if (allowHidden) {
+        await this.allowHiddenClick();
+      }
+
       await this.header.waitUntilLoadingHasFinished();
       if (!isStandardIndexPattern) {
         await this.selectRollupIndexPatternType();

x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts

@@ -249,6 +249,7 @@ describe('LogViewsClient class', () => {
           },
         ],
         "dataViewReference": DataView {
+          "allowHidden": false,
           "allowNoIndex": false,
           "deleteFieldFormat": [Function],
           "fieldAttrs": Object {},
@@ -273,6 +274,7 @@ describe('LogViewsClient class', () => {
           },
           "fields": FldList [],
           "flattenHit": [Function],
+          "getAllowHidden": [Function],
           "getFieldAttrs": [Function],
           "getIndexPattern": [Function],
           "getName": [Function],