Revert "[data views] Allow data views created on hidden and system indices" (#168855)

Reverts elastic/kibana#168174

Reverting the previous PR - tests passed and the PR appeared to work
because ES is doesn't always enforce 'allowHidden' on hidden indices.

src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx

@@ -124,7 +124,6 @@ const IndexPatternEditorFlyoutContentComponent = ({
         timeFieldName: formData.timestampField?.value,
         id: formData.id,
         name: formData.name,
-        allowHidden: formData.allowHidden,
       };
 
       if (type === INDEX_PATTERN_TYPE.ROLLUP && rollupIndex) {

src/plugins/data_views/common/data_views/__snapshots__/data_views.test.ts.snap

@@ -28,7 +28,6 @@ exports[`IndexPatterns delete will throw if insufficient access 1`] = `[DataView
 
 exports[`IndexPatterns savedObjectToSpec 1`] = `
 Object {
-  "allowHidden": undefined,
   "allowNoIndex": undefined,
   "fieldAttrs": Object {
     "aRuntimeField": Object {

src/plugins/data_views/common/data_views/data_view.ts

@@ -152,8 +152,6 @@ export class DataView implements DataViewBase {
    */
   public matchedIndices: string[] = [];
 
-  private allowHidden: boolean = false;
-
   /**
    * constructor
    * @param config - config data and dependencies
@@ -189,7 +187,6 @@ export class DataView implements DataViewBase {
     this.runtimeFieldMap = cloneDeep(spec.runtimeFieldMap) || {};
     this.namespaces = spec.namespaces || [];
     this.name = spec.name || '';
-    this.allowHidden = spec.allowHidden || false;
   }
 
   /**
@@ -204,8 +201,6 @@ export class DataView implements DataViewBase {
 
   getIndexPattern = () => this.title;
 
-  getAllowHidden = () => this.allowHidden;
-
   /**
    * Set index pattern
    * @param string index pattern string

src/plugins/data_views/common/data_views/data_views.ts

@@ -584,7 +584,6 @@ export class DataViewsService {
       allowNoIndex: true,
       pattern: dataView.getIndexPattern(),
       metaFields,
-      allowHidden: dataView.getAllowHidden(),
     });
   };
 
@@ -597,7 +596,6 @@ export class DataViewsService {
       rollupIndex: options.rollupIndex,
       allowNoIndex: true,
       indexFilter: options.indexFilter,
-      allowHidden: options.allowHidden,
     });
   };
 
@@ -749,7 +747,6 @@ export class DataViewsService {
         fieldAttrs,
         allowNoIndex,
         name,
-        allowHidden,
       },
     } = savedObject;
 
@@ -777,7 +774,6 @@ export class DataViewsService {
       allowNoIndex,
       runtimeFieldMap: parsedRuntimeFieldMap,
       name,
-      allowHidden,
     };
   };
 

src/plugins/data_views/common/types.ts

@@ -157,10 +157,6 @@ export interface DataViewAttributes {
    * Name of the data view. Human readable name used to differentiate data view.
    */
   name?: string;
-  /**
-   * Allow hidden and system indices when loading field list
-   */
-  allowHidden?: boolean;
 }
 
 /**
@@ -313,7 +309,6 @@ export interface GetFieldsOptions {
   indexFilter?: QueryDslQueryContainer;
   includeUnmapped?: boolean;
   fields?: string[];
-  allowHidden?: boolean;
 }
 
 /**
@@ -517,10 +512,6 @@ export type DataViewSpec = {
    * Name of the data view. Human readable name used to differentiate data view.
    */
   name?: string;
-  /**
-   * Whether the data view is hidden from the user
-   */
-  allowHidden?: boolean;
 };
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-definitions

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -69,23 +69,12 @@ export class IndexPatternsFetcher {
     rollupIndex?: string;
     indexFilter?: QueryDslQueryContainer;
     fields?: string[];
-    allowHidden?: boolean;
   }): Promise<{ fields: FieldDescriptor[]; indices: string[] }> {
-    const {
-      pattern,
-      metaFields = [],
-      fieldCapsOptions,
-      type,
-      rollupIndex,
-      indexFilter,
-      allowHidden,
-    } = options;
+    const { pattern, metaFields = [], fieldCapsOptions, type, rollupIndex, indexFilter } = options;
     const allowNoIndices = fieldCapsOptions
       ? fieldCapsOptions.allow_no_indices
       : this.allowNoIndices;
 
-    const expandWildcards = allowHidden ? 'all' : 'open';
-
     const fieldCapsResponse = await getFieldCapabilities({
       callCluster: this.elasticsearchClient,
       indices: pattern,
@@ -96,7 +85,6 @@ export class IndexPatternsFetcher {
       },
       indexFilter,
       fields: options.fields || ['*'],
-      expandWildcards,
     });
 
     if (this.rollupsEnabled && type === 'rollup' && rollupIndex) {

src/plugins/data_views/server/fetcher/lib/es_api.ts

@@ -7,7 +7,6 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core/server';
-import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { QueryDslQueryContainer } from '../../../common/types';
 import { convertEsError } from './errors';
 
@@ -46,7 +45,6 @@ interface FieldCapsApiParams {
   fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
   indexFilter?: QueryDslQueryContainer;
   fields?: string[];
-  expandWildcard?: ExpandWildcard;
 }
 
 /**

src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.test.js

@@ -34,7 +34,6 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
   const fillUndefinedParams = (args) => ({
     callCluster: undefined,
     indices: undefined,
-    expandWildcard: undefined,
     fieldCapsOptions: undefined,
     indexFilter: undefined,
     fields: undefined,

src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts

@@ -9,7 +9,6 @@
 import { defaults, keyBy, sortBy } from 'lodash';
 
 import { ElasticsearchClient } from '@kbn/core/server';
-import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { callFieldCapsApi } from '../es_api';
 import { readFieldCapsResponse } from './field_caps_response';
 import { mergeOverrides } from './overrides';
@@ -23,7 +22,6 @@ interface FieldCapabilitiesParams {
   fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
   indexFilter?: QueryDslQueryContainer;
   fields?: string[];
-  expandWildcards?: ExpandWildcard;
 }
 
 /**
@@ -44,7 +42,6 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
     indexFilter,
     metaFields = [],
     fields,
-    expandWildcards,
   } = params;
   const esFieldCaps = await callFieldCapsApi({
     callCluster,
@@ -52,7 +49,6 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
     fieldCapsOptions,
     indexFilter,
     fields,
-    expandWildcard: expandWildcards,
   });
   const fieldCapsArr = readFieldCapsResponse(esFieldCaps.body);
   const fieldsFromFieldCapsByName = keyBy(fieldCapsArr, 'name');

src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts

@@ -50,7 +50,6 @@ interface IQuery {
   allow_no_index?: boolean;
   include_unmapped?: boolean;
   fields?: string[];
-  allow_hidden?: boolean;
 }
 
 const querySchema = schema.object({
@@ -63,7 +62,6 @@ const querySchema = schema.object({
   allow_no_index: schema.maybe(schema.boolean()),
   include_unmapped: schema.maybe(schema.boolean()),
   fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])),
-  allow_hidden: schema.maybe(schema.boolean()),
 });
 
 const fieldSubTypeSchema = schema.object({
@@ -124,7 +122,6 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
       rollup_index: rollupIndex,
       allow_no_index: allowNoIndex,
       include_unmapped: includeUnmapped,
-      allow_hidden: allowHidden,
     } = request.query;
 
     // not available to get request
@@ -150,7 +147,6 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           includeUnmapped,
         },
         indexFilter,
-        allowHidden,
         ...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
       });
 

test/functional/apps/management/data_views/_data_view_create_delete.ts

@@ -17,7 +17,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const retry = getService('retry');
   const testSubjects = getService('testSubjects');
   const find = getService('find');
-  const es = getService('es');
   const PageObjects = getPageObjects(['settings', 'common', 'header']);
 
   describe('creating and deleting default data view', function describeIndexTests() {
@@ -251,32 +250,5 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         });
       });
     });
-
-    describe('hidden index support', () => {
-      it('can create data view against hidden index', async () => {
-        const pattern = 'logstash-2015.09.21';
-
-        await es.transport.request({
-          path: '/logstash-2015.09.21/_settings',
-          method: 'PUT',
-          body: {
-            index: {
-              hidden: true,
-            },
-          },
-        });
-
-        await PageObjects.settings.createIndexPattern(
-          pattern,
-          undefined,
-          undefined,
-          undefined,
-          undefined,
-          true
-        );
-        const patternName = await PageObjects.settings.getIndexPageHeading();
-        expect(patternName).to.be(pattern);
-      });
-    });
   });
 }

test/functional/page_objects/settings_page.ts

@@ -470,19 +470,13 @@ export class SettingsPageObject extends FtrService {
     await customDataViewIdInput.type(value);
   }
 
-  async allowHiddenClick() {
-    await this.testSubjects.click('toggleAdvancedSetting');
-    await this.testSubjects.click('allowHiddenField');
-  }
-
   async createIndexPattern(
     indexPatternName: string,
     // null to bypass default value
     timefield: string | null = '@timestamp',
     isStandardIndexPattern = true,
     customDataViewId?: string,
-    dataViewName?: string,
-    allowHidden?: boolean
+    dataViewName?: string
   ) {
     await this.retry.try(async () => {
       await this.header.waitUntilLoadingHasFinished();
@@ -495,11 +489,6 @@ export class SettingsPageObject extends FtrService {
       } else {
         await this.clickAddNewIndexPatternButton();
       }
-
-      if (allowHidden) {
-        await this.allowHiddenClick();
-      }
-
       await this.header.waitUntilLoadingHasFinished();
       if (!isStandardIndexPattern) {
         await this.selectRollupIndexPatternType();

x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts

@@ -249,7 +249,6 @@ describe('LogViewsClient class', () => {
           },
         ],
         "dataViewReference": DataView {
-          "allowHidden": false,
           "allowNoIndex": false,
           "deleteFieldFormat": [Function],
           "fieldAttrs": Object {},
@@ -274,7 +273,6 @@ describe('LogViewsClient class', () => {
           },
           "fields": FldList [],
           "flattenHit": [Function],
-          "getAllowHidden": [Function],
           "getFieldAttrs": [Function],
           "getIndexPattern": [Function],
           "getName": [Function],