[8.x] [Data view] Fix allow_hidden usage in the request for fields (#217628) (#218250)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Data view] Fix allow_hidden usage in the request for fields
(#217628)](https://github.com/elastic/kibana/pull/217628)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Matthias
Wilhelm","email":"matthias.wilhelm@elastic.co"},"sourceCommit":{"committedDate":"2025-04-15T10:11:00Z","message":"[Data
view] Fix allow_hidden usage in the request for fields (#217628)\n\nThis
PR fixes the missing allowHidden/allow_hidden usage for data views using
the \"allow hidden and system indices\" functionality when creating a
data view. There are 2 endpoints being requested in Discover: `/fields`
& `/fields_for_wildcards`\n\nBoth use `allow_hidden=true`, but `/fields`
ignored this parameter internally. This was
fixed.","sha":"c25d62739e08d114fcdec495924196a6b9a74fc4","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","Feature:Data
Views","Team:DataDiscovery","backport:prev-major","v9.1.0"],"title":"[Data
view] Fix allow_hidden usage in the request for
fields","number":217628,"url":"https://github.com/elastic/kibana/pull/217628","mergeCommit":{"message":"[Data
view] Fix allow_hidden usage in the request for fields (#217628)\n\nThis
PR fixes the missing allowHidden/allow_hidden usage for data views using
the \"allow hidden and system indices\" functionality when creating a
data view. There are 2 endpoints being requested in Discover: `/fields`
& `/fields_for_wildcards`\n\nBoth use `allow_hidden=true`, but `/fields`
ignored this parameter internally. This was
fixed.","sha":"c25d62739e08d114fcdec495924196a6b9a74fc4"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/217628","number":217628,"mergeCommit":{"message":"[Data
view] Fix allow_hidden usage in the request for fields (#217628)\n\nThis
PR fixes the missing allowHidden/allow_hidden usage for data views using
the \"allow hidden and system indices\" functionality when creating a
data view. There are 2 endpoints being requested in Discover: `/fields`
& `/fields_for_wildcards`\n\nBoth use `allow_hidden=true`, but `/fields`
ignored this parameter internally. This was
fixed.","sha":"c25d62739e08d114fcdec495924196a6b9a74fc4"}}]}]
BACKPORT-->

src/platform/plugins/shared/data_views/server/index_patterns_api_client.test.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+import { IndexPatternsApiServer } from './index_patterns_api_client';
+import { IndexPatternsFetcher } from './fetcher';
+import { coreMock } from '@kbn/core/server/mocks';
+
+jest.mock('./fetcher');
+
+describe('IndexPatternsApiServer', () => {
+  const coreRequestHandler = coreMock.createRequestHandlerContext();
+  let getFieldsForWildcard: jest.Mock;
+  let indexPatternsApiServer: IndexPatternsApiServer;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    getFieldsForWildcard = jest.fn().mockResolvedValue({
+      fields: [{ name: 'field1', type: 'string' }],
+      indices: ['index1'],
+    });
+
+    (IndexPatternsFetcher as jest.Mock).mockImplementation(() => ({
+      getFieldsForWildcard,
+    }));
+
+    indexPatternsApiServer = new IndexPatternsApiServer(
+      coreRequestHandler.elasticsearch.client.asInternalUser,
+      coreRequestHandler.savedObjects.client,
+      coreRequestHandler.uiSettings.client,
+      false
+    );
+  });
+
+  it('uses the allowHidden parameter in indexPatternsApiServer.getFieldsForWildcard', async () => {
+    const options = {
+      allowHidden: true,
+      allowNoIndex: true,
+      pattern: '*',
+    };
+
+    await indexPatternsApiServer.getFieldsForWildcard(options);
+
+    expect(getFieldsForWildcard).toHaveBeenCalledWith(
+      expect.objectContaining({
+        allowHidden: true,
+      })
+    );
+  });
+});

src/platform/plugins/shared/data_views/server/index_patterns_api_client.ts

@@ -35,6 +35,7 @@ export class IndexPatternsApiServer implements IDataViewsApiClient {
     includeEmptyFields,
     abortSignal,
     runtimeMappings,
+    allowHidden,
   }: GetFieldsOptions) {
     const indexPatterns = new IndexPatternsFetcher(this.esClient, {
       uiSettingsClient: this.uiSettingsClient,
@@ -52,6 +53,7 @@ export class IndexPatternsApiServer implements IDataViewsApiClient {
         includeEmptyFields,
         abortSignal,
         runtimeMappings,
+        allowHidden,
       })
       .catch((err) => {
         if (

src/platform/plugins/shared/data_views/server/rest_api_routes/internal/fields.test.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+import { createHandler } from './fields';
+import { IndexPatternsFetcher } from '../../fetcher';
+import { coreMock, httpServerMock } from '@kbn/core/server/mocks';
+import { RequestHandlerContext } from '@kbn/core/server';
+
+jest.mock('../../fetcher');
+
+describe('createHandler', () => {
+  const mockIsRollupsEnabled = jest.fn();
+  const mockContext = {
+    core: coreMock.createRequestHandlerContext(),
+  } as unknown as RequestHandlerContext;
+  const mockResponse = httpServerMock.createResponseFactory();
+  let getFieldsForWildcard: jest.Mock;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    getFieldsForWildcard = jest.fn().mockResolvedValue({
+      fields: [{ name: 'field1', type: 'string' }],
+      indices: ['index1'],
+    });
+
+    (IndexPatternsFetcher as jest.Mock).mockImplementation(() => ({
+      getFieldsForWildcard,
+    }));
+  });
+
+  it('uses the allow_hidden query parameter in indexPatterns.getFieldsForWildcard', async () => {
+    const query = {
+      allow_hidden: true,
+      field_types: [],
+      meta_fields: ['_id', '_type'],
+      type: 'test-type',
+    };
+    const mockRequest = httpServerMock.createKibanaRequest({
+      query,
+    });
+
+    const handlerFn = createHandler(mockIsRollupsEnabled);
+    await handlerFn(mockContext, mockRequest, mockResponse);
+    expect(getFieldsForWildcard).toHaveBeenCalledWith(
+      expect.objectContaining({
+        allowHidden: true,
+      })
+    );
+  });
+});

src/platform/plugins/shared/data_views/server/rest_api_routes/internal/fields.ts

@@ -8,13 +8,9 @@
  */
 
 import { createHash } from 'crypto';
-import { IRouter, RequestHandler, StartServicesAccessor } from '@kbn/core/server';
+import { IRouter, RequestHandler } from '@kbn/core/server';
 import { unwrapEtag } from '../../../common/utils';
 import { IndexPatternsFetcher } from '../../fetcher';
-import type {
-  DataViewsServerPluginStart,
-  DataViewsServerPluginStartDependencies,
-} from '../../types';
 import type { FieldDescriptorRestResponse } from '../route_types';
 import { FIELDS_PATH as path } from '../../../common/constants';
 import { parseFields, IBody, IQuery, querySchema, validate } from './fields_for';
@@ -26,7 +22,9 @@ export function calculateHash(srcBuffer: Buffer) {
   return hash.digest('hex');
 }
 
-const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> =
+export const createHandler: (
+  isRollupsEnabled: () => boolean
+) => RequestHandler<{}, IQuery, IBody> =
   (isRollupsEnabled) => async (context, request, response) => {
     const core = await context.core;
     const uiSettings = core.uiSettings.client;
@@ -44,6 +42,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
       allow_no_index: allowNoIndex,
       include_unmapped: includeUnmapped,
       field_types: fieldTypes,
+      allow_hidden: allowHidden,
     } = request.query;
 
     let parsedFields: string[] = [];
@@ -67,6 +66,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           allow_no_indices: allowNoIndex || false,
           includeUnmapped,
         },
+        allowHidden,
         fieldTypes: parsedFieldTypes,
         ...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
       });
@@ -138,14 +138,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
     }
   };
 
-export const registerFields = (
-  router: IRouter,
-  getStartServices: StartServicesAccessor<
-    DataViewsServerPluginStartDependencies,
-    DataViewsServerPluginStart
-  >,
-  isRollupsEnabled: () => boolean
-) => {
+export const registerFields = (router: IRouter, isRollupsEnabled: () => boolean) => {
   router.versioned.get({ path, access: 'internal', enableQueryVersion: true }).addVersion(
     {
       version: '1',
@@ -157,6 +150,6 @@ export const registerFields = (
       },
       validate: { request: { query: querySchema }, response: validate.response },
     },
-    handler(isRollupsEnabled)
+    createHandler(isRollupsEnabled)
   );
 };

src/platform/plugins/shared/data_views/server/routes.ts

@@ -44,7 +44,7 @@ export function registerRoutes({
 
   registerExistingIndicesPath(router);
   registerFieldForWildcard(router, getStartServices, isRollupsEnabled);
-  registerFields(router, getStartServices, isRollupsEnabled);
+  registerFields(router, isRollupsEnabled);
   registerHasDataViewsRoute(router);
   registerHasEsDataRoute(router, logger, hasEsDataTimeout);
 }