[APM] Migrate settings API tests to be deployment-agnostic (#200762)

Closes https://github.com/elastic/kibana/issues/198989
Part of https://github.com/elastic/kibana/issues/193245

This PR contains the changes to migrate `settings` test folder to
deployment-agnostic testing strategy.


**Not Migrated**
- `agent_configuration`: Not available in Serverless.
- `anomaly_detection/no_access`: Involves the `noAccess` user role; we
are only migrating tests for `viewer`, `editor`, or `admin` roles.
- `anomaly_detection/update_to_v3`: Involves the deletion of ML jobs; we
will wait until an "ml" service is available to properly migrate these
tests.
- `anomaly_detection/write_user`: Involves the deletion of ML jobs; we
will wait until an "ml" service is available to properly migrate these
tests.

**Partially Migrated**
- `anomaly_detection/read_user`: Involves the
`apmAllPrivilegesWithoutWriteSettingsUser` role; only tests for the
`read` role have been migrated.
- `anomaly_detection/write_user`: Involves the
`apmReadPrivilegesWithWriteSettingsUser` role; only tests for the
`write` role have been migrated.
- `apm_indices`: Tests based on license have not been migrated.
custom_link: Involves the `apmReadPrivilegesWithWriteSettingsUser` role;
only tests for the trial `write` role have been migrated.
- `agent_keys`: Involves the `manageOwnAgentKeysUser` and
`createAndAllAgentKeysUser` roles; only tests for the `write` role have
been migrated.

### How to test


- Serverless

```
node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.apm.serverless.config.ts
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.apm.serverless.config.ts 
```

It's recommended to be run against
[MKI](https://github.com/crespocarlos/kibana/blob/main/x-pack/test_serverless/README.md#run-tests-on-mki)

- Stateful
```
node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.apm.stateful.config.ts 
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.apm.stateful.config.ts  
```

## Checks

- [ ] (OPTIONAL, only if a test has been unskipped) Run flaky test suite
- [x] local run for serverless
- [x] local run for stateful
- [x] MKI run for serverless 

<!--ONMERGE {"backportTargets":["8.x"]} ONMERGE-->

---------

Co-authored-by: Carlos Crespo <crespocarlos@users.noreply.github.com>
Co-authored-by: Carlos Crespo <carloshenrique.leonelcrespo@elastic.co>

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts

@@ -12,33 +12,34 @@ export default function apmApiIntegrationTests({
 }: DeploymentAgnosticFtrProviderContext) {
   describe('APM', function () {
     loadTestFile(require.resolve('./agent_explorer'));
-    loadTestFile(require.resolve('./errors'));
     loadTestFile(require.resolve('./alerts'));
-    loadTestFile(require.resolve('./mobile'));
+    loadTestFile(require.resolve('./cold_start'));
+    loadTestFile(require.resolve('./correlations'));
     loadTestFile(require.resolve('./custom_dashboards'));
+    loadTestFile(require.resolve('./data_view'));
     loadTestFile(require.resolve('./dependencies'));
+    loadTestFile(require.resolve('./diagnostics'));
+    loadTestFile(require.resolve('./entities'));
     loadTestFile(require.resolve('./environment'));
     loadTestFile(require.resolve('./error_rate'));
-    loadTestFile(require.resolve('./data_view'));
-    loadTestFile(require.resolve('./correlations'));
-    loadTestFile(require.resolve('./entities'));
-    loadTestFile(require.resolve('./cold_start'));
-    loadTestFile(require.resolve('./metrics'));
-    loadTestFile(require.resolve('./services'));
+    loadTestFile(require.resolve('./errors'));
     loadTestFile(require.resolve('./historical_data'));
-    loadTestFile(require.resolve('./observability_overview'));
-    loadTestFile(require.resolve('./latency'));
     loadTestFile(require.resolve('./infrastructure'));
-    loadTestFile(require.resolve('./service_maps'));
     loadTestFile(require.resolve('./inspect'));
+    loadTestFile(require.resolve('./latency'));
+    loadTestFile(require.resolve('./metrics'));
+    loadTestFile(require.resolve('./mobile'));
+    loadTestFile(require.resolve('./observability_overview'));
     loadTestFile(require.resolve('./service_groups'));
-    loadTestFile(require.resolve('./time_range_metadata'));
-    loadTestFile(require.resolve('./diagnostics'));
+    loadTestFile(require.resolve('./service_maps'));
     loadTestFile(require.resolve('./service_nodes'));
+    loadTestFile(require.resolve('./service_overview'));
+    loadTestFile(require.resolve('./services'));
+    loadTestFile(require.resolve('./settings'));
     loadTestFile(require.resolve('./span_links'));
     loadTestFile(require.resolve('./suggestions'));
     loadTestFile(require.resolve('./throughput'));
+    loadTestFile(require.resolve('./time_range_metadata'));
     loadTestFile(require.resolve('./transactions'));
-    loadTestFile(require.resolve('./service_overview'));
   });
 }

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/services/derived_annotations.spec.ts

@@ -118,7 +118,7 @@ export default function annotationApiTests({ getService }: DeploymentAgnosticFtr
         });
 
         response = (
-          await apmApiClient.readUser({
+          await apmApiClient.publicApi({
             endpoint: 'GET /api/apm/services/{serviceName}/annotation/search 2023-10-31',
             params: {
               path: {

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/settings/agent_keys/agent_keys.spec.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import expect from '@kbn/expect';
+import { PrivilegeType, ClusterPrivilegeType } from '@kbn/apm-plugin/common/privilege_type';
+import type { RoleCredentials } from '../../../../../services';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+import { expectToReject } from '../../../../../../../apm_api_integration/common/utils/expect_to_reject';
+import type { ApmApiError } from '../../../../../services/apm_api';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const apmApiClient = getService('apmApi');
+  const samlAuth = getService('samlAuth');
+
+  const agentKeyName = 'test';
+  const allApplicationPrivileges = [PrivilegeType.AGENT_CONFIG, PrivilegeType.EVENT];
+  const clusterPrivileges = [ClusterPrivilegeType.MANAGE_OWN_API_KEY];
+
+  async function createAgentKey(roleAuthc: RoleCredentials) {
+    return await apmApiClient.publicApi({
+      endpoint: 'POST /api/apm/agent_keys 2023-10-31',
+      params: {
+        body: {
+          name: agentKeyName,
+          privileges: allApplicationPrivileges,
+        },
+      },
+      roleAuthc,
+    });
+  }
+
+  async function invalidateAgentKey(id: string) {
+    return await apmApiClient.writeUser({
+      endpoint: 'POST /internal/apm/api_key/invalidate',
+      params: {
+        body: { id },
+      },
+    });
+  }
+
+  async function getAgentKeys() {
+    return await apmApiClient.writeUser({ endpoint: 'GET /internal/apm/agent_keys' });
+  }
+
+  describe('When the user does not have the required privileges', () => {
+    let roleAuthc: RoleCredentials;
+
+    before(async () => {
+      roleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('editor');
+    });
+
+    after(async () => {
+      await samlAuth.invalidateM2mApiKeyWithRoleScope(roleAuthc);
+    });
+
+    describe('When the user does not have the required cluster privileges', () => {
+      it('should return an error when creating an agent key', async () => {
+        const error = await expectToReject<ApmApiError>(() => createAgentKey(roleAuthc));
+        expect(error.res.status).to.be(403);
+        expect(error.res.body.message).contain('is missing the following requested privilege');
+        expect(error.res.body.attributes).to.eql({
+          _inspect: [],
+          data: {
+            missingPrivileges: allApplicationPrivileges,
+            missingClusterPrivileges: clusterPrivileges,
+          },
+        });
+      });
+
+      it('should return an error when invalidating an agent key', async () => {
+        const error = await expectToReject<ApmApiError>(() => invalidateAgentKey(agentKeyName));
+        expect(error.res.status).to.be(500);
+      });
+
+      it('should return an error when getting a list of agent keys', async () => {
+        const error = await expectToReject<ApmApiError>(() => getAgentKeys());
+        expect(error.res.status).to.be(500);
+      });
+    });
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/settings/anomaly_detection/basic.spec.ts

@@ -6,17 +6,13 @@
  */
 
 import expect from '@kbn/expect';
-import { FtrProviderContext } from '../../../common/ftr_provider_context';
-import { ApmApiError } from '../../../common/apm_api_supertest';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+import type { ApmApiError } from '../../../../../services/apm_api';
 
-export default function apiTest({ getService }: FtrProviderContext) {
-  const registry = getService('registry');
-  const apmApiClient = getService('apmApiClient');
+export default function apiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const apmApiClient = getService('apmApi');
 
-  type SupertestAsUser =
-    | typeof apmApiClient.readUser
-    | typeof apmApiClient.writeUser
-    | typeof apmApiClient.noAccessUser;
+  type SupertestAsUser = typeof apmApiClient.readUser | typeof apmApiClient.writeUser;
 
   function getJobs(user: SupertestAsUser) {
     return user({ endpoint: `GET /internal/apm/settings/anomaly-detection/jobs` });
@@ -34,7 +30,6 @@ export default function apiTest({ getService }: FtrProviderContext) {
   async function expectForbidden(user: SupertestAsUser) {
     try {
       await getJobs(user);
-      expect(true).to.be(false);
     } catch (e) {
       const err = e as ApmApiError;
       expect(err.res.status).to.be(403);
@@ -42,20 +37,14 @@ export default function apiTest({ getService }: FtrProviderContext) {
 
     try {
       await createJobs(user, ['production', 'staging']);
-      expect(true).to.be(false);
     } catch (e) {
       const err = e as ApmApiError;
       expect(err.res.status).to.be(403);
     }
   }
 
-  registry.when('ML jobs return a 403 for', { config: 'basic', archives: [] }, () => {
+  describe('ML jobs return a 403 for', () => {
     describe('basic', function () {
-      this.tags('skipFIPS');
-      it('user without access', async () => {
-        await expectForbidden(apmApiClient.noAccessUser);
-      });
-
       it('read user', async () => {
         await expectForbidden(apmApiClient.readUser);
       });

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/settings/anomaly_detection/read_user.spec.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+import type { ApmApiError } from '../../../../../services/apm_api';
+
+export default function apiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const apmApiClient = getService('apmApi');
+
+  function getJobs() {
+    return apmApiClient.readUser({ endpoint: `GET /internal/apm/settings/anomaly-detection/jobs` });
+  }
+
+  function createJobs(environments: string[]) {
+    return apmApiClient.readUser({
+      endpoint: `POST /internal/apm/settings/anomaly-detection/jobs`,
+      params: {
+        body: { environments },
+      },
+    });
+  }
+
+  describe('ML jobs', () => {
+    describe(`when readUser has read access to ML`, () => {
+      describe('when calling the endpoint for listing jobs', () => {
+        it('returns a list of jobs', async () => {
+          const { body } = await getJobs();
+
+          expect(body.jobs).not.to.be(undefined);
+          expect(body.hasLegacyJobs).to.be(false);
+        });
+      });
+
+      describe('when calling create endpoint', () => {
+        it('returns an error because the user does not have access', async () => {
+          try {
+            await createJobs(['production', 'staging']);
+            expect(true).to.be(false);
+          } catch (e) {
+            const err = e as ApmApiError;
+            expect(err.res.status).to.be(403);
+          }
+        });
+      });
+    });
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/settings/apm_indices/apm_indices.spec.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  APM_INDEX_SETTINGS_SAVED_OBJECT_ID,
+  APM_INDEX_SETTINGS_SAVED_OBJECT_TYPE,
+} from '@kbn/apm-data-access-plugin/server/saved_objects/apm_indices';
+import expect from '@kbn/expect';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+
+export default function apmIndicesTests({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const kibanaServer = getService('kibanaServer');
+  const apmApiClient = getService('apmApi');
+
+  async function deleteSavedObject() {
+    try {
+      return await kibanaServer.savedObjects.delete({
+        type: APM_INDEX_SETTINGS_SAVED_OBJECT_TYPE,
+        id: APM_INDEX_SETTINGS_SAVED_OBJECT_ID,
+      });
+    } catch (e) {
+      if (e.response.status !== 404) {
+        throw e;
+      }
+    }
+  }
+
+  describe('APM Indices', () => {
+    beforeEach(async () => {
+      await deleteSavedObject();
+    });
+
+    afterEach(async () => {
+      await deleteSavedObject();
+    });
+
+    it('returns APM Indices', async () => {
+      const response = await apmApiClient.readUser({
+        endpoint: 'GET /internal/apm/settings/apm-indices',
+      });
+      expect(response.status).to.be(200);
+      expect(response.body).to.eql({
+        transaction: 'traces-apm*,apm-*,traces-*.otel-*',
+        span: 'traces-apm*,apm-*,traces-*.otel-*',
+        error: 'logs-apm*,apm-*,logs-*.otel-*',
+        metric: 'metrics-apm*,apm-*,metrics-*.otel-*',
+        onboarding: 'apm-*',
+        sourcemap: 'apm-*',
+      });
+    });
+
+    it('updates apm indices', async () => {
+      const INDEX_VALUE = 'foo-*';
+
+      const writeResponse = await apmApiClient.writeUser({
+        endpoint: 'POST /internal/apm/settings/apm-indices/save',
+        params: {
+          body: { transaction: INDEX_VALUE },
+        },
+      });
+      expect(writeResponse.status).to.be(200);
+
+      const readResponse = await apmApiClient.readUser({
+        endpoint: 'GET /internal/apm/settings/apm-indices',
+      });
+
+      expect(readResponse.status).to.be(200);
+      expect(readResponse.body.transaction).to.eql(INDEX_VALUE);
+    });
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/settings/custom_link/custom_link.spec.ts

@@ -0,0 +1,200 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import type { CustomLink } from '@kbn/apm-plugin/common/custom_link/custom_link_types';
+import type { ApmApiError } from '../../../../../services/apm_api';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+import { ARCHIVER_ROUTES } from '../../constants/archiver';
+
+export default function customLinksTests({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const apmApiClient = getService('apmApi');
+  const log = getService('log');
+
+  const archiveName = '8.0.0';
+
+  describe('Custom links with data', () => {
+    before(async () => {
+      await esArchiver.load(ARCHIVER_ROUTES[archiveName]);
+
+      const customLink = {
+        url: 'https://elastic.co',
+        label: 'with filters',
+        filters: [
+          { key: 'service.name', value: 'baz' },
+          { key: 'transaction.type', value: 'qux' },
+        ],
+      } as CustomLink;
+
+      await createCustomLink(customLink);
+    });
+
+    after(async () => {
+      await esArchiver.unload(ARCHIVER_ROUTES[archiveName]);
+    });
+
+    it('should fail if the user does not have write access', async () => {
+      const customLink = {
+        url: 'https://elastic.co',
+        label: 'with filters',
+        filters: [
+          { key: 'service.name', value: 'baz' },
+          { key: 'transaction.type', value: 'qux' },
+        ],
+      } as CustomLink;
+
+      const err = await expectToReject<ApmApiError>(() => createCustomLinkAsReadUser(customLink));
+      expect(err.res.status).to.be(403);
+    });
+
+    it('fetches a custom link', async () => {
+      const { status, body } = await searchCustomLinks({
+        'service.name': 'baz',
+        'transaction.type': 'qux',
+      });
+      const { label, url, filters } = body.customLinks[0];
+
+      expect(status).to.equal(200);
+      expect({ label, url, filters }).to.eql({
+        label: 'with filters',
+        url: 'https://elastic.co',
+        filters: [
+          { key: 'service.name', value: 'baz' },
+          { key: 'transaction.type', value: 'qux' },
+        ],
+      });
+    });
+
+    it(`creates a custom link as write user`, async () => {
+      const customLink = {
+        url: 'https://elastic.co',
+        label: 'with filters',
+        filters: [
+          { key: 'service.name', value: 'baz' },
+          { key: 'transaction.type', value: 'qux' },
+        ],
+      } as CustomLink;
+
+      await createCustomLink(customLink);
+    });
+
+    it(`updates a custom link as write user`, async () => {
+      const { status, body } = await searchCustomLinks({
+        'service.name': 'baz',
+        'transaction.type': 'qux',
+      });
+      expect(status).to.equal(200);
+
+      const id = body.customLinks[0].id!;
+      await updateCustomLink(id, {
+        label: 'foo',
+        url: 'https://elastic.co?service.name={{service.name}}',
+        filters: [
+          { key: 'service.name', value: 'quz' },
+          { key: 'transaction.name', value: 'bar' },
+        ],
+      });
+
+      const { status: newStatus, body: newBody } = await searchCustomLinks({
+        'service.name': 'quz',
+        'transaction.name': 'bar',
+      });
+
+      const { label, url, filters } = newBody.customLinks[0];
+      expect(newStatus).to.equal(200);
+      expect({ label, url, filters }).to.eql({
+        label: 'foo',
+        url: 'https://elastic.co?service.name={{service.name}}',
+        filters: [
+          { key: 'service.name', value: 'quz' },
+          { key: 'transaction.name', value: 'bar' },
+        ],
+      });
+    });
+
+    it(`deletes a custom link as write user`, async () => {
+      const { status, body } = await searchCustomLinks({
+        'service.name': 'quz',
+        'transaction.name': 'bar',
+      });
+      expect(status).to.equal(200);
+      expect(body.customLinks.length).to.be(1);
+
+      const id = body.customLinks[0].id!;
+      await deleteCustomLink(id);
+
+      const { status: newStatus, body: newBody } = await searchCustomLinks({
+        'service.name': 'quz',
+        'transaction.name': 'bar',
+      });
+      expect(newStatus).to.equal(200);
+      expect(newBody.customLinks.length).to.be(0);
+    });
+  });
+
+  function searchCustomLinks(filters?: any) {
+    return apmApiClient.readUser({
+      endpoint: 'GET /internal/apm/settings/custom_links',
+      params: {
+        query: filters,
+      },
+    });
+  }
+
+  async function createCustomLink(customLink: CustomLink) {
+    log.debug('creating configuration', customLink);
+
+    return apmApiClient.writeUser({
+      endpoint: 'POST /internal/apm/settings/custom_links',
+      params: {
+        body: customLink,
+      },
+    });
+  }
+
+  async function createCustomLinkAsReadUser(customLink: CustomLink) {
+    log.debug('creating configuration', customLink);
+
+    return apmApiClient.readUser({
+      endpoint: 'POST /internal/apm/settings/custom_links',
+      params: {
+        body: customLink,
+      },
+    });
+  }
+
+  async function updateCustomLink(id: string, customLink: CustomLink) {
+    log.debug('updating configuration', id, customLink);
+
+    return apmApiClient.writeUser({
+      endpoint: 'PUT /internal/apm/settings/custom_links/{id}',
+      params: {
+        path: { id },
+        body: customLink,
+      },
+    });
+  }
+
+  async function deleteCustomLink(id: string) {
+    log.debug('deleting configuration', id);
+
+    return apmApiClient.writeUser({
+      endpoint: 'DELETE /internal/apm/settings/custom_links/{id}',
+      params: { path: { id } },
+    });
+  }
+}
+
+async function expectToReject<T extends Error>(fn: () => Promise<any>): Promise<T> {
+  try {
+    await fn();
+  } catch (e) {
+    return e;
+  }
+  throw new Error(`Expected fn to throw`);
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/settings/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
+  describe('settings', () => {
+    loadTestFile(require.resolve('./agent_keys/agent_keys.spec.ts'));
+    loadTestFile(require.resolve('./anomaly_detection/basic.spec.ts'));
+    loadTestFile(require.resolve('./anomaly_detection/read_user.spec.ts'));
+    loadTestFile(require.resolve('./apm_indices/apm_indices.spec.ts'));
+    loadTestFile(require.resolve('./custom_link/custom_link.spec.ts'));
+  });
+}

x-pack/test/api_integration/deployment_agnostic/services/apm_api.ts

@@ -16,16 +16,7 @@ import { formatRequest } from '@kbn/server-route-repository';
 import { RoleCredentials } from '@kbn/ftr-common-functional-services';
 import type { DeploymentAgnosticFtrProviderContext } from '../ftr_provider_context';
 
-const INTERNAL_API_REGEX = /^\S+\s(\/)?internal\/[^\s]*$/;
-
-type InternalApi = `${string} /internal/${string}`;
-interface ExternalEndpointParams {
-  roleAuthc: RoleCredentials;
-}
-
-type Options<TEndpoint extends APIEndpoint> = (TEndpoint extends InternalApi
-  ? {}
-  : ExternalEndpointParams) & {
+type Options<TEndpoint extends APIEndpoint> = {
   type?: 'form-data';
   endpoint: TEndpoint;
   spaceId?: string;
@@ -33,33 +24,28 @@ type Options<TEndpoint extends APIEndpoint> = (TEndpoint extends InternalApi
     params?: { query?: { _inspect?: boolean } };
   };
 
-function isPublicApi<TEndpoint extends APIEndpoint>(
-  options: Options<TEndpoint>
-): options is Options<TEndpoint> & ExternalEndpointParams {
-  return !INTERNAL_API_REGEX.test(options.endpoint);
-}
+type InternalEndpoint<T extends APIEndpoint> = T extends `${string} /internal/${string}`
+  ? T
+  : never;
 
-function createApmApiClient({ getService }: DeploymentAgnosticFtrProviderContext, role: string) {
+type PublicEndpoint<T extends APIEndpoint> = T extends `${string} /api/${string}` ? T : never;
+
+function createApmApiClient({ getService }: DeploymentAgnosticFtrProviderContext) {
   const supertestWithoutAuth = getService('supertestWithoutAuth');
   const samlAuth = getService('samlAuth');
   const logger = getService('log');
 
-  return async <TEndpoint extends APIEndpoint>(
-    options: Options<TEndpoint>
-  ): Promise<SupertestReturnType<TEndpoint>> => {
+  async function makeApiRequest<TEndpoint extends APIEndpoint>({
+    options,
+    headers,
+  }: {
+    options: Options<TEndpoint>;
+    headers: Record<string, string>;
+  }): Promise<SupertestReturnType<TEndpoint>> {
     const { endpoint, type } = options;
 
     const params = 'params' in options ? (options.params as Record<string, any>) : {};
 
-    const credentials = isPublicApi(options)
-      ? options.roleAuthc.apiKeyHeader
-      : await samlAuth.getM2MApiCookieCredentialsWithRoleScope(role);
-
-    const headers: Record<string, string> = {
-      ...samlAuth.getInternalRequestHeader(),
-      ...credentials,
-    };
-
     const { method, pathname, version } = formatRequest(endpoint, params.path);
     const pathnameWithSpaceId = options.spaceId ? `/s/${options.spaceId}${pathname}` : pathname;
     const url = format({ pathname: pathnameWithSpaceId, query: params?.query });
@@ -71,6 +57,7 @@ function createApmApiClient({ getService }: DeploymentAgnosticFtrProviderContext
     }
 
     let res: request.Response;
+
     if (type === 'form-data') {
       const fields: Array<[string, any]> = Object.entries(params.body);
       const formDataRequest = supertestWithoutAuth[method](url)
@@ -94,6 +81,45 @@ function createApmApiClient({ getService }: DeploymentAgnosticFtrProviderContext
     }
 
     return res;
+  }
+
+  function makeInternalApiRequest(role: string) {
+    return async <TEndpoint extends InternalEndpoint<APIEndpoint>>(
+      options: Options<TEndpoint>
+    ): Promise<SupertestReturnType<TEndpoint>> => {
+      const headers: Record<string, string> = {
+        ...samlAuth.getInternalRequestHeader(),
+        ...(await samlAuth.getM2MApiCookieCredentialsWithRoleScope(role)),
+      };
+
+      return makeApiRequest({
+        options,
+        headers,
+      });
+    };
+  }
+
+  function makePublicApiRequest() {
+    return async <TEndpoint extends PublicEndpoint<APIEndpoint>>(
+      options: Options<TEndpoint> & {
+        roleAuthc: RoleCredentials;
+      }
+    ): Promise<SupertestReturnType<TEndpoint>> => {
+      const headers: Record<string, string> = {
+        ...samlAuth.getInternalRequestHeader(),
+        ...options.roleAuthc.apiKeyHeader,
+      };
+
+      return makeApiRequest({
+        options,
+        headers,
+      });
+    };
+  }
+
+  return {
+    makeInternalApiRequest,
+    makePublicApiRequest,
   };
 }
 
@@ -129,10 +155,12 @@ export interface SupertestReturnType<TEndpoint extends APIEndpoint> {
 }
 
 export function ApmApiProvider(context: DeploymentAgnosticFtrProviderContext) {
+  const apmClient = createApmApiClient(context);
   return {
-    readUser: createApmApiClient(context, 'viewer'),
-    adminUser: createApmApiClient(context, 'admin'),
-    writeUser: createApmApiClient(context, 'editor'),
+    readUser: apmClient.makeInternalApiRequest('viewer'),
+    adminUser: apmClient.makeInternalApiRequest('admin'),
+    writeUser: apmClient.makeInternalApiRequest('editor'),
+    publicApi: apmClient.makePublicApiRequest(),
   };
 }
 

x-pack/test/apm_api_integration/tests/settings/agent_keys/agent_keys.spec.ts

@@ -6,7 +6,7 @@
  */
 import expect from '@kbn/expect';
 import { first } from 'lodash';
-import { PrivilegeType, ClusterPrivilegeType } from '@kbn/apm-plugin/common/privilege_type';
+import { PrivilegeType } from '@kbn/apm-plugin/common/privilege_type';
 import { ApmUsername } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/authentication';
 import { FtrProviderContext } from '../../../common/ftr_provider_context';
 import { ApmApiError, ApmApiSupertest } from '../../../common/apm_api_supertest';
@@ -19,7 +19,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
 
   const agentKeyName = 'test';
   const allApplicationPrivileges = [PrivilegeType.AGENT_CONFIG, PrivilegeType.EVENT];
-  const clusterPrivileges = [ClusterPrivilegeType.MANAGE_OWN_API_KEY];
 
   async function createAgentKey(apiClient: ApmApiSupertest, privileges = allApplicationPrivileges) {
     return await apiClient({
@@ -50,37 +49,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     'When the user does not have the required privileges',
     { config: 'basic', archives: [] },
     () => {
-      describe('When the user does not have the required cluster privileges', () => {
-        it('should return an error when creating an agent key', async () => {
-          const error = await expectToReject<ApmApiError>(() =>
-            createAgentKey(apmApiClient.writeUser)
-          );
-          expect(error.res.status).to.be(403);
-          expect(error.res.body.message).contain('is missing the following requested privilege');
-          expect(error.res.body.attributes).to.eql({
-            _inspect: [],
-            data: {
-              missingPrivileges: allApplicationPrivileges,
-              missingClusterPrivileges: clusterPrivileges,
-            },
-          });
-        });
-
-        it('should return an error when invalidating an agent key', async () => {
-          const error = await expectToReject<ApmApiError>(() =>
-            invalidateAgentKey(apmApiClient.writeUser, agentKeyName)
-          );
-          expect(error.res.status).to.be(500);
-        });
-
-        it('should return an error when getting a list of agent keys', async () => {
-          const error = await expectToReject<ApmApiError>(() =>
-            getAgentKeys(apmApiClient.writeUser)
-          );
-          expect(error.res.status).to.be(500);
-        });
-      });
-
       describe('When the user does not have the required application privileges', () => {
         allApplicationPrivileges.map((privilege) => {
           it(`should return an error when creating an agent key with ${privilege} privilege`, async () => {

x-pack/test/apm_api_integration/tests/settings/anomaly_detection/read_user.spec.ts

@@ -35,37 +35,35 @@ export default function apiTest({ getService }: FtrProviderContext) {
   }
 
   registry.when('ML jobs', { config: 'trial', archives: [] }, () => {
-    (['readUser', 'apmAllPrivilegesWithoutWriteSettingsUser'] as ApmApiClientKey[]).forEach(
-      (user) => {
-        describe(`when ${user} has read access to ML`, () => {
-          before(async () => {
-            const res = await getJobs({ user });
-            const jobIds = res.body.jobs.map((job: any) => job.jobId);
-            await deleteJobs(jobIds);
-          });
+    (['apmAllPrivilegesWithoutWriteSettingsUser'] as ApmApiClientKey[]).forEach((user) => {
+      describe(`when ${user} has read access to ML`, () => {
+        before(async () => {
+          const res = await getJobs({ user });
+          const jobIds = res.body.jobs.map((job: any) => job.jobId);
+          await deleteJobs(jobIds);
+        });
 
-          describe('when calling the endpoint for listing jobs', () => {
-            it('returns a list of jobs', async () => {
-              const { body } = await getJobs({ user });
+        describe('when calling the endpoint for listing jobs', () => {
+          it('returns a list of jobs', async () => {
+            const { body } = await getJobs({ user });
 
-              expect(body.jobs.length).to.be(0);
-              expect(body.hasLegacyJobs).to.be(false);
-            });
-          });
-
-          describe('when calling create endpoint', () => {
-            it('returns an error because the user does not have access', async () => {
-              try {
-                await createJobs(['production', 'staging'], { user });
-                expect(true).to.be(false);
-              } catch (e) {
-                const err = e as ApmApiError;
-                expect(err.res.status).to.be(403);
-              }
-            });
+            expect(body.jobs.length).to.be(0);
+            expect(body.hasLegacyJobs).to.be(false);
           });
         });
-      }
-    );
+
+        describe('when calling create endpoint', () => {
+          it('returns an error because the user does not have access', async () => {
+            try {
+              await createJobs(['production', 'staging'], { user });
+              expect(true).to.be(false);
+            } catch (e) {
+              const err = e as ApmApiError;
+              expect(err.res.status).to.be(403);
+            }
+          });
+        });
+      });
+    });
   });
 }

x-pack/test/apm_api_integration/tests/settings/anomaly_detection/write_user.spec.ts

@@ -35,59 +35,57 @@ export default function apiTest({ getService }: FtrProviderContext) {
   }
 
   registry.when('ML jobs', { config: 'trial', archives: [] }, () => {
-    (['writeUser', 'apmReadPrivilegesWithWriteSettingsUser'] as ApmApiClientKey[]).forEach(
-      (user) => {
-        describe(`when ${user} has write access to ML`, () => {
-          before(async () => {
-            const res = await getJobs({ user });
-            const jobIds = res.body.jobs.map((job: any) => job.jobId);
-            await deleteJobs(jobIds);
-          });
+    (['apmReadPrivilegesWithWriteSettingsUser'] as ApmApiClientKey[]).forEach((user) => {
+      describe(`when ${user} has write access to ML`, () => {
+        before(async () => {
+          const res = await getJobs({ user });
+          const jobIds = res.body.jobs.map((job: any) => job.jobId);
+          await deleteJobs(jobIds);
+        });
 
-          after(async () => {
-            const res = await getJobs({ user });
-            const jobIds = res.body.jobs.map((job: any) => job.jobId);
-            await deleteJobs(jobIds);
-          });
+        after(async () => {
+          const res = await getJobs({ user });
+          const jobIds = res.body.jobs.map((job: any) => job.jobId);
+          await deleteJobs(jobIds);
+        });
 
-          describe('when calling the endpoint for listing jobs', () => {
-            it('returns a list of jobs', async () => {
-              const { body } = await getJobs({ user });
-              expect(body.jobs.length).to.be(0);
-              expect(body.hasLegacyJobs).to.be(false);
+        describe('when calling the endpoint for listing jobs', () => {
+          it('returns a list of jobs', async () => {
+            const { body } = await getJobs({ user });
+            expect(body.jobs.length).to.be(0);
+            expect(body.hasLegacyJobs).to.be(false);
+          });
+        });
+
+        describe('when calling create endpoint', () => {
+          it('creates two jobs', async () => {
+            await createJobs(['production', 'staging'], { user });
+
+            const { body } = await getJobs({ user });
+            expect(body.hasLegacyJobs).to.be(false);
+            expect(countBy(body.jobs, 'environment')).to.eql({
+              production: 1,
+              staging: 1,
             });
           });
 
-          describe('when calling create endpoint', () => {
-            it('creates two jobs', async () => {
+          describe('with existing ML jobs', () => {
+            before(async () => {
               await createJobs(['production', 'staging'], { user });
+            });
+            it('skips duplicate job creation', async () => {
+              await createJobs(['production', 'test'], { user });
 
               const { body } = await getJobs({ user });
-              expect(body.hasLegacyJobs).to.be(false);
               expect(countBy(body.jobs, 'environment')).to.eql({
                 production: 1,
                 staging: 1,
-              });
-            });
-
-            describe('with existing ML jobs', () => {
-              before(async () => {
-                await createJobs(['production', 'staging'], { user });
-              });
-              it('skips duplicate job creation', async () => {
-                await createJobs(['production', 'test'], { user });
-
-                const { body } = await getJobs({ user });
-                expect(countBy(body.jobs, 'environment')).to.eql({
-                  production: 1,
-                  staging: 1,
-                  test: 1,
-                });
+                test: 1,
               });
             });
           });
         });
-      }
-    );
+      });
+    });
   });
 }

x-pack/test/apm_api_integration/tests/settings/apm_indices/apm_indices.spec.ts

@@ -107,40 +107,6 @@ export default function apmIndicesTests({ getService }: FtrProviderContext) {
       await deleteSavedObject();
     });
 
-    it('[trial] returns APM Indices', async () => {
-      const response = await apmApiClient.readUser({
-        endpoint: 'GET /internal/apm/settings/apm-indices',
-      });
-      expect(response.status).to.be(200);
-      expect(response.body).to.eql({
-        transaction: 'traces-apm*,apm-*,traces-*.otel-*',
-        span: 'traces-apm*,apm-*,traces-*.otel-*',
-        error: 'logs-apm*,apm-*,logs-*.otel-*',
-        metric: 'metrics-apm*,apm-*,metrics-*.otel-*',
-        onboarding: 'apm-*',
-        sourcemap: 'apm-*',
-      });
-    });
-
-    it('[trial] updates apm indices', async () => {
-      const INDEX_VALUE = 'foo-*';
-
-      const writeResponse = await apmApiClient.writeUser({
-        endpoint: 'POST /internal/apm/settings/apm-indices/save',
-        params: {
-          body: { transaction: INDEX_VALUE },
-        },
-      });
-      expect(writeResponse.status).to.be(200);
-
-      const readResponse = await apmApiClient.readUser({
-        endpoint: 'GET /internal/apm/settings/apm-indices',
-      });
-
-      expect(readResponse.status).to.be(200);
-      expect(readResponse.body.transaction).to.eql(INDEX_VALUE);
-    });
-
     it('[trial] updates apm indices as read privileges with modify settings user', async () => {
       const INDEX_VALUE = 'foo-*';
 

x-pack/test/apm_api_integration/tests/settings/custom_link/custom_link.spec.ts

@@ -91,79 +91,77 @@ export default function customLinksTests({ getService }: FtrProviderContext) {
         });
       });
 
-      (['writeUser', 'apmReadPrivilegesWithWriteSettingsUser'] as ApmApiClientKey[]).forEach(
-        (user) => {
-          it(`creates a custom link as ${user}`, async () => {
-            const customLink = {
-              url: 'https://elastic.co',
-              label: 'with filters',
-              filters: [
-                { key: 'service.name', value: 'baz' },
-                { key: 'transaction.type', value: 'qux' },
-              ],
-            } as CustomLink;
+      (['apmReadPrivilegesWithWriteSettingsUser'] as ApmApiClientKey[]).forEach((user) => {
+        it(`creates a custom link as ${user}`, async () => {
+          const customLink = {
+            url: 'https://elastic.co',
+            label: 'with filters',
+            filters: [
+              { key: 'service.name', value: 'baz' },
+              { key: 'transaction.type', value: 'qux' },
+            ],
+          } as CustomLink;
 
-            await createCustomLink(customLink, { user });
+          await createCustomLink(customLink, { user });
+        });
+
+        it(`updates a custom link as ${user}`, async () => {
+          const { status, body } = await searchCustomLinks({
+            'service.name': 'baz',
+            'transaction.type': 'qux',
           });
+          expect(status).to.equal(200);
 
-          it(`updates a custom link as ${user}`, async () => {
-            const { status, body } = await searchCustomLinks({
-              'service.name': 'baz',
-              'transaction.type': 'qux',
-            });
-            expect(status).to.equal(200);
-
-            const id = body.customLinks[0].id!;
-            await updateCustomLink(
-              id,
-              {
-                label: 'foo',
-                url: 'https://elastic.co?service.name={{service.name}}',
-                filters: [
-                  { key: 'service.name', value: 'quz' },
-                  { key: 'transaction.name', value: 'bar' },
-                ],
-              },
-              { user }
-            );
-
-            const { status: newStatus, body: newBody } = await searchCustomLinks({
-              'service.name': 'quz',
-              'transaction.name': 'bar',
-            });
-
-            const { label, url, filters } = newBody.customLinks[0];
-            expect(newStatus).to.equal(200);
-            expect({ label, url, filters }).to.eql({
+          const id = body.customLinks[0].id!;
+          await updateCustomLink(
+            id,
+            {
               label: 'foo',
               url: 'https://elastic.co?service.name={{service.name}}',
               filters: [
                 { key: 'service.name', value: 'quz' },
                 { key: 'transaction.name', value: 'bar' },
               ],
-            });
+            },
+            { user }
+          );
+
+          const { status: newStatus, body: newBody } = await searchCustomLinks({
+            'service.name': 'quz',
+            'transaction.name': 'bar',
           });
 
-          it(`deletes a custom link as ${user}`, async () => {
-            const { status, body } = await searchCustomLinks({
-              'service.name': 'quz',
-              'transaction.name': 'bar',
-            });
-            expect(status).to.equal(200);
-            expect(body.customLinks.length).to.be(1);
-
-            const id = body.customLinks[0].id!;
-            await deleteCustomLink(id, { user });
-
-            const { status: newStatus, body: newBody } = await searchCustomLinks({
-              'service.name': 'quz',
-              'transaction.name': 'bar',
-            });
-            expect(newStatus).to.equal(200);
-            expect(newBody.customLinks.length).to.be(0);
+          const { label, url, filters } = newBody.customLinks[0];
+          expect(newStatus).to.equal(200);
+          expect({ label, url, filters }).to.eql({
+            label: 'foo',
+            url: 'https://elastic.co?service.name={{service.name}}',
+            filters: [
+              { key: 'service.name', value: 'quz' },
+              { key: 'transaction.name', value: 'bar' },
+            ],
           });
-        }
-      );
+        });
+
+        it(`deletes a custom link as ${user}`, async () => {
+          const { status, body } = await searchCustomLinks({
+            'service.name': 'quz',
+            'transaction.name': 'bar',
+          });
+          expect(status).to.equal(200);
+          expect(body.customLinks.length).to.be(1);
+
+          const id = body.customLinks[0].id!;
+          await deleteCustomLink(id, { user });
+
+          const { status: newStatus, body: newBody } = await searchCustomLinks({
+            'service.name': 'quz',
+            'transaction.name': 'bar',
+          });
+          expect(newStatus).to.equal(200);
+          expect(newBody.customLinks.length).to.be(0);
+        });
+      });
 
       it('fetches a transaction sample', async () => {
         const response = await apmApiClient.readUser({