[APM] Add api key to APM package policies (#147650)

Closes https://github.com/elastic/kibana/issues/146368
Depends on: https://github.com/elastic/kibana/pull/148509

This adds an api key to every APM package policy. This will make it
possible for APM Server to query the source map index (
`.apm-source-map` ) and the agent config index
(`.apm-agent-configuration`)

x-pack/plugins/apm/jest_setup.js

@@ -9,6 +9,7 @@
 
 // When a `console.error` is encountered, throw the error to make the test fail.
 // This effectively treats logged errors during the test run as failures.
-jest.spyOn(console, 'error').mockImplementation((message) => {
+jest.spyOn(console, 'error').mockImplementation((message, ...args) => {
+  console.log(message, ...args);
   throw new Error(message);
 });

x-pack/plugins/apm/server/lib/apm_telemetry/index.ts

@@ -56,7 +56,8 @@ export async function createApmTelemetry({
     },
   });
 
-  const savedObjectsClient = await getInternalSavedObjectsClient(core);
+  const [coreStart] = await core.getStartServices();
+  const savedObjectsClient = await getInternalSavedObjectsClient(coreStart);
   const indices = await getApmIndices({ config, savedObjectsClient });
   const telemetryClient = await getTelemetryClient({ core });
 

x-pack/plugins/apm/server/lib/helpers/get_internal_saved_objects_client.ts

@@ -5,16 +5,12 @@
  * 2.0.
  */
 
-import { CoreSetup } from '@kbn/core/server';
-import { withApmSpan } from '../../utils/with_apm_span';
+import { CoreStart } from '@kbn/core/server';
 
 export type InternalSavedObjectsClient = Awaited<
   ReturnType<typeof getInternalSavedObjectsClient>
 >;
-export async function getInternalSavedObjectsClient(core: CoreSetup) {
-  return withApmSpan('get_internal_saved_objects_client', () =>
-    core.getStartServices().then(async ([coreStart]) => {
-      return coreStart.savedObjects.createInternalRepository();
-    })
-  );
+
+export async function getInternalSavedObjectsClient(coreStart: CoreStart) {
+  return coreStart.savedObjects.createInternalRepository();
 }

x-pack/plugins/apm/server/plugin.ts

@@ -58,6 +58,7 @@ import { tutorialProvider } from './tutorial';
 import { migrateLegacyAPMIndicesToSpaceAware } from './saved_objects/migrations/migrate_legacy_apm_indices_to_space_aware';
 import { scheduleSourceMapMigration } from './routes/source_maps/schedule_source_map_migration';
 import { createApmSourceMapIndexTemplate } from './routes/source_maps/create_apm_source_map_index_template';
+import { addApiKeysToEveryPackagePolicyIfMissing } from './routes/fleet/api_keys/add_api_keys_to_policies_if_missing';
 
 export class APMPlugin
   implements
@@ -169,11 +170,13 @@ export class APMPlugin
       };
     }) as APMRouteHandlerResources['plugins'];
 
-    const boundGetApmIndices = async () =>
-      getApmIndices({
-        savedObjectsClient: await getInternalSavedObjectsClient(core),
+    const boundGetApmIndices = async () => {
+      const coreStart = await getCoreStart();
+      return getApmIndices({
+        savedObjectsClient: await getInternalSavedObjectsClient(coreStart),
         config: await firstValueFrom(config$),
       });
+    };
 
     boundGetApmIndices().then((indices) => {
       plugins.home?.tutorials.registerTutorial(
@@ -218,21 +221,25 @@ export class APMPlugin
     }
 
     registerFleetPolicyCallbacks({
-      plugins: resourcePlugins,
-      ruleDataClient,
-      config: currentConfig,
       logger: this.logger,
-      kibanaVersion: this.initContext.env.packageInfo.version,
+      coreStartPromise: getCoreStart(),
+      plugins: resourcePlugins,
+      config: currentConfig,
+    });
+
+    // This will add an API key to all existing APM package policies
+    addApiKeysToEveryPackagePolicyIfMissing({
+      coreStartPromise: getCoreStart(),
+      pluginStartPromise: getPluginStart(),
+      logger: this.logger,
     });
 
-    const fleetStartPromise = resourcePlugins.fleet?.start();
     const taskManager = plugins.taskManager;
 
     // create source map index and run migrations
     scheduleSourceMapMigration({
       coreStartPromise: getCoreStart(),
       pluginStartPromise: getPluginStart(),
-      fleetStartPromise,
       taskManager,
       logger: this.logger,
     }).catch((e) => {

x-pack/plugins/apm/server/routes/agent_keys/get_agent_keys.ts

@@ -17,11 +17,11 @@ export async function getAgentKeys({
     query: {
       bool: {
         filter: [
-          {
-            term: {
-              'metadata.application': 'apm',
-            },
-          },
+          // only retrieve APM keys
+          { term: { 'metadata.application': 'apm' } },
+
+          // exclude system keys
+          { bool: { must_not: { term: { 'metadata.system': true } } } },
         ],
       },
     },

x-pack/plugins/apm/server/routes/fleet/api_keys/add_api_keys_to_policies_if_missing.ts

@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { omit } from 'lodash';
+import { PackagePolicy } from '@kbn/fleet-plugin/common';
+import { CoreStart, Logger } from '@kbn/core/server';
+import { FleetStartContract } from '@kbn/fleet-plugin/server';
+import { getInternalSavedObjectsClient } from '../../../lib/helpers/get_internal_saved_objects_client';
+import { APMPluginStartDependencies } from '../../../types';
+import { getApmPackagePolicies } from '../get_apm_package_policies';
+import {
+  createApmAgentConfigApiKey,
+  createApmSourceMapApiKey,
+} from './create_apm_api_keys';
+import {
+  getPackagePolicyWithApiKeys,
+  policyHasApiKey,
+} from '../get_package_policy_decorators';
+
+export async function addApiKeysToEveryPackagePolicyIfMissing({
+  coreStartPromise,
+  pluginStartPromise,
+  logger,
+}: {
+  coreStartPromise: Promise<CoreStart>;
+  pluginStartPromise: Promise<APMPluginStartDependencies>;
+  logger: Logger;
+}) {
+  const coreStart = await coreStartPromise;
+  const { fleet } = await pluginStartPromise;
+  if (!fleet) {
+    return;
+  }
+
+  const apmFleetPolicies = await getApmPackagePolicies({
+    coreStart,
+    fleetPluginStart: fleet,
+  });
+
+  return Promise.all(
+    apmFleetPolicies.items.map((policy) => {
+      return addApiKeysToPackagePolicyIfMissing({
+        policy,
+        coreStart,
+        fleet,
+        logger,
+      });
+    })
+  );
+}
+
+export async function addApiKeysToPackagePolicyIfMissing({
+  policy,
+  coreStart,
+  fleet,
+  logger,
+}: {
+  policy: PackagePolicy;
+  coreStart: CoreStart;
+  fleet: FleetStartContract;
+  logger: Logger;
+}) {
+  if (policyHasApiKey(policy)) {
+    logger.debug(`Policy (${policy.id}) already has api key`);
+    return;
+  }
+
+  logger.debug(`Policy (${policy.id}) does not have api key`);
+
+  const agentConfigApiKey = await createApmAgentConfigApiKey({
+    coreStart,
+    logger,
+    packagePolicyId: policy.id,
+  });
+
+  const sourceMapApiKey = await createApmSourceMapApiKey({
+    coreStart,
+    logger,
+    packagePolicyId: policy.id,
+  });
+
+  const packagePolicyTrimmed = omit(policy, [
+    'id',
+    'revision',
+    'updated_at',
+    'updated_by',
+  ]);
+  const policyWithApiKeys = getPackagePolicyWithApiKeys({
+    packagePolicy: packagePolicyTrimmed,
+    agentConfigApiKey,
+    sourceMapApiKey,
+  });
+
+  const internalESClient = coreStart.elasticsearch.client.asInternalUser;
+  const savedObjectsClient = await getInternalSavedObjectsClient(coreStart);
+  const newPolicy = await fleet.packagePolicyService.update(
+    savedObjectsClient,
+    internalESClient,
+    policy.id,
+    policyWithApiKeys
+  );
+
+  logger.debug(`Added api keys to policy ${policy.id}`);
+
+  return newPolicy;
+}

x-pack/plugins/apm/server/routes/fleet/api_keys/create_apm_api_keys.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreStart, Logger } from '@kbn/core/server';
+
+const apiKeyMetadata = {
+  application: 'apm',
+  consumer: 'apm-server',
+  system: true,
+};
+
+const indexLevelPrivileges = ['read' as const];
+
+export async function createApmSourceMapApiKey({
+  coreStart,
+  logger,
+  packagePolicyId,
+}: {
+  coreStart: CoreStart;
+  logger: Logger;
+  packagePolicyId: string;
+}) {
+  logger.debug('Creating source map API Key');
+
+  const response =
+    await coreStart.elasticsearch.client.asInternalUser.security.createApiKey({
+      body: {
+        name: `Source map read access (Package policy: "${packagePolicyId}")`,
+        metadata: {
+          ...apiKeyMetadata,
+          description:
+            'Provides read access to the source maps index. Created for APM Server',
+          package_policy_id: packagePolicyId,
+          type: 'source-map',
+        },
+        role_descriptors: {
+          apmSystemIndices: {
+            index: [
+              {
+                names: ['.apm-source-map'],
+                privileges: indexLevelPrivileges,
+                allow_restricted_indices: true,
+              },
+            ],
+          },
+        },
+      },
+    });
+
+  logger.debug('Created source map API Key');
+
+  return `${response.id}:${response.api_key}`;
+}
+
+export async function createApmAgentConfigApiKey({
+  coreStart,
+  logger,
+  packagePolicyId,
+}: {
+  coreStart: CoreStart;
+  logger: Logger;
+  packagePolicyId: string;
+}) {
+  logger.debug('Creating agent configuration API Key');
+
+  const response =
+    await coreStart.elasticsearch.client.asInternalUser.security.createApiKey({
+      body: {
+        name: `Agent Configuration read access (Package policy: "${packagePolicyId}")`,
+        metadata: {
+          ...apiKeyMetadata,
+          description:
+            'Provides read access to the agent configurations index. Created for APM Server',
+          package_policy_id: packagePolicyId,
+          type: 'agent-configuration',
+        },
+        role_descriptors: {
+          apmSystemIndices: {
+            index: [
+              {
+                names: ['.apm-agent-configuration'],
+                privileges: indexLevelPrivileges,
+                allow_restricted_indices: true,
+              },
+            ],
+          },
+        },
+      },
+    });
+
+  logger.debug('Created agent configuration API Key');
+
+  return `${response.id}:${response.api_key}`;
+}

x-pack/plugins/apm/server/routes/fleet/create_cloud_apm_package_policy.ts

@@ -21,7 +21,7 @@ import {
   APMPluginStartDependencies,
 } from '../../types';
 import { getApmPackagePolicyDefinition } from './get_apm_package_policy_definition';
-import { mergePackagePolicyWithApm } from './merge_package_policy_with_apm';
+import { decoratePackagePolicyWithAgentConfigAndSourceMap } from './merge_package_policy_with_apm';
 import { ELASTIC_CLOUD_APM_AGENT_POLICY_ID } from '../../../common/fleet';
 import { APMInternalESClient } from '../../lib/helpers/create_es_client/create_internal_es_client';
 
@@ -56,11 +56,12 @@ export async function createCloudApmPackgePolicy({
     fleetPluginStart,
     request,
   });
-  const mergedAPMPackagePolicy = await mergePackagePolicyWithApm({
-    internalESClient,
-    packagePolicy: apmPackagePolicyDefinition,
-    fleetPluginStart,
-  });
+  const mergedAPMPackagePolicy =
+    await decoratePackagePolicyWithAgentConfigAndSourceMap({
+      internalESClient,
+      packagePolicy: apmPackagePolicyDefinition,
+      fleetPluginStart,
+    });
   logger.info(`Fleet migration on Cloud - apmPackagePolicy create start`);
   const apmPackagePolicy = await fleetPluginStart.packagePolicyService.create(
     savedObjectsClient,

x-pack/plugins/apm/server/routes/fleet/get_agents.ts

@@ -5,26 +5,22 @@
  * 2.0.
  */
 
-import {
-  CoreSetup,
-  CoreStart,
-  SavedObjectsClientContract,
-} from '@kbn/core/server';
+import { CoreStart, SavedObjectsClientContract } from '@kbn/core/server';
 import { APMPluginStartDependencies } from '../../types';
 import { getInternalSavedObjectsClient } from '../../lib/helpers/get_internal_saved_objects_client';
 
 export async function getFleetAgents({
   policyIds,
-  core,
+  coreStart,
   fleetPluginStart,
 }: {
   policyIds: string[];
-  core: { setup: CoreSetup; start: () => Promise<CoreStart> };
+  coreStart: CoreStart;
   fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
 }) {
   // @ts-ignore
   const savedObjectsClient: SavedObjectsClientContract =
-    await getInternalSavedObjectsClient(core.setup);
+    await getInternalSavedObjectsClient(coreStart);
 
   return await fleetPluginStart.agentPolicyService.getByIds(
     savedObjectsClient,

x-pack/plugins/apm/server/routes/fleet/get_apm_package_policies.ts

@@ -5,24 +5,20 @@
  * 2.0.
  */
 
-import {
-  CoreSetup,
-  CoreStart,
-  SavedObjectsClientContract,
-} from '@kbn/core/server';
+import { CoreStart, SavedObjectsClientContract } from '@kbn/core/server';
 import { APMPluginStartDependencies } from '../../types';
 import { getInternalSavedObjectsClient } from '../../lib/helpers/get_internal_saved_objects_client';
 
 export async function getApmPackagePolicies({
-  core,
+  coreStart,
   fleetPluginStart,
 }: {
-  core: { setup: CoreSetup; start: () => Promise<CoreStart> };
+  coreStart: CoreStart;
   fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
 }) {
   // @ts-ignore
   const savedObjectsClient: SavedObjectsClientContract =
-    await getInternalSavedObjectsClient(core.setup);
+    await getInternalSavedObjectsClient(coreStart);
   return await fleetPluginStart.packagePolicyService.list(savedObjectsClient, {
     kuery: 'ingest-package-policies.package.name:apm',
   });

x-pack/plugins/apm/server/routes/fleet/get_package_policy_decorators.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { cloneDeep, get, set } from 'lodash';
+import { NewPackagePolicy } from '@kbn/fleet-plugin/common';
+import { AgentConfiguration } from '../../../common/agent_configuration/configuration_types';
+import { AGENT_NAME } from '../../../common/es_fields/apm';
+import { ArtifactSourceMap } from './source_maps';
+
+// agent config
+export const AGENT_CONFIG_PATH = `inputs[0].config['apm-server'].value.agent_config`;
+export const AGENT_CONFIG_API_KEY_PATH = `inputs[0].config['apm-server'].value.agent.config.elasticsearch.api_key`;
+
+// source map
+export const SOURCE_MAP_API_KEY_PATH = `inputs[0].config['apm-server'].value.rum.source_mapping.elasticsearch.api_key`;
+export const SOURCE_MAP_PATH = `inputs[0].config['apm-server'].value.rum.source_mapping.metadata`;
+
+/*
+ * Will decorate the package policy with agent configurations
+ */
+export function getPackagePolicyWithAgentConfigurations(
+  packagePolicy: NewPackagePolicy,
+  agentConfigurations: AgentConfiguration[]
+) {
+  const packagePolicyClone = cloneDeep(packagePolicy);
+
+  const value = agentConfigurations.map((configuration) => ({
+    service: configuration.service,
+    config: configuration.settings,
+    etag: configuration.etag,
+    [AGENT_NAME]: configuration.agent_name,
+  }));
+
+  set(packagePolicyClone, AGENT_CONFIG_PATH, value);
+  return packagePolicyClone;
+}
+
+/*
+ * Will decorate the package policy with source maps
+ */
+export function getPackagePolicyWithSourceMap({
+  packagePolicy,
+  artifacts,
+}: {
+  packagePolicy: NewPackagePolicy;
+  artifacts: ArtifactSourceMap[];
+}): NewPackagePolicy {
+  const packagePolicyClone = cloneDeep(packagePolicy);
+
+  const value = artifacts.map((artifact) => ({
+    'service.name': artifact.body.serviceName,
+    'service.version': artifact.body.serviceVersion,
+    'bundle.filepath': artifact.body.bundleFilepath,
+    'sourcemap.url': artifact.relative_url,
+  }));
+  set(packagePolicyClone, SOURCE_MAP_PATH, value);
+  return packagePolicyClone;
+}
+
+/*
+ * Will decorate the package policy with api keys for source maps and agent configurations
+ */
+export function getPackagePolicyWithApiKeys({
+  packagePolicy,
+  agentConfigApiKey,
+  sourceMapApiKey,
+}: {
+  packagePolicy: NewPackagePolicy;
+  agentConfigApiKey: string;
+  sourceMapApiKey: string;
+}) {
+  const packagePolicyClone = cloneDeep(packagePolicy);
+  set(packagePolicyClone, SOURCE_MAP_API_KEY_PATH, sourceMapApiKey);
+  set(packagePolicyClone, AGENT_CONFIG_API_KEY_PATH, agentConfigApiKey);
+  return packagePolicyClone;
+}
+
+export function policyHasApiKey(packagePolicy: NewPackagePolicy) {
+  return (
+    get(packagePolicy, AGENT_CONFIG_API_KEY_PATH) !== undefined ||
+    get(packagePolicy, SOURCE_MAP_API_KEY_PATH) !== undefined
+  );
+}

x-pack/plugins/apm/server/routes/fleet/merge_package_policy_with_apm.ts

@@ -5,31 +5,40 @@
  * 2.0.
  */
 
+import { NewPackagePolicy } from '@kbn/fleet-plugin/common';
 import { APMInternalESClient } from '../../lib/helpers/create_es_client/create_internal_es_client';
 import { APMPluginStartDependencies } from '../../types';
 import { listConfigurations } from '../settings/agent_configuration/list_configurations';
 import {
   getPackagePolicyWithAgentConfigurations,
-  PackagePolicy,
-} from './register_fleet_policy_callbacks';
-import {
   getPackagePolicyWithSourceMap,
-  listSourceMapArtifacts,
-} from './source_maps';
+} from './get_package_policy_decorators';
+import { listSourceMapArtifacts } from './source_maps';
 
-export async function mergePackagePolicyWithApm({
+export async function decoratePackagePolicyWithAgentConfigAndSourceMap({
   packagePolicy,
   internalESClient,
   fleetPluginStart,
 }: {
-  packagePolicy: PackagePolicy;
+  packagePolicy: NewPackagePolicy;
   internalESClient: APMInternalESClient;
   fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
 }) {
-  const agentConfigurations = await listConfigurations(internalESClient);
-  const { artifacts } = await listSourceMapArtifacts({ fleetPluginStart });
-  return getPackagePolicyWithAgentConfigurations(
-    getPackagePolicyWithSourceMap({ packagePolicy, artifacts }),
-    agentConfigurations
-  );
+  const [agentConfigurations, { artifacts }] = await Promise.all([
+    listConfigurations(internalESClient),
+    listSourceMapArtifacts({ fleetPluginStart }),
+  ]);
+
+  const policyWithSourceMaps = getPackagePolicyWithSourceMap({
+    packagePolicy,
+    artifacts,
+  });
+
+  const policyWithAgentConfigAndSourceMaps =
+    getPackagePolicyWithAgentConfigurations(
+      policyWithSourceMaps,
+      agentConfigurations
+    );
+
+  return policyWithAgentConfigAndSourceMaps;
 }

x-pack/plugins/apm/server/routes/fleet/register_fleet_policy_callbacks.ts

@@ -5,95 +5,150 @@
  * 2.0.
  */
 
+import { Logger, CoreStart } from '@kbn/core/server';
 import {
+  FleetStartContract,
   PostPackagePolicyCreateCallback,
+  PostPackagePolicyDeleteCallback,
+  PostPackagePolicyPostCreateCallback,
   PutPackagePolicyUpdateCallback,
 } from '@kbn/fleet-plugin/server';
-import {
-  NewPackagePolicy,
-  UpdatePackagePolicy,
-} from '@kbn/fleet-plugin/common';
+import { get } from 'lodash';
 import { APMPlugin, APMRouteHandlerResources } from '../..';
 import { createInternalESClient } from '../../lib/helpers/create_es_client/create_internal_es_client';
-import { AgentConfiguration } from '../../../common/agent_configuration/configuration_types';
-import { AGENT_NAME } from '../../../common/es_fields/apm';
-import { APMPluginStartDependencies } from '../../types';
-import { mergePackagePolicyWithApm } from './merge_package_policy_with_apm';
+import { decoratePackagePolicyWithAgentConfigAndSourceMap } from './merge_package_policy_with_apm';
+import { addApiKeysToPackagePolicyIfMissing } from './api_keys/add_api_keys_to_policies_if_missing';
+import {
+  AGENT_CONFIG_API_KEY_PATH,
+  SOURCE_MAP_API_KEY_PATH,
+} from './get_package_policy_decorators';
 
 export async function registerFleetPolicyCallbacks({
-  plugins,
-  ruleDataClient,
-  config,
   logger,
-  kibanaVersion,
+  coreStartPromise,
+  plugins,
+  config,
 }: {
+  logger: Logger;
+  coreStartPromise: Promise<CoreStart>;
   plugins: APMRouteHandlerResources['plugins'];
-  ruleDataClient: APMRouteHandlerResources['ruleDataClient'];
   config: NonNullable<APMPlugin['currentConfig']>;
-  logger: NonNullable<APMPlugin['logger']>;
-  kibanaVersion: string;
 }) {
   if (!plugins.fleet) {
     return;
   }
+
   const fleetPluginStart = await plugins.fleet.start();
+  const coreStart = await coreStartPromise;
 
-  // Registers a callback invoked when a policy is created to populate the APM
-  // integration policy with pre-existing agent configurations and source maps
-  registerPackagePolicyExternalCallback({
-    fleetPluginStart,
-    callbackName: 'packagePolicyCreate',
-    plugins,
-    ruleDataClient,
-    config,
-    logger,
-    kibanaVersion,
-  });
+  fleetPluginStart.registerExternalCallback(
+    'packagePolicyUpdate',
+    onPackagePolicyCreateOrUpdate({ fleetPluginStart, config })
+  );
 
-  // Registers a callback invoked when a policy is updated to populate the APM
-  // integration policy with existing agent configurations and source maps
-  registerPackagePolicyExternalCallback({
-    fleetPluginStart,
-    callbackName: 'packagePolicyUpdate',
-    plugins,
-    ruleDataClient,
-    config,
-    logger,
-    kibanaVersion,
-  });
+  fleetPluginStart.registerExternalCallback(
+    'packagePolicyCreate',
+    onPackagePolicyCreateOrUpdate({ fleetPluginStart, config })
+  );
+
+  fleetPluginStart.registerExternalCallback(
+    'packagePolicyDelete',
+    onPackagePolicyDelete({ coreStart, logger })
+  );
+
+  fleetPluginStart.registerExternalCallback(
+    'packagePolicyPostCreate',
+    onPackagePolicyPostCreate({
+      fleet: fleetPluginStart,
+      coreStart,
+      logger,
+    })
+  );
 }
 
-type ExternalCallbackParams =
-  | Parameters<PostPackagePolicyCreateCallback>
-  | Parameters<PutPackagePolicyUpdateCallback>;
-export type PackagePolicy = NewPackagePolicy | UpdatePackagePolicy;
-type Context = ExternalCallbackParams[1];
-type Request = ExternalCallbackParams[2];
-
-function registerPackagePolicyExternalCallback({
-  fleetPluginStart,
-  callbackName,
-  plugins,
-  ruleDataClient,
-  config,
+function onPackagePolicyDelete({
+  coreStart,
   logger,
-  kibanaVersion,
 }: {
-  fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
-  callbackName: 'packagePolicyCreate' | 'packagePolicyUpdate';
-  plugins: APMRouteHandlerResources['plugins'];
-  ruleDataClient: APMRouteHandlerResources['ruleDataClient'];
+  coreStart: CoreStart;
+  logger: Logger;
+}): PostPackagePolicyDeleteCallback {
+  return async (packagePolicies) => {
+    // console.log(`packagePolicyDelete:`, packagePolicies);
+    const promises = packagePolicies.map(async (packagePolicy) => {
+      if (packagePolicy.package?.name !== 'apm') {
+        return packagePolicy;
+      }
+
+      const internalESClient = coreStart.elasticsearch.client.asInternalUser;
+
+      const [agentConfigApiKeyId] = get(
+        packagePolicy,
+        AGENT_CONFIG_API_KEY_PATH
+      ).split(':');
+
+      const [sourceMapApiKeyId] = get(
+        packagePolicy,
+        SOURCE_MAP_API_KEY_PATH
+      ).split(':');
+
+      logger.debug(
+        `Deleting API keys: ${agentConfigApiKeyId}, ${sourceMapApiKeyId} (package policy: ${packagePolicy.id})`
+      );
+
+      try {
+        await internalESClient.security.invalidateApiKey({
+          body: { ids: [agentConfigApiKeyId, sourceMapApiKeyId], owner: true },
+        });
+      } catch (e) {
+        logger.error(
+          `Failed to delete API keys: ${agentConfigApiKeyId}, ${sourceMapApiKeyId} (package policy: ${packagePolicy.id})`
+        );
+      }
+    });
+
+    await Promise.all(promises);
+  };
+}
+
+function onPackagePolicyPostCreate({
+  fleet,
+  coreStart,
+  logger,
+}: {
+  fleet: FleetStartContract;
+  coreStart: CoreStart;
+  logger: Logger;
+}): PostPackagePolicyPostCreateCallback {
+  return async (packagePolicy) => {
+    if (packagePolicy.package?.name !== 'apm') {
+      return packagePolicy;
+    }
+
+    // add api key to new package policy
+    await addApiKeysToPackagePolicyIfMissing({
+      policy: packagePolicy,
+      coreStart,
+      fleet,
+      logger,
+    });
+
+    return packagePolicy;
+  };
+}
+
+/*
+ * This is called when a new package policy is created.
+ * It will add an API key to the package policy.
+ */
+function onPackagePolicyCreateOrUpdate({
+  fleetPluginStart,
+  config,
+}: {
+  fleetPluginStart: FleetStartContract;
   config: NonNullable<APMPlugin['currentConfig']>;
-  logger: NonNullable<APMPlugin['logger']>;
-  kibanaVersion: string;
-}) {
-  const callbackFn:
-    | PostPackagePolicyCreateCallback
-    | PutPackagePolicyUpdateCallback = async (
-    packagePolicy: PackagePolicy,
-    context: Context,
-    request: Request
-  ) => {
+}): PutPackagePolicyUpdateCallback & PostPackagePolicyCreateCallback {
+  return async (packagePolicy, context, request) => {
     if (packagePolicy.package?.name !== 'apm') {
       return packagePolicy;
     }
@@ -105,46 +160,10 @@ function registerPackagePolicyExternalCallback({
       config,
     });
 
-    return await mergePackagePolicyWithApm({
+    return await decoratePackagePolicyWithAgentConfigAndSourceMap({
       internalESClient,
       fleetPluginStart,
       packagePolicy,
     });
   };
-
-  fleetPluginStart.registerExternalCallback(callbackName, callbackFn);
-}
-
-export const APM_SERVER = 'apm-server';
-
-// Immutable function applies the given package policy with a set of agent configurations
-export function getPackagePolicyWithAgentConfigurations(
-  packagePolicy: PackagePolicy,
-  agentConfigurations: AgentConfiguration[]
-) {
-  const [firstInput, ...restInputs] = packagePolicy.inputs;
-  const apmServerValue = firstInput?.config?.[APM_SERVER].value;
-  return {
-    ...packagePolicy,
-    inputs: [
-      {
-        ...firstInput,
-        config: {
-          ...firstInput.config,
-          [APM_SERVER]: {
-            value: {
-              ...apmServerValue,
-              agent_config: agentConfigurations.map((configuration) => ({
-                service: configuration.service,
-                config: configuration.settings,
-                etag: configuration.etag,
-                [AGENT_NAME]: configuration.agent_name,
-              })),
-            },
-          },
-        },
-      },
-      ...restInputs,
-    ],
-  };
 }

x-pack/plugins/apm/server/routes/fleet/route.ts

@@ -36,8 +36,9 @@ const hasFleetDataRoute = createApmServerRoute({
     if (!fleetPluginStart) {
       return { hasApmPolicies: false };
     }
+    const coreStart = await core.start();
     const packagePolicies = await getApmPackagePolicies({
-      core,
+      coreStart,
       fleetPluginStart,
     });
     return { hasApmPolicies: packagePolicies.total > 0 };
@@ -90,8 +91,9 @@ const fleetAgentsRoute = createApmServerRoute({
       return { cloudStandaloneSetup, fleetAgents: [], isFleetEnabled: false };
     }
     // fetches package policies that contains APM integrations
+    const coreStart = await core.start();
     const packagePolicies = await getApmPackagePolicies({
-      core,
+      coreStart,
       fleetPluginStart,
     });
 
@@ -100,7 +102,7 @@ const fleetAgentsRoute = createApmServerRoute({
     // fetches all agents with the found package policies
     const fleetAgents = await getFleetAgents({
       policyIds: Object.keys(policiesGroupedById),
-      core,
+      coreStart,
       fleetPluginStart,
     });
 
@@ -140,7 +142,8 @@ const saveApmServerSchemaRoute = createApmServerRoute({
   }),
   handler: async (resources): Promise<void> => {
     const { params, logger, core } = resources;
-    const savedObjectsClient = await getInternalSavedObjectsClient(core.setup);
+    const coreStart = await core.start();
+    const savedObjectsClient = await getInternalSavedObjectsClient(coreStart);
     const { schema } = params.body;
     await savedObjectsClient.create(
       APM_SERVER_SCHEMA_SAVED_OBJECT_TYPE,
@@ -199,8 +202,9 @@ const getMigrationCheckRoute = createApmServerRoute({
         })
       : undefined;
     const apmPackagePolicy = getApmPackagePolicy(cloudAgentPolicy);
+    const coreStart = await core.start();
     const packagePolicies = await getApmPackagePolicies({
-      core,
+      coreStart,
       fleetPluginStart,
     });
     const latestApmPackage = await getLatestApmPackage({

x-pack/plugins/apm/server/routes/fleet/source_maps.test.ts

@@ -5,11 +5,8 @@
  * 2.0.
  */
 
-import {
-  ArtifactSourceMap,
-  getPackagePolicyWithSourceMap,
-  getCleanedBundleFilePath,
-} from './source_maps';
+import { getPackagePolicyWithSourceMap } from './get_package_policy_decorators';
+import { ArtifactSourceMap, getCleanedBundleFilePath } from './source_maps';
 
 const packagePolicy = {
   id: '123',

x-pack/plugins/apm/server/routes/fleet/source_maps.ts

@@ -5,7 +5,6 @@
  * 2.0.
  */
 import {
-  CoreSetup,
   CoreStart,
   ElasticsearchClient,
   SavedObjectsClientContract,
@@ -16,7 +15,7 @@ import { Artifact } from '@kbn/fleet-plugin/server';
 import { SourceMap } from '../source_maps/route';
 import { APMPluginStartDependencies } from '../../types';
 import { getApmPackagePolicies } from './get_apm_package_policies';
-import { APM_SERVER, PackagePolicy } from './register_fleet_policy_callbacks';
+import { getPackagePolicyWithSourceMap } from './get_package_policy_decorators';
 
 const doUnzip = promisify(unzip);
 
@@ -97,57 +96,20 @@ export async function deleteFleetSourcemapArtifact({
   return apmArtifactClient.deleteArtifact(id);
 }
 
-export function getPackagePolicyWithSourceMap({
-  packagePolicy,
-  artifacts,
-}: {
-  packagePolicy: PackagePolicy;
-  artifacts: ArtifactSourceMap[];
-}) {
-  const [firstInput, ...restInputs] = packagePolicy.inputs;
-  return {
-    ...packagePolicy,
-    inputs: [
-      {
-        ...firstInput,
-        config: {
-          ...firstInput.config,
-          [APM_SERVER]: {
-            value: {
-              ...firstInput?.config?.[APM_SERVER].value,
-              rum: {
-                source_mapping: {
-                  metadata: artifacts.map((artifact) => ({
-                    'service.name': artifact.body.serviceName,
-                    'service.version': artifact.body.serviceVersion,
-                    'bundle.filepath': artifact.body.bundleFilepath,
-                    'sourcemap.url': artifact.relative_url,
-                  })),
-                },
-              },
-            },
-          },
-        },
-      },
-      ...restInputs,
-    ],
-  };
-}
-
 export async function updateSourceMapsOnFleetPolicies({
-  core,
+  coreStart,
   fleetPluginStart,
   savedObjectsClient,
   internalESClient,
 }: {
-  core: { setup: CoreSetup; start: () => Promise<CoreStart> };
+  coreStart: CoreStart;
   fleetPluginStart: FleetPluginStart;
   savedObjectsClient: SavedObjectsClientContract;
   internalESClient: ElasticsearchClient;
 }) {
   const { artifacts } = await listSourceMapArtifacts({ fleetPluginStart });
   const apmFleetPolicies = await getApmPackagePolicies({
-    core,
+    coreStart,
     fleetPluginStart,
   });
 

x-pack/plugins/apm/server/routes/fleet/sync_agent_configs_to_apm_package_policies.ts

@@ -5,26 +5,22 @@
  * 2.0.
  */
 
-import {
-  CoreSetup,
-  CoreStart,
-  SavedObjectsClientContract,
-} from '@kbn/core/server';
+import { CoreStart, SavedObjectsClientContract } from '@kbn/core/server';
 import { TelemetryUsageCounter } from '../typings';
 import { APMPluginStartDependencies } from '../../types';
 import { getInternalSavedObjectsClient } from '../../lib/helpers/get_internal_saved_objects_client';
 import { listConfigurations } from '../settings/agent_configuration/list_configurations';
 import { getApmPackagePolicies } from './get_apm_package_policies';
-import { getPackagePolicyWithAgentConfigurations } from './register_fleet_policy_callbacks';
+import { getPackagePolicyWithAgentConfigurations } from './get_package_policy_decorators';
 import { APMInternalESClient } from '../../lib/helpers/create_es_client/create_internal_es_client';
 
 export async function syncAgentConfigsToApmPackagePolicies({
-  core,
+  coreStartPromise,
   fleetPluginStart,
   internalESClient,
   telemetryUsageCounter,
 }: {
-  core: { setup: CoreSetup; start: () => Promise<CoreStart> };
+  coreStartPromise: Promise<CoreStart>;
   fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
   internalESClient: APMInternalESClient;
   telemetryUsageCounter?: TelemetryUsageCounter;
@@ -35,16 +31,13 @@ export async function syncAgentConfigsToApmPackagePolicies({
       counterType: 'success',
     });
   }
-  const coreStart = await core.start();
+  const coreStart = await coreStartPromise;
   const esClient = coreStart.elasticsearch.client.asInternalUser;
   const [savedObjectsClient, agentConfigurations, packagePolicies] =
     await Promise.all([
-      getInternalSavedObjectsClient(core.setup),
+      getInternalSavedObjectsClient(coreStart),
       listConfigurations(internalESClient),
-      getApmPackagePolicies({
-        core,
-        fleetPluginStart,
-      }),
+      getApmPackagePolicies({ coreStart, fleetPluginStart }),
     ]);
 
   return Promise.all(

x-pack/plugins/apm/server/routes/settings/agent_configuration/route.ts

@@ -143,7 +143,7 @@ const deleteAgentConfigurationRoute = createApmServerRoute({
 
     if (resources.plugins.fleet) {
       await syncAgentConfigsToApmPackagePolicies({
-        core,
+        coreStartPromise: core.start(),
         fleetPluginStart: await resources.plugins.fleet.start(),
         internalESClient,
         telemetryUsageCounter,
@@ -214,7 +214,7 @@ const createOrUpdateAgentConfigurationRoute = createApmServerRoute({
 
     if (resources.plugins.fleet) {
       await syncAgentConfigsToApmPackagePolicies({
-        core,
+        coreStartPromise: core.start(),
         fleetPluginStart: await resources.plugins.fleet.start(),
         internalESClient,
         telemetryUsageCounter,

x-pack/plugins/apm/server/routes/source_maps/route.ts

@@ -108,7 +108,7 @@ const uploadSourceMapRoute = createApmServerRoute({
     const fleetPluginStart = await plugins.fleet?.start();
     const coreStart = await core.start();
     const internalESClient = coreStart.elasticsearch.client.asInternalUser;
-    const savedObjectsClient = await getInternalSavedObjectsClient(core.setup);
+    const savedObjectsClient = await getInternalSavedObjectsClient(coreStart);
     try {
       if (fleetPluginStart) {
         // create source map as fleet artifact
@@ -136,7 +136,7 @@ const uploadSourceMapRoute = createApmServerRoute({
 
         // sync source map to fleet policy
         await updateSourceMapsOnFleetPolicies({
-          core,
+          coreStart,
           fleetPluginStart,
           savedObjectsClient:
             savedObjectsClient as unknown as SavedObjectsClientContract,
@@ -167,13 +167,13 @@ const deleteSourceMapRoute = createApmServerRoute({
     const { id } = params.path;
     const coreStart = await core.start();
     const internalESClient = coreStart.elasticsearch.client.asInternalUser;
-    const savedObjectsClient = await getInternalSavedObjectsClient(core.setup);
+    const savedObjectsClient = await getInternalSavedObjectsClient(coreStart);
     try {
       if (fleetPluginStart) {
         await deleteFleetSourcemapArtifact({ id, fleetPluginStart });
         await deleteApmSourceMap({ internalESClient, fleetId: id });
         await updateSourceMapsOnFleetPolicies({
-          core,
+          coreStart,
           fleetPluginStart,
           savedObjectsClient:
             savedObjectsClient as unknown as SavedObjectsClientContract,

x-pack/plugins/apm/server/routes/source_maps/schedule_source_map_migration.ts

@@ -24,13 +24,11 @@ const TASK_TYPE = 'apm-source-map-migration-task';
 export async function scheduleSourceMapMigration({
   coreStartPromise,
   pluginStartPromise,
-  fleetStartPromise,
   taskManager,
   logger,
 }: {
   coreStartPromise: Promise<CoreStart>;
   pluginStartPromise: Promise<APMPluginStartDependencies>;
-  fleetStartPromise?: Promise<FleetStartContract>;
   taskManager?: TaskManagerSetupContract;
   logger: Logger;
 }) {
@@ -64,7 +62,8 @@ export async function scheduleSourceMapMigration({
               logger,
             });
 
-            const fleet = await fleetStartPromise;
+            const pluginStart = await pluginStartPromise;
+            const fleet = await pluginStart.fleet;
             if (fleet) {
               await runFleetSourcemapArtifactsMigration({
                 taskState,

x-pack/plugins/fleet/server/index.ts

@@ -27,6 +27,7 @@ export type { FleetSetupContract, FleetSetupDeps, FleetStartContract } from './p
 export type {
   ExternalCallback,
   PutPackagePolicyUpdateCallback,
+  PostPackagePolicyDeleteCallback,
   PostPackagePolicyPostDeleteCallback,
   PostPackagePolicyCreateCallback,
   FleetRequestHandlerContext,

x-pack/test/apm_api_integration/common/bettertest.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { format } from 'url';
+import supertest from 'supertest';
+import request from 'superagent';
+
+type HttpMethod = 'get' | 'post' | 'put' | 'delete';
+
+export type BetterTest = <T extends any>(options: {
+  pathname: string;
+  query?: Record<string, any>;
+  method?: HttpMethod;
+  body?: any;
+}) => Promise<{ status: number; body: T }>;
+
+/*
+ * This is a wrapper around supertest that throws an error if the response status is not 200.
+ * This is useful for tests that expect a 200 response
+ * It also makes it easier to debug tests that fail because of a 500 response.
+ */
+export function getBettertest(st: supertest.SuperTest<supertest.Test>): BetterTest {
+  return async ({ pathname, method = 'get', query, body }) => {
+    const url = format({ pathname, query });
+
+    let res: request.Response;
+    if (body) {
+      res = await st[method](url).send(body).set('kbn-xsrf', 'true');
+    } else {
+      res = await st[method](url).set('kbn-xsrf', 'true');
+    }
+
+    // supertest doesn't throw on http errors
+    if (res?.status !== 200) {
+      throw new BetterTestError(res);
+    }
+
+    return res;
+  };
+}
+
+type ErrorResponse = Omit<request.Response, 'body'> & {
+  body: {
+    statusCode: number;
+    error: string;
+    message: string;
+    attributes: object;
+  };
+};
+
+export class BetterTestError extends Error {
+  res: ErrorResponse;
+
+  constructor(res: request.Response) {
+    // @ts-expect-error
+    const req = res.req as any;
+    super(
+      `Unhandled BetterTestError:
+Status: "${res.status}"      
+Path: "${req.method} ${req.path}"
+Body: ${JSON.stringify(res.body)}`
+    );
+
+    this.res = res;
+  }
+}

x-pack/test/apm_api_integration/common/utils/expect_to_reject.ts

@@ -6,10 +6,12 @@
  */
 
 export async function expectToReject<T extends Error>(fn: () => Promise<any>): Promise<T> {
+  let res: any;
   try {
-    await fn();
+    res = await fn();
   } catch (e) {
     return e;
   }
-  throw new Error(`Expected fn to throw`);
+
+  throw new Error(`expectToReject resolved: "${JSON.stringify(res)}"`);
 }

x-pack/test/apm_api_integration/tests/fleet/apm_package_policy.spec.ts

@@ -5,20 +5,51 @@
  * 2.0.
  */
 
+import * as Url from 'url';
+import { PackagePolicy } from '@kbn/fleet-plugin/common';
+import {
+  AGENT_CONFIG_PATH,
+  AGENT_CONFIG_API_KEY_PATH,
+  SOURCE_MAP_API_KEY_PATH,
+  SOURCE_MAP_PATH,
+} from '@kbn/apm-plugin/server/routes/fleet/get_package_policy_decorators';
 import expect from '@kbn/expect';
+import { get } from 'lodash';
+import type { SourceMap } from '@kbn/apm-plugin/server/routes/source_maps/route';
+import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
+import {
+  APM_AGENT_CONFIGURATION_INDEX,
+  APM_SOURCE_MAP_INDEX,
+} from '@kbn/apm-plugin/server/routes/settings/apm_indices/get_apm_indices';
+import { createEsClientForTesting } from '@kbn/test';
 import { FtrProviderContext } from '../../common/ftr_provider_context';
 import {
   createAgentPolicy,
   createPackagePolicy,
   deleteAgentPolicy,
   deletePackagePolicy,
+  getPackagePolicy,
   setupFleet,
 } from './apm_package_policy_setup';
+import { getBettertest } from '../../common/bettertest';
+import { expectToReject } from '../../common/utils/expect_to_reject';
 
 export default function ApiTest(ftrProviderContext: FtrProviderContext) {
   const { getService } = ftrProviderContext;
   const registry = getService('registry');
   const apmApiClient = getService('apmApiClient');
+  const supertest = getService('supertest');
+  const es = getService('es');
+  const bettertest = getBettertest(supertest);
+
+  function createEsClientWithApiKeyAuth({ id, apiKey }: { id: string; apiKey: string }) {
+    const config = getService('config');
+    return createEsClientForTesting({
+      esUrl: Url.format(config.get('servers.elasticsearch')),
+      requestTimeout: config.get('timeouts.esRequestTimeout'),
+      auth: { apiKey: { id, api_key: apiKey } },
+    });
+  }
 
   async function createConfiguration(configuration: any) {
     return apmApiClient.writeUser({
@@ -34,42 +65,128 @@ export default function ApiTest(ftrProviderContext: FtrProviderContext) {
     });
   }
 
+  async function uploadSourcemap({
+    bundleFilePath,
+    serviceName,
+    serviceVersion,
+    sourcemap,
+  }: {
+    bundleFilePath: string;
+    serviceName: string;
+    serviceVersion: string;
+    sourcemap: SourceMap;
+  }) {
+    const response = await apmApiClient.writeUser({
+      endpoint: 'POST /api/apm/sourcemaps',
+      type: 'form-data',
+      params: {
+        body: {
+          bundle_filepath: bundleFilePath,
+          service_name: serviceName,
+          service_version: serviceVersion,
+          sourcemap: JSON.stringify(sourcemap),
+        },
+      },
+    });
+    return response.body;
+  }
+
+  async function getActiveApiKeysCount(packagePolicyId: string) {
+    const res = await es.transport.request({
+      method: 'GET',
+      path: '_security/_query/api_key',
+      body: {
+        size: 10000,
+        query: {
+          bool: {
+            filter: [
+              { term: { 'metadata.application': 'apm' } },
+              { term: { 'metadata.package_policy_id': packagePolicyId } },
+              { term: { invalidated: false } },
+            ],
+          },
+        },
+      },
+    });
+
+    // @ts-expect-error
+    return res.total as number;
+  }
+
   registry.when('APM package policy', { config: 'basic', archives: [] }, () => {
+    let apmPackagePolicy: PackagePolicy;
     let agentPolicyId: string;
+    let packagePolicyId: string;
 
     before(async () => {
-      await setupFleet(ftrProviderContext);
-      agentPolicyId = await createAgentPolicy(ftrProviderContext);
+      await setupFleet(bettertest);
+      agentPolicyId = await createAgentPolicy(bettertest);
+      packagePolicyId = await createPackagePolicy(bettertest, agentPolicyId);
+      apmPackagePolicy = await getPackagePolicy(bettertest, packagePolicyId); // make sure to get the latest package policy
     });
 
     after(async () => {
-      await deleteAgentPolicy(ftrProviderContext, agentPolicyId);
+      await deleteAgentPolicy(bettertest, agentPolicyId);
+      await deletePackagePolicy(bettertest, packagePolicyId);
+      expect(await getActiveApiKeysCount(packagePolicyId)).to.eql(0); // make sure all api keys for the policy are invalidated
     });
 
     describe('APM package policy callbacks', () => {
-      let apmPackagePolicy: any;
-
-      beforeEach(async () => {
-        apmPackagePolicy = await createPackagePolicy(ftrProviderContext, agentPolicyId);
+      it('sets default values for source maps', async () => {
+        const sourceMap = get(apmPackagePolicy, SOURCE_MAP_PATH);
+        expect(sourceMap).to.eql([]);
       });
 
-      afterEach(async () => {
-        await deletePackagePolicy(ftrProviderContext, apmPackagePolicy.id);
+      it('sets default values for agent configs', async () => {
+        const agentConfigs = get(apmPackagePolicy, AGENT_CONFIG_PATH);
+        expect(agentConfigs).to.eql([]);
       });
 
-      it('sets default values for agent configs and source mapping in a new package policy', async () => {
-        const apmPackageConfig = apmPackagePolicy.inputs[0].config['apm-server'].value;
-        expect(apmPackageConfig.rum.source_mapping).to.eql({ metadata: [] });
-        expect(apmPackageConfig.agent_config).to.eql([]);
+      it('creates two API keys for the package policy', async () => {
+        expect(await getActiveApiKeysCount(packagePolicyId)).to.eql(2);
+      });
+
+      it('has api key that provides access to source maps only', async () => {
+        const [id, apiKey] = get(apmPackagePolicy, SOURCE_MAP_API_KEY_PATH).split(':');
+        expect(id).to.not.be.empty();
+        expect(apiKey).to.not.be.empty();
+
+        const esClient = createEsClientWithApiKeyAuth({ id, apiKey });
+        const res = await esClient.search({ index: APM_SOURCE_MAP_INDEX });
+        expect(res.hits.hits.length).to.be(0);
+      });
+
+      it('has api api key that provides access to the agent configurations index', async () => {
+        const [id, apiKey] = get(apmPackagePolicy, AGENT_CONFIG_API_KEY_PATH).split(':');
+        expect(id).to.not.be.empty();
+        expect(apiKey).to.not.be.empty();
+
+        const esClient = createEsClientWithApiKeyAuth({ id, apiKey });
+        const res = await esClient.search({
+          index: APM_AGENT_CONFIGURATION_INDEX,
+        });
+
+        expect(res.hits.hits.length).to.be(0);
+      });
+
+      it('throws when querying agent config index with source map api key', async () => {
+        const [id, apiKey] = get(apmPackagePolicy, SOURCE_MAP_API_KEY_PATH).split(':');
+        expect(id).to.not.be.empty();
+        expect(apiKey).to.not.be.empty();
+
+        const esClient = createEsClientWithApiKeyAuth({ id, apiKey });
+        await expectToReject(() => esClient.search({ index: APM_AGENT_CONFIGURATION_INDEX }));
       });
 
       describe('Agent config', () => {
+        let packagePolicyWithAgentConfig: PackagePolicy;
         const testConfiguration = {
           service: {},
           settings: { transaction_sample_rate: '0.55' },
         };
         before(async () => {
           await createConfiguration(testConfiguration);
+          packagePolicyWithAgentConfig = await getPackagePolicy(bettertest, packagePolicyId);
         });
 
         after(async () => {
@@ -77,13 +194,50 @@ export default function ApiTest(ftrProviderContext: FtrProviderContext) {
         });
 
         it('sets the expected agent configs on the new package policy object', async () => {
-          const {
-            agent_config: [{ service, config }],
-          } = apmPackagePolicy.inputs[0].config['apm-server'].value;
+          const agentConfigs = get(packagePolicyWithAgentConfig, AGENT_CONFIG_PATH);
+          const { service, config } = agentConfigs[0];
           expect(service).to.eql({});
           expect(config).to.eql({ transaction_sample_rate: '0.55' });
         });
       });
+
+      describe('Source maps', () => {
+        let resp: APIReturnType<'POST /api/apm/sourcemaps'>;
+
+        after(async () => {
+          await apmApiClient.writeUser({
+            endpoint: 'DELETE /api/apm/sourcemaps/{id}',
+            params: { path: { id: resp.id } },
+          });
+        });
+
+        before(async () => {
+          resp = await uploadSourcemap({
+            serviceName: 'uploading-test',
+            serviceVersion: '1.0.0',
+            bundleFilePath: 'bar',
+            sourcemap: {
+              version: 123,
+              sources: [''],
+              mappings: '',
+            },
+          });
+        });
+
+        it('sets the expected source maps on the new package policy object', async () => {
+          const packagePolicyWithSourceMap = await getPackagePolicy(bettertest, packagePolicyId);
+          const sourceMap = get(packagePolicyWithSourceMap, SOURCE_MAP_PATH);
+          expect(sourceMap).to.eql([
+            {
+              'bundle.filepath': 'bar',
+              'service.name': 'uploading-test',
+              'service.version': '1.0.0',
+              'sourcemap.url':
+                '/api/fleet/artifacts/uploading-test-1.0.0/2f5d4e64ffde4acde832039186ca1652ed315fb0ecbcc1b398677b3bcba521df',
+            },
+          ]);
+        });
+      });
     });
   });
 }

x-pack/test/apm_api_integration/tests/fleet/apm_package_policy_setup.ts

@@ -4,51 +4,41 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { FtrProviderContext } from '../../common/ftr_provider_context';
+import { PackagePolicy, NewPackagePolicy, AgentPolicy } from '@kbn/fleet-plugin/common';
+import { BetterTest } from '../../common/bettertest';
 
-export async function setupFleet(ftrProviderContext: FtrProviderContext) {
-  const { getService } = ftrProviderContext;
-  const supertest = getService('supertest');
-  // Initialize fleet setup
-  await supertest.post('/api/fleet/setup').set('kbn-xsrf', 'xxx').send().expect(200);
+export function setupFleet(bettertest: BetterTest) {
+  return bettertest({ pathname: '/api/fleet/setup', method: 'post' });
 }
 
-export async function createAgentPolicy(ftrProviderContext: FtrProviderContext) {
-  const { getService } = ftrProviderContext;
-  const supertest = getService('supertest');
-  // Ceate agent policy and get id
-  const agentPolicyResponse = await supertest
-    .post('/api/fleet/agent_policies?sys_monitoring=true')
-    .set('kbn-xsrf', 'true')
-    .send({
+export async function createAgentPolicy(bettertest: BetterTest) {
+  const agentPolicyResponse = await bettertest<{ item: AgentPolicy }>({
+    pathname: '/api/fleet/agent_policies',
+    method: 'post',
+    query: { sys_monitoring: true },
+    body: {
       name: 'test_agent_policy',
       description: '',
       namespace: 'default',
       monitoring_enabled: ['logs', 'metrics'],
-    })
-    .expect(200);
+    },
+  });
+
   return agentPolicyResponse.body.item.id;
 }
 
-export async function createPackagePolicy(
-  ftrProviderContext: FtrProviderContext,
-  agentPolicyId: string
-) {
-  const { getService } = ftrProviderContext;
-  const supertest = getService('supertest');
-
+export async function createPackagePolicy(bettertest: BetterTest, agentPolicyId: string) {
   // Get version of available APM package
-  const apmPackageResponse = await supertest
-    .get(`/api/fleet/epm/packages/apm`)
-    .set('kbn-xsrf', 'true')
-    .expect(200);
+  const apmPackageResponse = await bettertest<{ item: any }>({
+    pathname: `/api/fleet/epm/packages/apm`,
+  });
   const apmPackageVersion = apmPackageResponse.body.item.version;
 
   // Create package policy for APM attached to given agent policy id
-  const packagePolicyResponse = await supertest
-    .post('/api/fleet/package_policies')
-    .set('kbn-xsrf', 'true')
-    .send({
+  const packagePolicyResponse = await bettertest<{ item: NewPackagePolicy }>({
+    pathname: '/api/fleet/package_policies',
+    method: 'post',
+    body: {
       name: 'apm-integration-test-policy',
       description: '',
       namespace: 'default',
@@ -56,33 +46,33 @@ export async function createPackagePolicy(
       enabled: true,
       inputs: [{ type: 'apm', policy_template: 'apmserver', enabled: true, streams: [], vars: {} }],
       package: { name: 'apm', title: 'Elastic APM', version: apmPackageVersion },
-    })
-    .expect(200);
-  return packagePolicyResponse.body.item;
+    },
+  });
+  return packagePolicyResponse.body.item.id as string;
 }
 
-export async function deleteAgentPolicy(
-  ftrProviderContext: FtrProviderContext,
-  agentPolicyId: string
-) {
-  const { getService } = ftrProviderContext;
-  const supertest = getService('supertest');
-  await supertest
-    .post('/api/fleet/agent_policies/delete')
-    .set('kbn-xsrf', 'true')
-    .send({ agentPolicyId })
-    .expect(200);
+export async function deleteAgentPolicy(bettertest: BetterTest, agentPolicyId: string) {
+  return await bettertest({
+    pathname: '/api/fleet/agent_policies/delete',
+    method: 'post',
+    body: { agentPolicyId },
+  });
 }
 
-export async function deletePackagePolicy(
-  ftrProviderContext: FtrProviderContext,
+export async function deletePackagePolicy(bettertest: BetterTest, packagePolicyId: string) {
+  return bettertest({
+    pathname: `/api/fleet/package_policies/delete`,
+    method: 'post',
+    body: { packagePolicyIds: [packagePolicyId] },
+  });
+}
+
+export async function getPackagePolicy(
+  bettertest: BetterTest,
   packagePolicyId: string
-) {
-  const { getService } = ftrProviderContext;
-  const supertest = getService('supertest');
-  await supertest
-    .post(`/api/fleet/package_policies/delete`)
-    .set('kbn-xsrf', 'true')
-    .send({ packagePolicyIds: [packagePolicyId] })
-    .expect(200);
+): Promise<PackagePolicy> {
+  const res = await bettertest<{ item: PackagePolicy }>({
+    pathname: `/api/fleet/package_policies/${packagePolicyId}`,
+  });
+  return res.body.item;
 }