[APM] Fleet migration support for bundled APM package (#153159)

Closes #149342.

It accomplishes this by returning the ArchivePackage, unzipped bundled
package that includes most of the same fields as the RegistryPackage.
These fields are used in APM to support the fleet migration workflow.

.buildkite/ftr_configs.yml

@@ -184,6 +184,7 @@ enabled:
   - x-pack/test/api_integration/apis/uptime/config.ts
   - x-pack/test/api_integration/apis/watcher/config.ts
   - x-pack/test/apm_api_integration/basic/config.ts
+  - x-pack/test/apm_api_integration/cloud/config.ts
   - x-pack/test/apm_api_integration/rules/config.ts
   - x-pack/test/apm_api_integration/trial/config.ts
   - x-pack/test/banners_functional/config.ts

x-pack/plugins/apm/public/components/app/settings/schema/schema_overview.tsx

@@ -248,7 +248,7 @@ function getDisabledReason({
       )
     );
   }
-  if (!hasCloudAgentPolicy) {
+  if (hasRequiredRole && !hasCloudAgentPolicy) {
     reasons.push(
       i18n.translate(
         'xpack.apm.settings.schema.disabledReason.hasCloudAgentPolicy',

x-pack/plugins/apm/server/routes/fleet/get_latest_apm_package.ts

@@ -17,12 +17,19 @@ export async function getLatestApmPackage({
   request: KibanaRequest;
 }) {
   const packageClient = fleetPluginStart.packageService.asScoped(request);
-  const { name, version } = await packageClient.fetchFindLatestPackage(
+  const latestPackage = await packageClient.fetchFindLatestPackage(
     APM_PACKAGE_NAME
   );
-  const registryPackage = await packageClient.getPackage(name, version);
-  const { title, policy_templates: policyTemplates } =
-    registryPackage.packageInfo;
+  const packageInfo =
+    'buffer' in latestPackage
+      ? (await packageClient.readBundledPackage(latestPackage)).packageInfo
+      : latestPackage;
+  const {
+    name,
+    version,
+    title,
+    policy_templates: policyTemplates,
+  } = packageInfo;
   const firstTemplate = policyTemplates?.[0];
   const policyTemplateInputVars =
     firstTemplate && 'inputs' in firstTemplate

x-pack/plugins/apm/server/routes/fleet/run_migration_check.ts

@@ -49,6 +49,17 @@ export async function runMigrationCheck({
   ]);
 
   const hasRequiredRole = isSuperuser({ securityPluginStart, request });
+  if (!hasRequiredRole) {
+    return {
+      has_cloud_agent_policy: false,
+      has_cloud_apm_package_policy: false,
+      cloud_apm_migration_enabled: cloudApmMigrationEnabled,
+      has_required_role: false,
+      cloud_apm_package_policy: undefined,
+      has_apm_integrations: false,
+      latest_apm_package_version: '',
+    };
+  }
   const cloudAgentPolicy = hasRequiredRole
     ? await getCloudAgentPolicy({
         savedObjectsClient,
@@ -57,14 +68,15 @@ export async function runMigrationCheck({
     : undefined;
   const apmPackagePolicy = getApmPackagePolicy(cloudAgentPolicy);
   const coreStart = await core.start();
-  const packagePolicies = await getApmPackagePolicies({
-    coreStart,
-    fleetPluginStart,
-  });
   const latestApmPackage = await getLatestApmPackage({
     fleetPluginStart,
     request,
   });
+
+  const packagePolicies = await getApmPackagePolicies({
+    coreStart,
+    fleetPluginStart,
+  });
   return {
     has_cloud_agent_policy: !!cloudAgentPolicy,
     has_cloud_apm_package_policy: !!apmPackagePolicy,

x-pack/plugins/apm/server/test_helpers/create_apm_users/authentication.ts

@@ -16,6 +16,7 @@ export enum ApmUsername {
   apmManageOwnAgentKeys = 'apm_manage_own_agent_keys',
   apmManageOwnAndCreateAgentKeys = 'apm_manage_own_and_create_agent_keys',
   apmMonitorClusterAndIndices = 'apm_monitor_cluster_and_indices',
+  apmManageServiceAccount = 'apm_manage_service_account',
 }
 
 export enum ApmCustomRolename {
@@ -24,6 +25,7 @@ export enum ApmCustomRolename {
   apmManageOwnAgentKeys = 'apm_manage_own_agent_keys',
   apmManageOwnAndCreateAgentKeys = 'apm_manage_own_and_create_agent_keys',
   apmMonitorClusterAndIndices = 'apm_monitor_cluster_and_indices',
+  apmManageServiceAccount = 'apm_manage_service_account',
 }
 
 export const customRoles = {
@@ -88,6 +90,11 @@ export const customRoles = {
       cluster: ['monitor'],
     },
   },
+  [ApmCustomRolename.apmManageServiceAccount]: {
+    elasticsearch: {
+      cluster: ['manage_service_account'],
+    },
+  },
 };
 
 export const users: Record<
@@ -123,6 +130,10 @@ export const users: Record<
     builtInRoleNames: ['viewer'],
     customRoleNames: [ApmCustomRolename.apmMonitorClusterAndIndices],
   },
+  [ApmUsername.apmManageServiceAccount]: {
+    builtInRoleNames: ['editor'],
+    customRoleNames: [ApmCustomRolename.apmManageServiceAccount],
+  },
 };
 
 export const APM_TEST_PASSWORD = 'changeme';

x-pack/plugins/fleet/server/services/epm/package_service.mock.ts

@@ -11,6 +11,7 @@ const createClientMock = (): jest.Mocked<PackageClient> => ({
   getInstallation: jest.fn(),
   ensureInstalledPackage: jest.fn(),
   fetchFindLatestPackage: jest.fn(),
+  readBundledPackage: jest.fn(),
   getPackage: jest.fn(),
   getPackages: jest.fn(),
   reinstallEsAssets: jest.fn(),

x-pack/plugins/fleet/server/services/epm/package_service.test.ts

@@ -26,6 +26,7 @@ import * as epmPackagesGet from './packages/get';
 import * as epmPackagesInstall from './packages/install';
 import * as epmRegistry from './registry';
 import * as epmTransformsInstall from './elasticsearch/transform/install';
+import * as epmArchiveParse from './archive/parse';
 
 const testKeys = [
   'getInstallation',
@@ -33,6 +34,7 @@ const testKeys = [
   'fetchFindLatestPackage',
   'getPackage',
   'reinstallEsAssets',
+  'readBundledPackage',
 ];
 
 function getTest(
@@ -144,6 +146,23 @@ function getTest(
         ],
       };
       break;
+    case testKeys[5]:
+      const bundledPackage = { name: 'package name', version: '8.0.0', buffer: Buffer.from([]) };
+      test = {
+        method: mocks.packageClient.readBundledPackage.bind(mocks.packageClient),
+        args: [bundledPackage],
+        spy: jest.spyOn(epmArchiveParse, 'generatePackageInfoFromArchiveBuffer'),
+        spyArgs: [bundledPackage.buffer, 'application/zip'],
+        spyResponse: {
+          packageInfo: { name: 'readBundledPackage test' },
+          paths: ['/some/test/path'],
+        },
+        expectedReturnValue: {
+          packageInfo: { name: 'readBundledPackage test' },
+          paths: ['/some/test/path'],
+        },
+      };
+      break;
     default:
       throw new Error('invalid test key');
   }

x-pack/plugins/fleet/server/services/epm/package_service.ts

@@ -32,6 +32,7 @@ import { installTransforms, isTransform } from './elasticsearch/transform/instal
 import type { FetchFindLatestPackageOptions } from './registry';
 import { fetchFindLatestPackageOrThrow, getPackage } from './registry';
 import { ensureInstalledPackage, getInstallation, getPackages } from './packages';
+import { generatePackageInfoFromArchiveBuffer } from './archive';
 
 export type InstalledAssetType = EsAssetReference;
 
@@ -54,6 +55,10 @@ export interface PackageClient {
     options?: FetchFindLatestPackageOptions
   ): Promise<RegistryPackage | BundledPackage>;
 
+  readBundledPackage(
+    bundledPackage: BundledPackage
+  ): Promise<{ packageInfo: ArchivePackage; paths: string[] }>;
+
   getPackage(
     packageName: string,
     packageVersion: string
@@ -137,6 +142,11 @@ class PackageClientImpl implements PackageClient {
     return fetchFindLatestPackageOrThrow(packageName, options);
   }
 
+  public async readBundledPackage(bundledPackage: BundledPackage) {
+    await this.#runPreflight();
+    return generatePackageInfoFromArchiveBuffer(bundledPackage.buffer, 'application/zip');
+  }
+
   public async getPackage(
     packageName: string,
     packageVersion: string,

x-pack/test/apm_api_integration/cloud/config.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { configs } from '../configs';
+
+export default configs.cloud;

x-pack/test/apm_api_integration/common/config.ts

@@ -56,7 +56,8 @@ type ApmApiClientKey =
   | 'noMlAccessUser'
   | 'manageOwnAgentKeysUser'
   | 'createAndAllAgentKeysUser'
-  | 'monitorClusterAndIndicesUser';
+  | 'monitorClusterAndIndicesUser'
+  | 'manageServiceAccount';
 
 export type ApmApiClient = Record<ApmApiClientKey, Awaited<ReturnType<typeof getApmApiClient>>>;
 
@@ -146,6 +147,10 @@ export function createTestConfig(
               kibanaServer,
               username: ApmUsername.apmMonitorClusterAndIndices,
             }),
+            manageServiceAccount: await getApmApiClient({
+              kibanaServer,
+              username: ApmUsername.apmManageServiceAccount,
+            }),
           };
         },
         ml: MachineLearningAPIProvider,

x-pack/test/apm_api_integration/configs/index.ts

@@ -37,6 +37,14 @@ const apmFtrConfigs = {
       'logging.loggers': [apmDebugLogger],
     },
   },
+  cloud: {
+    license: 'basic' as const,
+    kibanaConfig: {
+      'xpack.apm.agent.migrations.enabled': 'true',
+      'xpack.apm.forceSyntheticSource': 'true',
+      'logging.loggers': [apmDebugLogger],
+    },
+  },
 };
 
 export type APMFtrConfigName = keyof typeof apmFtrConfigs;

x-pack/test/apm_api_integration/tests/fleet/apm_package_policy_setup.ts

@@ -11,7 +11,7 @@ export function setupFleet(bettertest: BetterTest) {
   return bettertest({ pathname: '/api/fleet/setup', method: 'post' });
 }
 
-export async function createAgentPolicy(bettertest: BetterTest) {
+export async function createAgentPolicy(bettertest: BetterTest, id?: string) {
   const agentPolicyResponse = await bettertest<{ item: AgentPolicy }>({
     pathname: '/api/fleet/agent_policies',
     method: 'post',
@@ -19,6 +19,7 @@ export async function createAgentPolicy(bettertest: BetterTest) {
     body: {
       name: 'test_agent_policy',
       description: '',
+      id,
       namespace: 'default',
       monitoring_enabled: ['logs', 'metrics'],
     },
@@ -27,7 +28,11 @@ export async function createAgentPolicy(bettertest: BetterTest) {
   return agentPolicyResponse.body.item.id;
 }
 
-export async function createPackagePolicy(bettertest: BetterTest, agentPolicyId: string) {
+export async function createPackagePolicy(
+  bettertest: BetterTest,
+  agentPolicyId: string,
+  id?: string
+) {
   // Get version of available APM package
   const apmPackageResponse = await bettertest<{ item: any }>({
     pathname: `/api/fleet/epm/packages/apm`,
@@ -43,6 +48,7 @@ export async function createPackagePolicy(bettertest: BetterTest, agentPolicyId:
       description: '',
       namespace: 'default',
       policy_id: agentPolicyId,
+      id,
       enabled: true,
       inputs: [{ type: 'apm', policy_template: 'apmserver', enabled: true, streams: [], vars: {} }],
       package: { name: 'apm', title: 'Elastic APM', version: apmPackageVersion },

x-pack/test/apm_api_integration/tests/fleet/migration_check.spec.ts

@@ -0,0 +1,172 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+import {
+  createAgentPolicy,
+  createPackagePolicy,
+  deleteAgentPolicy,
+  deletePackagePolicy,
+  setupFleet,
+} from './apm_package_policy_setup';
+import { getBettertest } from '../../common/bettertest';
+
+export default function ApiTest(ftrProviderContext: FtrProviderContext) {
+  const { getService } = ftrProviderContext;
+  const registry = getService('registry');
+  const supertest = getService('supertest');
+  const bettertest = getBettertest(supertest);
+  const apmApiClient = getService('apmApiClient');
+
+  registry.when('Fleet migration check - basic', { config: 'basic', archives: [] }, () => {
+    before(async () => {
+      await setupFleet(bettertest);
+    });
+
+    describe('cloud_apm_migration_enabled', () => {
+      it('should be false when when config not set', async () => {
+        const { body } = await bettertest({
+          pathname: '/internal/apm/fleet/migration_check',
+        });
+        expect(body).to.have.property('cloud_apm_migration_enabled', false);
+      });
+    });
+  });
+
+  registry.when('Fleet migration check - cloud', { config: 'cloud', archives: [] }, () => {
+    before(async () => {
+      await setupFleet(bettertest);
+    });
+
+    describe('migration check properties', () => {
+      it('should contain all expected properties', async () => {
+        const { status, body } = await bettertest({
+          pathname: '/internal/apm/fleet/migration_check',
+        });
+        expect(status).to.equal(200);
+        expect(body).to.have.property('has_cloud_agent_policy');
+        expect(body).to.have.property('has_cloud_apm_package_policy');
+        expect(body).to.have.property('cloud_apm_migration_enabled');
+        expect(body).to.have.property('has_required_role');
+        expect(body).to.have.property('has_apm_integrations');
+        expect(body).to.have.property('latest_apm_package_version');
+      });
+    });
+
+    describe('cloud_apm_migration_enabled', () => {
+      it('should be true when when config is set', async () => {
+        const { body } = await bettertest({
+          pathname: '/internal/apm/fleet/migration_check',
+        });
+        expect(body).to.have.property('cloud_apm_migration_enabled', true);
+      });
+    });
+
+    describe('has_cloud_agent_policy', () => {
+      it('should be false when cloud agent policy does not exist', async () => {
+        const { body } = await bettertest({
+          pathname: '/internal/apm/fleet/migration_check',
+        });
+        expect(body).to.have.property('has_cloud_agent_policy', false);
+      });
+      describe('with Cloud agent policy', () => {
+        before(async () => {
+          await createAgentPolicy(bettertest, 'policy-elastic-agent-on-cloud');
+        });
+        after(async () => {
+          await deleteAgentPolicy(bettertest, 'policy-elastic-agent-on-cloud');
+        });
+        it('should be true when cloud agent policy exists', async () => {
+          const { body } = await bettertest({
+            pathname: '/internal/apm/fleet/migration_check',
+          });
+          expect(body).to.have.property('has_cloud_agent_policy', true);
+        });
+      });
+    });
+
+    describe('has_cloud_apm_package_policy', () => {
+      before(async () => {
+        await createAgentPolicy(bettertest, 'policy-elastic-agent-on-cloud');
+      });
+      after(async () => {
+        await deleteAgentPolicy(bettertest, 'policy-elastic-agent-on-cloud');
+      });
+      it('should be false when the Cloud APM package policy does not exist', async () => {
+        const { body } = await bettertest({
+          pathname: '/internal/apm/fleet/migration_check',
+        });
+        expect(body).to.have.property('has_cloud_apm_package_policy', false);
+        expect(body).to.not.have.property('cloud_apm_package_policy');
+        expect(body).to.have.property('has_apm_integrations', false);
+      });
+      describe('with Cloud APM package policy', () => {
+        before(async () => {
+          await createPackagePolicy(bettertest, 'policy-elastic-agent-on-cloud', 'apm');
+        });
+        after(async () => {
+          await deletePackagePolicy(bettertest, 'apm');
+        });
+        it('should be true when the Cloud APM package policy exists', async () => {
+          const { body } = await bettertest({
+            pathname: '/internal/apm/fleet/migration_check',
+          });
+          expect(body).to.have.property('has_cloud_apm_package_policy', true);
+          expect(body).to.have.property('cloud_apm_package_policy');
+          expect(body).to.have.property('has_apm_integrations', true);
+        });
+      });
+    });
+
+    describe('has_apm_integrations', () => {
+      before(async () => {
+        await createAgentPolicy(bettertest, 'test-agent-policy');
+      });
+      after(async () => {
+        await deleteAgentPolicy(bettertest, 'test-agent-policy');
+      });
+      it('should be false when no APM package policies exist', async () => {
+        const { body } = await bettertest({
+          pathname: '/internal/apm/fleet/migration_check',
+        });
+        expect(body).to.have.property('has_apm_integrations', false);
+        expect(body).to.have.property('has_cloud_apm_package_policy', false);
+      });
+      describe('with custom APM package policy', () => {
+        before(async () => {
+          await createPackagePolicy(bettertest, 'test-agent-policy', 'test-apm-package-policy');
+        });
+        after(async () => {
+          await deletePackagePolicy(bettertest, 'test-apm-package-policy');
+        });
+        it('should be true when any APM package policy exists', async () => {
+          const { body } = await bettertest({
+            pathname: '/internal/apm/fleet/migration_check',
+          });
+          expect(body).to.have.property('has_apm_integrations', true);
+          expect(body).to.have.property('has_cloud_apm_package_policy', false);
+        });
+      });
+    });
+
+    describe('has_required_role', () => {
+      it('should be true when user is superuser', async () => {
+        const { body } = await bettertest({
+          pathname: '/internal/apm/fleet/migration_check',
+        });
+        expect(body).to.have.property('has_required_role', true);
+      });
+      it('should be false when user is not superuser', async () => {
+        const { body } = await apmApiClient.manageServiceAccount({
+          endpoint: 'GET /internal/apm/fleet/migration_check',
+        });
+        expect(body).to.have.property('has_required_role', false);
+      });
+    });
+  });
+}