[Profiling-APM] Removing Profiling dependency from APM (#166253)

This PR removes the Profiling dependency from APM, introduced on `8.10`.

- Exposes a new service in profiling-data-access plugin
- Create a new APM API that calls the new service and checks if
Profiling is initialized
- Move Locators from the Profiling plugin to the Observability-shared
plugin
- Move logic to check Profiling status (has_setup/has_data...) from
Profiling server to profiling-data-access plugin
- Create API tests, testing the status services based on different
scenarios:
  - When profiling hasn't been initialized and there's no data
  - When profiling is initialized but has no data
  - When collector integration is not installed
  - When symbolized integration is not installed
  - When APM server integration is not found

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-profiling-utils/common/profiling_status.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+export interface ProfilingStatus {
+  has_setup: boolean;
+  has_data: boolean;
+  pre_8_9_1_data: boolean;
+  unauthorized?: boolean;
+}

packages/kbn-profiling-utils/index.ts

@@ -49,4 +49,5 @@ export type {
   StackTrace,
   StackTraceID,
 } from './common/profiling';
+export type { ProfilingStatus } from './common/profiling_status';
 export type { TopNFunctions } from './common/functions';

x-pack/plugins/apm/kibana.jsonc

@@ -50,7 +50,6 @@
       "usageCollection",
       "customIntegrations", // Move this to requiredPlugins after completely migrating from the Tutorials Home App
       "licenseManagement",
-      "profiling",
       "profilingDataAccess"
     ],
     "requiredBundles": [

x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.test.tsx

@@ -79,7 +79,8 @@ const expectInfraLocatorsToBeCalled = () => {
 describe('TransactionActionMenu component', () => {
   beforeAll(() => {
     jest.spyOn(hooks, 'useFetcher').mockReturnValue({
-      data: [],
+      // return as Profiling had been initialized
+      data: { initialized: true },
       status: hooks.FETCH_STATUS.SUCCESS,
       refetch: jest.fn(),
     });
@@ -253,6 +254,27 @@ describe('TransactionActionMenu component', () => {
     expect(container).toMatchSnapshot();
   });
 
+  describe('Profiling items', () => {
+    it('renders flamegraph item', async () => {
+      const component = await renderTransaction(
+        Transactions.transactionWithHostData
+      );
+      expectTextsInDocument(component, ['Host flamegraph']);
+    });
+    it('renders topN functions item', async () => {
+      const component = await renderTransaction(
+        Transactions.transactionWithHostData
+      );
+      expectTextsInDocument(component, ['Host topN functions']);
+    });
+    it('renders stacktraces item', async () => {
+      const component = await renderTransaction(
+        Transactions.transactionWithHostData
+      );
+      expectTextsInDocument(component, ['Host stacktraces']);
+    });
+  });
+
   describe('Custom links', () => {
     beforeAll(() => {
       // Mocks callApmAPI because it's going to be used to fecth the transaction in the custom links flyout.
@@ -393,3 +415,35 @@ describe('TransactionActionMenu component', () => {
     });
   });
 });
+
+describe('Profiling not initialized', () => {
+  beforeAll(() => {
+    jest.spyOn(hooks, 'useFetcher').mockReturnValue({
+      // return as Profiling had not been initialized
+      data: { initialized: false },
+      status: hooks.FETCH_STATUS.SUCCESS,
+      refetch: jest.fn(),
+    });
+  });
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+  it('does not render flamegraph item', async () => {
+    const component = await renderTransaction(
+      Transactions.transactionWithHostData
+    );
+    expectTextsNotInDocument(component, ['Host flamegraph']);
+  });
+  it('does not render topN functions item', async () => {
+    const component = await renderTransaction(
+      Transactions.transactionWithHostData
+    );
+    expectTextsNotInDocument(component, ['Host topN functions']);
+  });
+  it('does not render stacktraces item', async () => {
+    const component = await renderTransaction(
+      Transactions.transactionWithHostData
+    );
+    expectTextsNotInDocument(component, ['Host stacktraces']);
+  });
+});

x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx

@@ -18,6 +18,7 @@ import { MlLocatorDefinition } from '@kbn/ml-plugin/public';
 import { enableComparisonByDefault } from '@kbn/observability-plugin/public';
 import { sharePluginMock } from '@kbn/share-plugin/public/mocks';
 import type { InfraLocators } from '@kbn/infra-plugin/common/locators';
+import { apmEnableProfilingIntegration } from '@kbn/observability-plugin/common';
 import { ApmPluginContext, ApmPluginContextValue } from './apm_plugin_context';
 import { ConfigSchema } from '../..';
 import { createCallApmApi } from '../../services/rest/create_call_apm_api';
@@ -57,6 +58,7 @@ const mockCore = merge({}, coreStart, {
           value: 100000,
         },
         [enableComparisonByDefault]: true,
+        [apmEnableProfilingIntegration]: true,
       };
       return uiSettings[key];
     },
@@ -108,6 +110,21 @@ const mockPlugin = {
       },
     },
   },
+  observabilityShared: {
+    locators: {
+      profiling: {
+        flamegraphLocator: {
+          getRedirectUrl: () => '/profiling/flamegraphs/flamegraph',
+        },
+        topNFunctionsLocator: {
+          getRedirectUrl: () => '/profiling/functions/topn',
+        },
+        stacktracesLocator: {
+          getRedirectUrl: () => '/profiling/stacktraces/threads',
+        },
+      },
+    },
+  },
 };
 
 export const infraLocatorsMock: InfraLocators = {

x-pack/plugins/apm/public/hooks/use_profiling_plugin.tsx

@@ -5,9 +5,9 @@
  * 2.0.
  */
 
-import { useEffect, useState } from 'react';
 import { apmEnableProfilingIntegration } from '@kbn/observability-plugin/common';
 import { useApmPluginContext } from '../context/apm_plugin/use_apm_plugin_context';
+import { useFetcher } from './use_fetcher';
 
 export function useProfilingPlugin() {
   const { plugins, core } = useApmPluginContext();
@@ -15,30 +15,19 @@ export function useProfilingPlugin() {
     apmEnableProfilingIntegration,
     false
   );
-  const [isProfilingPluginInitialized, setIsProfilingPluginInitialized] =
-    useState<boolean | undefined>();
 
-  useEffect(() => {
+  const { data } = useFetcher((callApmApi) => {
-    async function fetchIsProfilingSetup() {
+    return callApmApi('GET /internal/apm/profiling/status');
-      if (!plugins.profiling) {
+  }, []);
-        setIsProfilingPluginInitialized(false);
-        return;
-      }
-      const resp = await plugins.profiling.hasSetup();
-      setIsProfilingPluginInitialized(resp);
-    }
-
-    fetchIsProfilingSetup();
-  }, [plugins.profiling]);
 
   const isProfilingAvailable =
-    isProfilingIntegrationEnabled && isProfilingPluginInitialized;
+    isProfilingIntegrationEnabled && data?.initialized;
 
   return {
-    isProfilingPluginInitialized,
     profilingLocators: isProfilingAvailable
-      ? plugins.profiling?.locators
+      ? plugins.observabilityShared.locators.profiling
       : undefined,
+    isProfilingPluginInitialized: data?.initialized,
     isProfilingIntegrationEnabled,
     isProfilingAvailable,
   };

x-pack/plugins/apm/server/routes/profiling/route.ts

@@ -131,7 +131,38 @@ const profilingFunctionsRoute = createApmServerRoute({
   },
 });
 
+const profilingStatusRoute = createApmServerRoute({
+  endpoint: 'GET /internal/apm/profiling/status',
+  options: { tags: ['access:apm'] },
+  handler: async (resources): Promise<{ initialized: boolean }> => {
+    const { context, plugins, logger } = resources;
+    const [esClient, profilingDataAccessStart] = await Promise.all([
+      (await context.core).elasticsearch.client,
+      await plugins.profilingDataAccess?.start(),
+    ]);
+    if (profilingDataAccessStart) {
+      try {
+        const response = await profilingDataAccessStart?.services.getStatus({
+          esClient: esClient.asCurrentUser,
+          soClient: (await context.core).savedObjects.client,
+          spaceId: (
+            await plugins.spaces?.start()
+          )?.spacesService.getSpaceId(resources.request),
+        });
+
+        return { initialized: response.has_setup };
+      } catch (e) {
+        // If any error happens just return as if profiling has not been initialized
+        logger.warn('Could not check Universal Profiling status');
+      }
+    }
+
+    return { initialized: false };
+  },
+});
+
 export const profilingRouteRepository = {
   ...profilingFlamegraphRoute,
+  ...profilingStatusRoute,
   ...profilingFunctionsRoute,
 };

x-pack/plugins/observability_shared/kibana.jsonc

@@ -7,7 +7,7 @@
     "server": false,
     "browser": true,
     "configPath": ["xpack", "observability_shared"],
-    "requiredPlugins": ["cases", "guidedOnboarding", "uiActions", "embeddable"],
+    "requiredPlugins": ["cases", "guidedOnboarding", "uiActions", "embeddable", "share"],
     "optionalPlugins": [],
     "requiredBundles": ["data", "inspector", "kibanaReact", "kibanaUtils"],
     "extraPublicDirs": ["common"]

x-pack/plugins/observability_shared/public/plugin.ts

@@ -6,15 +6,22 @@
  */
 
 import { BehaviorSubject } from 'rxjs';
-
+import type { CoreStart, Plugin, CoreSetup } from '@kbn/core/public';
-import type { CoreStart, Plugin } from '@kbn/core/public';
 import type { GuidedOnboardingPluginStart } from '@kbn/guided-onboarding-plugin/public';
 import { CasesUiStart } from '@kbn/cases-plugin/public';
 import { SpacesPluginStart } from '@kbn/spaces-plugin/public';
 import type { EmbeddableStart } from '@kbn/embeddable-plugin/public';
+import type { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public';
 import { createNavigationRegistry } from './components/page_template/helpers/navigation_registry';
 import { createLazyObservabilityPageTemplate } from './components/page_template';
 import { updateGlobalNavigation } from './services/update_global_navigation';
+import { FlamegraphLocatorDefinition } from './locators/profiling/flamegraph_locator';
+import { TopNFunctionsLocatorDefinition } from './locators/profiling/topn_functions_locator';
+import { StacktracesLocatorDefinition } from './locators/profiling/stacktraces_locator';
+
+export interface ObservabilitySharedSetup {
+  share: SharePluginSetup;
+}
 
 export interface ObservabilitySharedStart {
   spaces?: SpacesPluginStart;
@@ -22,6 +29,7 @@ export interface ObservabilitySharedStart {
   guidedOnboarding: GuidedOnboardingPluginStart;
   setIsSidebarEnabled: (isEnabled: boolean) => void;
   embeddable: EmbeddableStart;
+  share: SharePluginStart;
 }
 
 export type ObservabilitySharedPluginSetup = ReturnType<ObservabilitySharedPlugin['setup']>;
@@ -35,8 +43,21 @@ export class ObservabilitySharedPlugin implements Plugin {
     this.isSidebarEnabled$ = new BehaviorSubject<boolean>(true);
   }
 
-  public setup() {
+  public setup(coreSetup: CoreSetup, pluginsSetup: ObservabilitySharedSetup) {
     return {
+      locators: {
+        profiling: {
+          flamegraphLocator: pluginsSetup.share.url.locators.create(
+            new FlamegraphLocatorDefinition()
+          ),
+          topNFunctionsLocator: pluginsSetup.share.url.locators.create(
+            new TopNFunctionsLocatorDefinition()
+          ),
+          stacktracesLocator: pluginsSetup.share.url.locators.create(
+            new StacktracesLocatorDefinition()
+          ),
+        },
+      },
       navigation: {
         registerSections: this.navigationRegistry.registerSections,
       },

x-pack/plugins/observability_shared/tsconfig.json

@@ -33,7 +33,9 @@
     "@kbn/kibana-utils-plugin",
     "@kbn/shared-ux-router",
     "@kbn/embeddable-plugin",
-    "@kbn/profiling-utils"
+    "@kbn/profiling-utils",
+    "@kbn/utility-types",
+    "@kbn/share-plugin"
   ],
   "exclude": ["target/**/*"]
 }

x-pack/plugins/profiling/public/plugin.tsx

@@ -17,13 +17,10 @@ import type { NavigationSection } from '@kbn/observability-shared-plugin/public'
 import type { Location } from 'history';
 import { BehaviorSubject, combineLatest, from, map } from 'rxjs';
 import { registerEmbeddables } from './embeddables/register_embeddables';
-import { FlamegraphLocatorDefinition } from './locators/flamegraph_locator';
-import { StacktracesLocatorDefinition } from './locators/stacktraces_locator';
-import { TopNFunctionsLocatorDefinition } from './locators/topn_functions_locator';
 import { getServices } from './services';
 import type { ProfilingPluginPublicSetupDeps, ProfilingPluginPublicStartDeps } from './types';
 
-export type ProfilingPluginSetup = ReturnType<ProfilingPlugin['setup']>;
+export type ProfilingPluginSetup = void;
 export type ProfilingPluginStart = void;
 
 export class ProfilingPlugin implements Plugin {
@@ -133,33 +130,7 @@ export class ProfilingPlugin implements Plugin {
 
     registerEmbeddables(pluginsSetup.embeddable);
 
-    return {
+    return {};
-      locators: {
-        flamegraphLocator: pluginsSetup.share.url.locators.create(
-          new FlamegraphLocatorDefinition()
-        ),
-        topNFunctionsLocator: pluginsSetup.share.url.locators.create(
-          new TopNFunctionsLocatorDefinition()
-        ),
-        stacktracesLocator: pluginsSetup.share.url.locators.create(
-          new StacktracesLocatorDefinition()
-        ),
-      },
-      hasSetup: async () => {
-        try {
-          const response = (await coreSetup.http.get('/internal/profiling/setup/es_resources')) as {
-            has_setup: boolean;
-            has_data: boolean;
-            unauthorized: boolean;
-          };
-
-          return response.has_setup;
-        } catch (e) {
-          // If any error happens while checking return as it has not been set up
-          return false;
-        }
-      },
-    };
   }
 
   public start(core: CoreStart) {

x-pack/plugins/profiling/server/lib/setup/cluster_settings.ts

@@ -5,22 +5,8 @@
  * 2.0.
  */
 
+import { MAX_BUCKETS } from '@kbn/profiling-data-access-plugin/common';
 import { ProfilingSetupOptions } from './types';
-import { PartialSetupState } from '../../../common/setup';
-
-const MAX_BUCKETS = 150000;
-
-export async function validateMaximumBuckets({
-  client,
-}: ProfilingSetupOptions): Promise<PartialSetupState> {
-  const settings = await client.getEsClient().cluster.getSettings({});
-  const maxBuckets = settings.persistent.search?.max_buckets;
-  return {
-    settings: {
-      configured: maxBuckets === MAX_BUCKETS.toString(),
-    },
-  };
-}
 
 export async function setMaximumBuckets({ client }: ProfilingSetupOptions) {
   await client.getEsClient().cluster.putSettings({
@@ -32,21 +18,6 @@ export async function setMaximumBuckets({ client }: ProfilingSetupOptions) {
   });
 }
 
-export async function validateResourceManagement({
-  client,
-}: ProfilingSetupOptions): Promise<PartialSetupState> {
-  const statusResponse = await client.profilingStatus();
-  return {
-    resource_management: {
-      enabled: statusResponse.resource_management.enabled,
-    },
-    resources: {
-      created: statusResponse.resources.created,
-      pre_8_9_1_data: statusResponse.resources.pre_8_9_1_data,
-    },
-  };
-}
-
 export async function enableResourceManagement({ client }: ProfilingSetupOptions) {
   await client.getEsClient().cluster.putSettings({
     persistent: {

x-pack/plugins/profiling/server/lib/setup/fleet_policies.ts

@@ -5,53 +5,18 @@
  * 2.0.
  */
 
-import { SavedObjectsClientContract } from '@kbn/core/server';
-import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
 import { fetchFindLatestPackageOrThrow } from '@kbn/fleet-plugin/server/services/epm/registry';
+import {
+  COLLECTOR_PACKAGE_POLICY_NAME,
+  ELASTIC_CLOUD_APM_POLICY,
+  SYMBOLIZER_PACKAGE_POLICY_NAME,
+  getApmPolicy,
+} from '@kbn/profiling-data-access-plugin/common';
 import { omit } from 'lodash';
 import { PackageInputType } from '../..';
-import { PartialSetupState } from '../../../common/setup';
-import { ELASTIC_CLOUD_APM_POLICY, getApmPolicy } from './get_apm_policy';
 import { ProfilingSetupOptions } from './types';
 
 const CLOUD_AGENT_POLICY_ID = 'policy-elastic-agent-on-cloud';
-const COLLECTOR_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-collector';
-const SYMBOLIZER_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-symbolizer';
-
-async function getPackagePolicy({
-  soClient,
-  packagePolicyClient,
-  packageName,
-}: {
-  packagePolicyClient: PackagePolicyClient;
-  soClient: SavedObjectsClientContract;
-  packageName: string;
-}) {
-  const packagePolicies = await packagePolicyClient.list(soClient, {});
-  return packagePolicies.items.find((pkg) => pkg.name === packageName);
-}
-
-export async function getCollectorPolicy({
-  soClient,
-  packagePolicyClient,
-}: {
-  packagePolicyClient: PackagePolicyClient;
-  soClient: SavedObjectsClientContract;
-}) {
-  return getPackagePolicy({
-    soClient,
-    packagePolicyClient,
-    packageName: COLLECTOR_PACKAGE_POLICY_NAME,
-  });
-}
-
-export async function validateCollectorPackagePolicy({
-  soClient,
-  packagePolicyClient,
-}: ProfilingSetupOptions): Promise<PartialSetupState> {
-  const collectorPolicy = await getCollectorPolicy({ soClient, packagePolicyClient });
-  return { policies: { collector: { installed: !!collectorPolicy } } };
-}
 
 export function generateSecretToken() {
   let result = '';
@@ -126,28 +91,6 @@ export async function createCollectorPackagePolicy({
   });
 }
 
-export async function getSymbolizerPolicy({
-  soClient,
-  packagePolicyClient,
-}: {
-  packagePolicyClient: PackagePolicyClient;
-  soClient: SavedObjectsClientContract;
-}) {
-  return getPackagePolicy({
-    soClient,
-    packagePolicyClient,
-    packageName: SYMBOLIZER_PACKAGE_POLICY_NAME,
-  });
-}
-
-export async function validateSymbolizerPackagePolicy({
-  soClient,
-  packagePolicyClient,
-}: ProfilingSetupOptions): Promise<PartialSetupState> {
-  const symbolizerPackagePolicy = await getSymbolizerPolicy({ soClient, packagePolicyClient });
-  return { policies: { symbolizer: { installed: !!symbolizerPackagePolicy } } };
-}
-
 export async function createSymbolizerPackagePolicy({
   client,
   soClient,
@@ -185,29 +128,6 @@ export async function createSymbolizerPackagePolicy({
   });
 }
 
-export async function validateProfilingInApmPackagePolicy({
-  soClient,
-  packagePolicyClient,
-}: ProfilingSetupOptions): Promise<PartialSetupState> {
-  try {
-    const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
-    return {
-      policies: {
-        apm: {
-          profilingEnabled: !!(
-            apmPolicy && apmPolicy?.inputs[0].config?.['apm-server'].value?.profiling
-          ),
-        },
-      },
-    };
-  } catch (e) {
-    // In case apm integration is not available ignore the error and return as profiling is not enabled on the integration
-    return {
-      policies: { apm: { profilingEnabled: false } },
-    };
-  }
-}
-
 export async function removeProfilingFromApmPackagePolicy({
   client,
   soClient,

x-pack/plugins/profiling/server/lib/setup/get_setup_instructions.ts

@@ -8,7 +8,7 @@
 import { SavedObjectsClientContract } from '@kbn/core/server';
 import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
 import { fetchFindLatestPackageOrThrow } from '@kbn/fleet-plugin/server/services/epm/registry';
-import { getCollectorPolicy, getSymbolizerPolicy } from './fleet_policies';
+import { getCollectorPolicy, getSymbolizerPolicy } from '@kbn/profiling-data-access-plugin/common';
 
 export interface SetupDataCollectionInstructions {
   collector: {

x-pack/plugins/profiling/server/lib/setup/security_role.ts

@@ -5,24 +5,11 @@
  * 2.0.
  */
 
+import {
+  METADATA_VERSION,
+  PROFILING_READER_ROLE_NAME,
+} from '@kbn/profiling-data-access-plugin/common';
 import { ProfilingSetupOptions } from './types';
-import { PartialSetupState } from '../../../common/setup';
-
-const PROFILING_READER_ROLE_NAME = 'profiling-reader';
-const METADATA_VERSION = 1;
-
-export async function validateSecurityRole({
-  client,
-}: ProfilingSetupOptions): Promise<PartialSetupState> {
-  const esClient = client.getEsClient();
-  const roles = await esClient.security.getRole();
-  const profilingRole = roles[PROFILING_READER_ROLE_NAME];
-  return {
-    permissions: {
-      configured: !!profilingRole && profilingRole.metadata.version === METADATA_VERSION,
-    },
-  };
-}
 
 export async function setSecurityRole({ client }: ProfilingSetupOptions) {
   const esClient = client.getEsClient();

x-pack/plugins/profiling/server/routes/setup.ts

@@ -8,29 +8,14 @@
 import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common';
 import { RouteRegisterParameters } from '.';
 import { getRoutePaths } from '../../common';
-import {
+import { enableResourceManagement, setMaximumBuckets } from '../lib/setup/cluster_settings';
-  areResourcesSetup,
-  createDefaultSetupState,
-  mergePartialSetupStates,
-} from '../../common/setup';
-import {
-  enableResourceManagement,
-  setMaximumBuckets,
-  validateMaximumBuckets,
-  validateResourceManagement,
-} from '../lib/setup/cluster_settings';
 import {
   createCollectorPackagePolicy,
   createSymbolizerPackagePolicy,
   removeProfilingFromApmPackagePolicy,
-  validateCollectorPackagePolicy,
-  validateProfilingInApmPackagePolicy,
-  validateSymbolizerPackagePolicy,
 } from '../lib/setup/fleet_policies';
 import { getSetupInstructions } from '../lib/setup/get_setup_instructions';
-import { hasProfilingData } from '../lib/setup/has_profiling_data';
+import { setSecurityRole } from '../lib/setup/security_role';
-import { setSecurityRole, validateSecurityRole } from '../lib/setup/security_role';
-import { ProfilingSetupOptions } from '../lib/setup/types';
 import { handleRouteHandlerError } from '../utils/handle_route_error_handler';
 import { getClient } from './compat';
 
@@ -52,82 +37,15 @@ export function registerSetupRoute({
       try {
         const esClient = await getClient(context);
         const core = await context.core;
-        const clientWithDefaultAuth = createProfilingEsClient({
-          esClient,
-          request,
-          useDefaultAuth: true,
-        });
-        const clientWithProfilingAuth = createProfilingEsClient({
-          esClient,
-          request,
-          useDefaultAuth: false,
-        });
 
-        const setupOptions: ProfilingSetupOptions = {
+        const profilingStatus = await dependencies.start.profilingDataAccess.services.getStatus({
-          client: clientWithDefaultAuth,
+          esClient,
-          logger,
-          packagePolicyClient: dependencies.start.fleet.packagePolicyService,
           soClient: core.savedObjects.client,
-          spaceId:
+          spaceId: dependencies.setup.spaces?.spacesService?.getSpaceId(request),
-            dependencies.setup.spaces?.spacesService?.getSpaceId(request) ?? DEFAULT_SPACE_ID,
-          isCloudEnabled: dependencies.setup.cloud.isCloudEnabled,
-          config: dependencies.config,
-        };
-
-        const state = createDefaultSetupState();
-        state.cloud.available = dependencies.setup.cloud.isCloudEnabled;
-
-        if (!state.cloud.available) {
-          const msg = `Elastic Cloud is required to set up Elasticsearch and Fleet for Universal Profiling`;
-          logger.error(msg);
-          return response.custom({
-            statusCode: 500,
-            body: {
-              message: msg,
-            },
-          });
-        }
-        const verifyFunctions = [
-          validateMaximumBuckets,
-          validateResourceManagement,
-          validateSecurityRole,
-          validateCollectorPackagePolicy,
-          validateSymbolizerPackagePolicy,
-          validateProfilingInApmPackagePolicy,
-        ];
-
-        const partialStates = await Promise.all([
-          ...verifyFunctions.map((fn) => fn(setupOptions)),
-          hasProfilingData({
-            ...setupOptions,
-            client: clientWithProfilingAuth,
-          }),
-        ]);
-
-        const mergedState = mergePartialSetupStates(state, partialStates);
-
-        return response.ok({
-          body: {
-            has_setup: areResourcesSetup(mergedState),
-            has_data: mergedState.data.available,
-            pre_8_9_1_data: mergedState.resources.pre_8_9_1_data,
-          },
         });
+
+        return response.ok({ body: profilingStatus });
       } catch (error) {
-        // We cannot fully check the status of all resources
-        // to make sure Profiling has been set up and has data
-        // for users with monitor privileges. This privileges
-        // is needed to call the profiling ES plugin for example.
-        if (error?.meta?.statusCode === 403) {
-          return response.ok({
-            body: {
-              has_setup: true,
-              pre_8_9_1_data: false,
-              has_data: true,
-              unauthorized: true,
-            },
-          });
-        }
         return handleRouteHandlerError({
           error,
           logger,
@@ -146,28 +64,9 @@ export function registerSetupRoute({
     },
     async (context, request, response) => {
       try {
-        const esClient = await getClient(context);
+        const isCloudEnabled = dependencies.setup.cloud.isCloudEnabled;
-        const core = await context.core;
-        const clientWithDefaultAuth = createProfilingEsClient({
-          esClient,
-          request,
-          useDefaultAuth: true,
-        });
-        const setupOptions: ProfilingSetupOptions = {
-          client: clientWithDefaultAuth,
-          logger,
-          packagePolicyClient: dependencies.start.fleet.packagePolicyService,
-          soClient: core.savedObjects.client,
-          spaceId:
-            dependencies.setup.spaces?.spacesService?.getSpaceId(request) ?? DEFAULT_SPACE_ID,
-          isCloudEnabled: dependencies.setup.cloud.isCloudEnabled,
-          config: dependencies.config,
-        };
 
-        const state = createDefaultSetupState();
+        if (!isCloudEnabled) {
-        state.cloud.available = dependencies.setup.cloud.isCloudEnabled;
-
-        if (!state.cloud.available) {
           const msg = `Elastic Cloud is required to set up Elasticsearch and Fleet for Universal Profiling`;
           logger.error(msg);
           return response.custom({
@@ -178,28 +77,44 @@ export function registerSetupRoute({
           });
         }
 
-        const partialStates = await Promise.all(
+        const esClient = await getClient(context);
-          [
+        const core = await context.core;
-            validateResourceManagement,
+        const clientWithDefaultAuth = createProfilingEsClient({
-            validateSecurityRole,
+          esClient,
-            validateMaximumBuckets,
+          request,
-            validateCollectorPackagePolicy,
+          useDefaultAuth: true,
-            validateSymbolizerPackagePolicy,
+        });
-            validateProfilingInApmPackagePolicy,
+        const clientWithProfilingAuth = createProfilingEsClient({
-          ].map((fn) => fn(setupOptions))
+          esClient,
+          request,
+          useDefaultAuth: false,
+        });
+
+        const commonParams = {
+          client: clientWithDefaultAuth,
+          logger,
+          packagePolicyClient: dependencies.start.fleet.packagePolicyService,
+          soClient: core.savedObjects.client,
+          spaceId:
+            dependencies.setup.spaces?.spacesService?.getSpaceId(request) ?? DEFAULT_SPACE_ID,
+          isCloudEnabled,
+        };
+
+        const setupState = await dependencies.start.profilingDataAccess.services.getSetupState(
+          commonParams,
+          clientWithProfilingAuth
         );
-        const mergedState = mergePartialSetupStates(state, partialStates);
 
         const executeAdminFunctions = [
-          ...(mergedState.resource_management.enabled ? [] : [enableResourceManagement]),
+          ...(setupState.resource_management.enabled ? [] : [enableResourceManagement]),
-          ...(mergedState.permissions.configured ? [] : [setSecurityRole]),
+          ...(setupState.permissions.configured ? [] : [setSecurityRole]),
-          ...(mergedState.settings.configured ? [] : [setMaximumBuckets]),
+          ...(setupState.settings.configured ? [] : [setMaximumBuckets]),
         ];
 
         const executeViewerFunctions = [
-          ...(mergedState.policies.collector.installed ? [] : [createCollectorPackagePolicy]),
+          ...(setupState.policies.collector.installed ? [] : [createCollectorPackagePolicy]),
-          ...(mergedState.policies.symbolizer.installed ? [] : [createSymbolizerPackagePolicy]),
+          ...(setupState.policies.symbolizer.installed ? [] : [createSymbolizerPackagePolicy]),
-          ...(mergedState.policies.apm.profilingEnabled
+          ...(setupState.policies.apm.profilingEnabled
             ? [removeProfilingFromApmPackagePolicy]
             : []),
         ];
@@ -208,8 +123,12 @@ export function registerSetupRoute({
           return response.ok();
         }
 
-        await Promise.all(executeAdminFunctions.map((fn) => fn(setupOptions)));
+        const setupParams = {
-        await Promise.all(executeViewerFunctions.map((fn) => fn(setupOptions)));
+          ...commonParams,
+          config: dependencies.config,
+        };
+        await Promise.all(executeAdminFunctions.map((fn) => fn(setupParams)));
+        await Promise.all(executeViewerFunctions.map((fn) => fn(setupParams)));
 
         if (dependencies.telemetryUsageCounter) {
           dependencies.telemetryUsageCounter.incrementCounter({

x-pack/plugins/profiling/tsconfig.json

@@ -45,7 +45,6 @@
     "@kbn/share-plugin",
     "@kbn/observability-shared-plugin",
     "@kbn/licensing-plugin",
-    "@kbn/utility-types",
     "@kbn/usage-collection-plugin",
     "@kbn/observability-ai-assistant-plugin",
     "@kbn/profiling-data-access-plugin",

x-pack/plugins/profiling_data_access/common/cluster_settings.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PartialSetupState, ProfilingSetupOptions } from './setup';
+
+export const MAX_BUCKETS = 150000;
+
+export async function validateMaximumBuckets({
+  client,
+}: ProfilingSetupOptions): Promise<PartialSetupState> {
+  const settings = await client.getEsClient().cluster.getSettings({});
+  const maxBuckets = settings.persistent.search?.max_buckets;
+  return {
+    settings: {
+      configured: maxBuckets === MAX_BUCKETS.toString(),
+    },
+  };
+}
+
+export async function validateResourceManagement({
+  client,
+}: ProfilingSetupOptions): Promise<PartialSetupState> {
+  const statusResponse = await client.profilingStatus();
+  return {
+    resource_management: {
+      enabled: statusResponse.resource_management.enabled,
+    },
+    resources: {
+      created: statusResponse.resources.created,
+      pre_8_9_1_data: statusResponse.resources.pre_8_9_1_data,
+    },
+  };
+}

x-pack/plugins/profiling_data_access/common/fleet_policies.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsClientContract } from '@kbn/core/server';
+import type { PackagePolicyClient } from '@kbn/fleet-plugin/server';
+import { getApmPolicy } from './get_apm_policy';
+import { PartialSetupState, ProfilingSetupOptions } from './setup';
+
+export const COLLECTOR_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-collector';
+export const SYMBOLIZER_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-symbolizer';
+
+async function getPackagePolicy({
+  soClient,
+  packagePolicyClient,
+  packageName,
+}: {
+  packagePolicyClient: PackagePolicyClient;
+  soClient: SavedObjectsClientContract;
+  packageName: string;
+}) {
+  const packagePolicies = await packagePolicyClient.list(soClient, {});
+  return packagePolicies.items.find((pkg) => pkg.name === packageName);
+}
+
+export async function getCollectorPolicy({
+  soClient,
+  packagePolicyClient,
+}: {
+  packagePolicyClient: PackagePolicyClient;
+  soClient: SavedObjectsClientContract;
+}) {
+  return getPackagePolicy({
+    soClient,
+    packagePolicyClient,
+    packageName: COLLECTOR_PACKAGE_POLICY_NAME,
+  });
+}
+
+export async function validateCollectorPackagePolicy({
+  soClient,
+  packagePolicyClient,
+}: ProfilingSetupOptions): Promise<PartialSetupState> {
+  const collectorPolicy = await getCollectorPolicy({ soClient, packagePolicyClient });
+  return { policies: { collector: { installed: !!collectorPolicy } } };
+}
+
+export function generateSecretToken() {
+  let result = '';
+  const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  for (let i = 0; i < 16; i++) {
+    const randomIndex = Math.floor(Math.random() * characters.length);
+    result += characters.charAt(randomIndex);
+  }
+
+  return result;
+}
+
+export async function getSymbolizerPolicy({
+  soClient,
+  packagePolicyClient,
+}: {
+  packagePolicyClient: PackagePolicyClient;
+  soClient: SavedObjectsClientContract;
+}) {
+  return getPackagePolicy({
+    soClient,
+    packagePolicyClient,
+    packageName: SYMBOLIZER_PACKAGE_POLICY_NAME,
+  });
+}
+
+export async function validateSymbolizerPackagePolicy({
+  soClient,
+  packagePolicyClient,
+}: ProfilingSetupOptions): Promise<PartialSetupState> {
+  const symbolizerPackagePolicy = await getSymbolizerPolicy({ soClient, packagePolicyClient });
+  return { policies: { symbolizer: { installed: !!symbolizerPackagePolicy } } };
+}
+
+export async function validateProfilingInApmPackagePolicy({
+  soClient,
+  packagePolicyClient,
+}: ProfilingSetupOptions): Promise<PartialSetupState> {
+  try {
+    const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
+    return {
+      policies: {
+        apm: {
+          profilingEnabled: !!(
+            apmPolicy && apmPolicy?.inputs[0].config?.['apm-server'].value?.profiling
+          ),
+        },
+      },
+    };
+  } catch (e) {
+    // In case apm integration is not available ignore the error and return as profiling is not enabled on the integration
+    return {
+      policies: { apm: { profilingEnabled: false } },
+    };
+  }
+}

x-pack/plugins/profiling_data_access/common/get_apm_policy.ts

@@ -6,7 +6,7 @@
  */
 
 import { SavedObjectsClientContract } from '@kbn/core/server';
-import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
+import type { PackagePolicyClient } from '@kbn/fleet-plugin/server';
 
 export const ELASTIC_CLOUD_APM_POLICY = 'elastic-cloud-apm';
 

x-pack/plugins/profiling_data_access/common/has_profiling_data.ts

@@ -5,8 +5,7 @@
  * 2.0.
  */
 
-import { PartialSetupState } from '../../../common/setup';
+import { PartialSetupState, ProfilingSetupOptions } from './setup';
-import { ProfilingSetupOptions } from './types';
 
 export async function hasProfilingData({
   client,

x-pack/plugins/profiling_data_access/common/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { getApmPolicy, ELASTIC_CLOUD_APM_POLICY } from './get_apm_policy';
+export { MAX_BUCKETS } from './cluster_settings';
+export { METADATA_VERSION, PROFILING_READER_ROLE_NAME } from './security_role';
+export {
+  getCollectorPolicy,
+  getSymbolizerPolicy,
+  COLLECTOR_PACKAGE_POLICY_NAME,
+  SYMBOLIZER_PACKAGE_POLICY_NAME,
+} from './fleet_policies';

x-pack/plugins/profiling_data_access/common/profiling_es_client.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { ElasticsearchClient } from '@kbn/core/server';
+import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
+import type { ProfilingStatusResponse, StackTraceResponse } from '@kbn/profiling-utils';
+
+export interface ProfilingESClient {
+  search<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
+    operationName: string,
+    searchRequest: TSearchRequest
+  ): Promise<InferSearchResponseOf<TDocument, TSearchRequest>>;
+  profilingStacktraces({}: {
+    query: QueryDslQueryContainer;
+    sampleSize: number;
+  }): Promise<StackTraceResponse>;
+  profilingStatus(): Promise<ProfilingStatusResponse>;
+  getEsClient(): ElasticsearchClient;
+}

x-pack/plugins/profiling_data_access/common/security_role.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PartialSetupState, ProfilingSetupOptions } from './setup';
+
+export const PROFILING_READER_ROLE_NAME = 'profiling-reader';
+export const METADATA_VERSION = 1;
+
+export async function validateSecurityRole({
+  client,
+}: ProfilingSetupOptions): Promise<PartialSetupState> {
+  const esClient = client.getEsClient();
+  const roles = await esClient.security.getRole();
+  const profilingRole = roles[PROFILING_READER_ROLE_NAME];
+  return {
+    permissions: {
+      configured: !!profilingRole && profilingRole.metadata.version === METADATA_VERSION,
+    },
+  };
+}

x-pack/plugins/profiling_data_access/common/setup.ts

@@ -4,9 +4,20 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
-import { merge } from 'lodash';
 import type { RecursivePartial } from '@elastic/eui';
+import { Logger, SavedObjectsClientContract } from '@kbn/core/server';
+import type { PackagePolicyClient } from '@kbn/fleet-plugin/server';
+import { merge } from 'lodash';
+import { ProfilingESClient } from './profiling_es_client';
+
+export interface ProfilingSetupOptions {
+  client: ProfilingESClient;
+  soClient: SavedObjectsClientContract;
+  packagePolicyClient: PackagePolicyClient;
+  logger: Logger;
+  spaceId: string;
+  isCloudEnabled: boolean;
+}
 
 export interface SetupState {
   cloud: {

x-pack/plugins/profiling_data_access/kibana.jsonc

@@ -7,7 +7,11 @@
     "server": true,
     "browser": false,
     "configPath": ["xpack", "profiling"],
-    "requiredPlugins": ["data"],
+    "requiredPlugins": [
+      "data", 
+      "fleet",
+      "cloud"
+    ],
     "optionalPlugins": [],
     "requiredBundles": []
   }

x-pack/plugins/profiling_data_access/server/plugin.ts

@@ -5,19 +5,30 @@
  * 2.0.
  */
 
-import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server';
+import type {
+  CoreSetup,
+  CoreStart,
+  Logger,
+  Plugin,
+  PluginInitializerContext,
+} from '@kbn/core/server';
 import { ProfilingConfig } from '.';
 import { registerServices } from './services/register_services';
 import { createProfilingEsClient } from './utils/create_profiling_es_client';
+import { ProfilingPluginStartDeps } from './types';
 
 export type ProfilingDataAccessPluginSetup = ReturnType<ProfilingDataAccessPlugin['setup']>;
 export type ProfilingDataAccessPluginStart = ReturnType<ProfilingDataAccessPlugin['start']>;
 
 export class ProfilingDataAccessPlugin implements Plugin {
-  constructor(private readonly initializerContext: PluginInitializerContext<ProfilingConfig>) {}
+  private readonly logger: Logger;
+
+  constructor(private readonly initializerContext: PluginInitializerContext<ProfilingConfig>) {
+    this.logger = initializerContext.logger.get();
+  }
   public setup(core: CoreSetup) {}
 
-  public start(core: CoreStart) {
+  public start(core: CoreStart, plugins: ProfilingPluginStartDeps) {
     const config = this.initializerContext.config.get();
 
     const profilingSpecificEsClient = config.elasticsearch
@@ -37,6 +48,11 @@ export class ProfilingDataAccessPlugin implements Plugin {
 
         return createProfilingEsClient({ esClient });
       },
+      logger: this.logger,
+      deps: {
+        fleet: plugins.fleet,
+        cloud: plugins.cloud,
+      },
     });
 
     // called after all plugins are set up

x-pack/plugins/profiling_data_access/server/services/get_setup_state/index.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  validateMaximumBuckets,
+  validateResourceManagement,
+} from '../../../common/cluster_settings';
+import {
+  validateCollectorPackagePolicy,
+  validateProfilingInApmPackagePolicy,
+  validateSymbolizerPackagePolicy,
+} from '../../../common/fleet_policies';
+import { hasProfilingData } from '../../../common/has_profiling_data';
+import { ProfilingESClient } from '../../../common/profiling_es_client';
+import { validateSecurityRole } from '../../../common/security_role';
+import {
+  ProfilingSetupOptions,
+  createDefaultSetupState,
+  mergePartialSetupStates,
+} from '../../../common/setup';
+import { RegisterServicesParams } from '../register_services';
+
+export async function getSetupState(
+  options: ProfilingSetupOptions,
+  clientWithProfilingAuth: ProfilingESClient
+) {
+  const state = createDefaultSetupState();
+  state.cloud.available = options.isCloudEnabled;
+
+  const verifyFunctions = [
+    validateMaximumBuckets,
+    validateResourceManagement,
+    validateSecurityRole,
+    validateCollectorPackagePolicy,
+    validateSymbolizerPackagePolicy,
+    validateProfilingInApmPackagePolicy,
+  ];
+
+  const partialStates = await Promise.all([
+    ...verifyFunctions.map((fn) => fn(options)),
+    hasProfilingData({
+      ...options,
+      client: clientWithProfilingAuth,
+    }),
+  ]);
+
+  return mergePartialSetupStates(state, partialStates);
+}
+
+export function createGetSetupState(params: RegisterServicesParams) {
+  return getSetupState;
+}

x-pack/plugins/profiling_data_access/server/services/register_services.ts

@@ -5,9 +5,13 @@
  * 2.0.
  */
 
-import { ElasticsearchClient } from '@kbn/core/server';
+import { CloudStart } from '@kbn/cloud-plugin/server';
+import { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { FleetStartContract } from '@kbn/fleet-plugin/server';
 import { createFetchFlamechart } from './fetch_flamechart';
-import { ProfilingESClient } from '../utils/create_profiling_es_client';
+import { createGetStatusService } from './status';
+import { createGetSetupState } from './get_setup_state';
+import { ProfilingESClient } from '../../common/profiling_es_client';
 import { createFetchFunctions } from './functions';
 
 export interface RegisterServicesParams {
@@ -15,11 +19,18 @@ export interface RegisterServicesParams {
     esClient: ElasticsearchClient;
     useDefaultAuth?: boolean;
   }) => ProfilingESClient;
+  logger: Logger;
+  deps: {
+    fleet: FleetStartContract;
+    cloud: CloudStart;
+  };
 }
 
 export function registerServices(params: RegisterServicesParams) {
   return {
     fetchFlamechartData: createFetchFlamechart(params),
+    getStatus: createGetStatusService(params),
+    getSetupState: createGetSetupState(params),
     fetchFunction: createFetchFunctions(params),
   };
 }

x-pack/plugins/profiling_data_access/server/services/search_stack_traces/index.ts

@@ -6,7 +6,7 @@
  */
 
 import { decodeStackTraceResponse } from '@kbn/profiling-utils';
-import { ProfilingESClient } from '../../utils/create_profiling_es_client';
+import { ProfilingESClient } from '../../../common/profiling_es_client';
 import { kqlQuery } from '../../utils/query';
 
 export async function searchStackTraces({

x-pack/plugins/profiling_data_access/server/services/status/index.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
+import { ProfilingStatus } from '@kbn/profiling-utils';
+import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common';
+import { getSetupState } from '../get_setup_state';
+import { RegisterServicesParams } from '../register_services';
+import { ProfilingSetupOptions, areResourcesSetup } from '../../../common/setup';
+
+interface HasSetupParams {
+  soClient: SavedObjectsClientContract;
+  esClient: ElasticsearchClient;
+  spaceId?: string;
+}
+
+export function createGetStatusService({
+  createProfilingEsClient,
+  deps,
+  logger,
+}: RegisterServicesParams) {
+  return async ({ esClient, soClient, spaceId }: HasSetupParams): Promise<ProfilingStatus> => {
+    try {
+      const isCloudEnabled = deps.cloud.isCloudEnabled;
+      if (!isCloudEnabled) {
+        // When not on cloud just return that is has not set up and has no data
+        return {
+          has_setup: false,
+          has_data: false,
+          pre_8_9_1_data: false,
+        };
+      }
+
+      const clientWithDefaultAuth = createProfilingEsClient({
+        esClient,
+        useDefaultAuth: true,
+      });
+      const clientWithProfilingAuth = createProfilingEsClient({
+        esClient,
+        useDefaultAuth: false,
+      });
+
+      const setupOptions: ProfilingSetupOptions = {
+        client: clientWithDefaultAuth,
+        logger,
+        packagePolicyClient: deps.fleet.packagePolicyService,
+        soClient,
+        spaceId: spaceId ?? DEFAULT_SPACE_ID,
+        isCloudEnabled,
+      };
+
+      const setupState = await getSetupState(setupOptions, clientWithProfilingAuth);
+
+      return {
+        has_setup: areResourcesSetup(setupState),
+        has_data: setupState.data.available,
+        pre_8_9_1_data: setupState.resources.pre_8_9_1_data,
+      };
+    } catch (error) {
+      // We cannot fully check the status of all resources
+      // to make sure Profiling has been set up and has data
+      // for users with monitor privileges. This privileges
+      // is needed to call the profiling ES plugin for example.
+      if (error?.meta?.statusCode === 403 || error?.originalError?.meta?.statusCode === 403) {
+        return {
+          has_setup: true,
+          pre_8_9_1_data: false,
+          has_data: true,
+          unauthorized: true,
+        };
+      }
+
+      throw error;
+    }
+  };
+}

x-pack/plugins/profiling_data_access/server/types.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { CloudStart } from '@kbn/cloud-plugin/server';
+import { FleetStartContract } from '@kbn/fleet-plugin/server';
+
+export interface ProfilingPluginStartDeps {
+  fleet: FleetStartContract;
+  cloud: CloudStart;
+}

x-pack/plugins/profiling_data_access/server/utils/create_profiling_es_client.ts

@@ -5,26 +5,13 @@
  * 2.0.
  */
 
-import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { ElasticsearchClient } from '@kbn/core/server';
 import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import type { ProfilingStatusResponse, StackTraceResponse } from '@kbn/profiling-utils';
+import { ProfilingESClient } from '../../common/profiling_es_client';
 import { unwrapEsResponse } from './unwrap_es_response';
 import { withProfilingSpan } from './with_profiling_span';
 
-export interface ProfilingESClient {
-  search<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
-    operationName: string,
-    searchRequest: TSearchRequest
-  ): Promise<InferSearchResponseOf<TDocument, TSearchRequest>>;
-  profilingStacktraces({}: {
-    query: QueryDslQueryContainer;
-    sampleSize: number;
-  }): Promise<StackTraceResponse>;
-  profilingStatus(): Promise<ProfilingStatusResponse>;
-  getEsClient(): ElasticsearchClient;
-}
-
 export function createProfilingEsClient({
   esClient,
 }: {

x-pack/plugins/profiling_data_access/tsconfig.json

@@ -4,6 +4,7 @@
     "outDir": "target/types"
   },
   "include": [
+    "common/**/*",
     "server/**/*",
     "jest.config.js"
   ],
@@ -16,6 +17,9 @@
     "@kbn/es-query",
     "@kbn/es-types",
     "@kbn/apm-utils",
-    "@kbn/profiling-utils"
+    "@kbn/profiling-utils",
+    "@kbn/fleet-plugin",
+    "@kbn/cloud-plugin",
+    "@kbn/spaces-plugin",
   ]
 }

x-pack/test/profiling_api_integration/common/bettertest.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { format } from 'url';
+import supertest from 'supertest';
+import request from 'superagent';
+
+type HttpMethod = 'get' | 'post' | 'put' | 'delete';
+
+export type BetterTest = <T extends any>(options: {
+  pathname: string;
+  query?: Record<string, any>;
+  method?: HttpMethod;
+  body?: any;
+}) => Promise<{ status: number; body: T }>;
+
+/*
+ * This is a wrapper around supertest that throws an error if the response status is not 200.
+ * This is useful for tests that expect a 200 response
+ * It also makes it easier to debug tests that fail because of a 500 response.
+ */
+export function getBettertest(st: supertest.SuperTest<supertest.Test>): BetterTest {
+  return async ({ pathname, method = 'get', query, body }) => {
+    const url = format({ pathname, query });
+
+    let res: request.Response;
+    if (body) {
+      res = await st[method](url).send(body).set('kbn-xsrf', 'true');
+    } else {
+      res = await st[method](url).set('kbn-xsrf', 'true');
+    }
+
+    // supertest doesn't throw on http errors
+    if (res?.status !== 200 && res?.status !== 202) {
+      throw new BetterTestError(res);
+    }
+
+    return res;
+  };
+}
+
+type ErrorResponse = Omit<request.Response, 'body'> & {
+  body: {
+    statusCode: number;
+    error: string;
+    message: string;
+    attributes: object;
+  };
+};
+
+export class BetterTestError extends Error {
+  res: ErrorResponse;
+
+  constructor(res: request.Response) {
+    // @ts-expect-error
+    const req = res.req as any;
+    super(
+      `Unhandled BetterTestError:
+Status: "${res.status}"      
+Path: "${req.method} ${req.path}"
+Body: ${JSON.stringify(res.body)}`
+    );
+
+    this.res = res;
+  }
+}

x-pack/test/profiling_api_integration/common/config.ts

@@ -5,26 +5,24 @@
  * 2.0.
  */
 
-import { format, UrlObject } from 'url';
 import { FtrConfigProviderContext } from '@kbn/test';
 import supertest from 'supertest';
-import { getRoutePaths } from '@kbn/profiling-plugin/common';
+import { format, UrlObject } from 'url';
 import { ProfilingFtrConfigName } from '../configs';
+import { createProfilingApiClient } from './api_supertest';
+import { createProfilingUsers } from './create_profiling_users';
+import {
+  PROFILING_TEST_PASSWORD,
+  ProfilingUsername,
+} from './create_profiling_users/authentication';
 import {
   FtrProviderContext,
   InheritedFtrProviderContext,
   InheritedServices,
 } from './ftr_provider_context';
 import { RegistryProvider } from './registry';
-import { createProfilingApiClient } from './api_supertest';
-import {
-  ProfilingUsername,
-  PROFILING_TEST_PASSWORD,
-} from './create_profiling_users/authentication';
-import { createProfilingUsers } from './create_profiling_users';
 
 export type CreateTestConfig = ReturnType<typeof createTestConfig>;
-const profilingRoutePaths = getRoutePaths();
 
 export async function getProfilingApiClient({
   kibanaServer,
@@ -112,19 +110,6 @@ export function createTestConfig(
 
           await supertest(kibanaServerUrl).post('/api/fleet/setup').set('kbn-xsrf', 'foo');
 
-          const result = await adminUser({
-            endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
-          });
-          if (!result.body.has_setup) {
-            // eslint-disable-next-line no-console
-            console.log('Setting up Universal Profiling');
-            await adminUser({
-              endpoint: `POST ${profilingRoutePaths.HasSetupESResources}`,
-            });
-            // eslint-disable-next-line no-console
-            console.log('Universal Profiling set up');
-          }
-
           return {
             noAccessUser: await getProfilingApiClient({
               kibanaServer,

x-pack/test/profiling_api_integration/common/registry.ts

@@ -9,12 +9,10 @@ import { joinByKey } from '@kbn/apm-plugin/common/utils/join_by_key';
 import { maybe } from '@kbn/apm-plugin/common/utils/maybe';
 import callsites from 'callsites';
 import { castArray, groupBy } from 'lodash';
-import Path from 'path';
-import fs from 'fs';
 import { ProfilingFtrConfigName } from '../configs';
+import { getBettertest } from './bettertest';
 import { FtrProviderContext } from './ftr_provider_context';
-
+import { cleanUpProfilingData } from '../utils/profiling_data';
-const esArchiversPath = Path.posix.join(__dirname, 'fixtures', 'es_archiver', 'profiling');
 
 interface RunCondition {
   config: ProfilingFtrConfigName;
@@ -22,6 +20,9 @@ interface RunCondition {
 
 export function RegistryProvider({ getService }: FtrProviderContext) {
   const profilingFtrConfig = getService('profilingFtrConfig');
+  const supertest = getService('supertest');
+  const bettertest = getBettertest(supertest);
+
   const es = getService('es');
 
   const callbacks: Array<
@@ -97,16 +98,6 @@ export function RegistryProvider({ getService }: FtrProviderContext) {
 
       const logger = getService('log');
 
-      const logWithTimer = () => {
-        const start = process.hrtime();
-
-        return (message: string) => {
-          const diff = process.hrtime(start);
-          const time = `${Math.round(diff[0] * 1000 + diff[1] / 1e6)}ms`;
-          logger.info(`(${time}) ${message}`);
-        };
-      };
-
       const groups = joinByKey(callbacks, ['config'], (a, b) => ({
         ...a,
         ...b,
@@ -126,36 +117,11 @@ export function RegistryProvider({ getService }: FtrProviderContext) {
           groupsForConfig.forEach((group) => {
             const { runs } = group;
 
-            const runBefore = async () => {
-              const log = logWithTimer();
-              const content = fs.readFileSync(`${esArchiversPath}/data.json`, 'utf8');
-              log(`Loading profiling data`);
-              await es.bulk({ operations: content.split('\n'), refresh: 'wait_for' });
-              log('Loaded profiling data');
-            };
-
             const runAfter = async () => {
-              const log = logWithTimer();
+              await cleanUpProfilingData({ es, bettertest, logger });
-              log(`Unloading Profiling data`);
-              const indices = await es.cat.indices({ format: 'json' });
-              const profilingIndices = indices
-                .filter((index) => index.index !== undefined)
-                .map((index) => index.index)
-                .filter((index) => {
-                  return index!.startsWith('profiling') || index!.startsWith('.profiling');
-                }) as string[];
-              await Promise.all([
-                ...profilingIndices.map((index) => es.indices.delete({ index })),
-                es.indices.deleteDataStream({
-                  name: 'profiling-events*',
-                }),
-              ]);
-              log('Unloaded Profiling data');
             };
 
             describe('Loading profiling data', () => {
-              before(runBefore);
-
               runs.forEach((run) => {
                 run.cb();
               });

x-pack/test/profiling_api_integration/tests/__snapshots__/functions.spec.snap

@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`Profiling API tests functions.spec.ts cloud Loading profiling data Functions api returns correct result 1`] = `
+exports[`Profiling API tests functions.spec.ts cloud Loading profiling data Functions api With data returns correct result 1`] = `
 Object {
   "SamplingRate": 1,
   "TopN": Array [

x-pack/test/profiling_api_integration/tests/feature_controls.spec.ts

@@ -10,6 +10,8 @@ import { getRoutePaths } from '@kbn/profiling-plugin/common';
 import { ProfilingApiError } from '../common/api_supertest';
 import { getProfilingApiClient } from '../common/config';
 import { FtrProviderContext } from '../common/ftr_provider_context';
+import { setupProfiling } from '../utils/profiling_data';
+import { getBettertest } from '../common/bettertest';
 
 const profilingRoutePaths = getRoutePaths();
 
@@ -17,7 +19,8 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
   const registry = getService('registry');
   const profilingApiClient = getService('profilingApiClient');
   const log = getService('log');
-
+  const supertest = getService('supertest');
+  const bettertest = getBettertest(supertest);
   const start = encodeURIComponent(new Date(Date.now() - 10000).valueOf());
   const end = encodeURIComponent(new Date().valueOf());
 
@@ -111,7 +114,9 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
   }
 
   registry.when('Profiling feature controls', { config: 'cloud' }, () => {
-    before(async () => {});
+    before(async () => {
+      await setupProfiling(bettertest, log);
+    });
     it(`returns forbidden for users with no access to profiling APIs`, async () => {
       await executeRequests({
         runAsUser: profilingApiClient.noAccessUser,

x-pack/test/profiling_api_integration/tests/functions.spec.ts

@@ -9,39 +9,55 @@ import { getRoutePaths } from '@kbn/profiling-plugin/common';
 import { TopNFunctions } from '@kbn/profiling-utils';
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../common/ftr_provider_context';
+import { loadProfilingData, setupProfiling } from '../utils/profiling_data';
+import { getBettertest } from '../common/bettertest';
 
 const profilingRoutePaths = getRoutePaths();
 
 export default function featureControlsTests({ getService }: FtrProviderContext) {
   const registry = getService('registry');
   const profilingApiClient = getService('profilingApiClient');
+  const log = getService('log');
+  const supertest = getService('supertest');
+  const bettertest = getBettertest(supertest);
+  const es = getService('es');
 
   const start = new Date('2023-03-17T01:00:00.000Z').getTime();
   const end = new Date('2023-03-17T01:05:00.000Z').getTime();
 
   registry.when('Functions api', { config: 'cloud' }, () => {
-    let functions: TopNFunctions;
     before(async () => {
-      const response = await profilingApiClient.adminUser({
+      await setupProfiling(bettertest, log);
-        endpoint: `GET ${profilingRoutePaths.TopNFunctions}`,
+      await loadProfilingData(es, log);
-        params: {
-          query: {
-            timeFrom: start,
-            timeTo: end,
-            kuery: '',
-            startIndex: 0,
-            endIndex: 5,
-          },
-        },
-      });
-      functions = response.body as TopNFunctions;
     });
-    it(`returns correct result`, async () => {
+
-      expect(functions.TopN.length).to.equal(5);
+    describe('With data', () => {
-      expect(functions.TotalCount).to.equal(3599);
+      let functions: TopNFunctions;
-      expect(functions.selfCPU).to.equal(397);
+      before(async () => {
-      expect(functions.totalCPU).to.equal(399);
+        await setupProfiling(bettertest, log);
-      expectSnapshot(functions).toMatch();
+        await loadProfilingData(es, log);
+        const response = await profilingApiClient.adminUser({
+          endpoint: `GET ${profilingRoutePaths.TopNFunctions}`,
+          params: {
+            query: {
+              timeFrom: start,
+              timeTo: end,
+              kuery: '',
+              startIndex: 0,
+              endIndex: 5,
+            },
+          },
+        });
+        functions = response.body as TopNFunctions;
+      });
+
+      it(`returns correct result`, async () => {
+        expect(functions.TopN.length).to.equal(5);
+        expect(functions.TotalCount).to.equal(3599);
+        expect(functions.selfCPU).to.equal(397);
+        expect(functions.totalCPU).to.equal(399);
+        expectSnapshot(functions).toMatch();
+      });
     });
   });
 }

x-pack/test/profiling_api_integration/tests/has_setup.spec.ts

@@ -0,0 +1,357 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { getRoutePaths } from '@kbn/profiling-plugin/common';
+import { ProfilingStatus } from '@kbn/profiling-utils';
+import { getBettertest } from '../common/bettertest';
+import { FtrProviderContext } from '../common/ftr_provider_context';
+import { deletePackagePolicy, getProfilingPackagePolicyIds } from '../utils/fleet';
+import { loadProfilingData, setupProfiling } from '../utils/profiling_data';
+
+const profilingRoutePaths = getRoutePaths();
+
+export default function featureControlsTests({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const profilingApiClient = getService('profilingApiClient');
+  const supertest = getService('supertest');
+  const bettertest = getBettertest(supertest);
+  const logger = getService('log');
+  const es = getService('es');
+
+  registry.when('Profiling status check', { config: 'cloud' }, () => {
+    describe('Profiling is not set up and no data is loaded', () => {
+      describe('Admin user', () => {
+        let statusCheck: ProfilingStatus;
+        before(async () => {
+          const response = await profilingApiClient.adminUser({
+            endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+          });
+          statusCheck = response.body;
+        });
+        it(`has not been set up`, async () => {
+          expect(statusCheck.has_setup).to.be(false);
+        });
+
+        it(`does not have data`, async () => {
+          expect(statusCheck.has_data).to.be(false);
+        });
+
+        it(`does not have pre 8.9.1 data`, async () => {
+          expect(statusCheck.pre_8_9_1_data).to.be(false);
+        });
+      });
+
+      describe('Viewer user', () => {
+        let statusCheck: ProfilingStatus;
+        before(async () => {
+          const response = await profilingApiClient.readUser({
+            endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+          });
+          statusCheck = response.body;
+        });
+        it(`has been set up`, async () => {
+          expect(statusCheck.has_setup).to.be(true);
+        });
+
+        it(`has data`, async () => {
+          expect(statusCheck.has_data).to.be(true);
+        });
+
+        it(`does not have pre 8.9.1 data`, async () => {
+          expect(statusCheck.pre_8_9_1_data).to.be(false);
+        });
+
+        it(`is unauthorized to fully check profiling status `, async () => {
+          expect(statusCheck.unauthorized).to.be(true);
+        });
+      });
+    });
+
+    describe('Collector integration is not installed', () => {
+      let collectorId: string | undefined;
+      before(async () => {
+        await setupProfiling(bettertest, logger);
+        const response = await getProfilingPackagePolicyIds(bettertest);
+        collectorId = response.collectorId;
+        if (collectorId) {
+          await deletePackagePolicy(bettertest, collectorId);
+        }
+      });
+
+      it('expectes a collector integration to exist', () => {
+        expect(collectorId).not.to.be(undefined);
+      });
+
+      describe('Admin user', () => {
+        let statusCheck: ProfilingStatus;
+        before(async () => {
+          const response = await profilingApiClient.adminUser({
+            endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+          });
+          statusCheck = response.body;
+        });
+        it(`has not been set up`, async () => {
+          expect(statusCheck.has_setup).to.be(false);
+        });
+
+        it(`does not have data`, async () => {
+          expect(statusCheck.has_data).to.be(false);
+        });
+
+        it(`does not have pre 8.9.1 data`, async () => {
+          expect(statusCheck.pre_8_9_1_data).to.be(false);
+        });
+      });
+
+      describe('Viewer user', () => {
+        let statusCheck: ProfilingStatus;
+        before(async () => {
+          const response = await profilingApiClient.readUser({
+            endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+          });
+          statusCheck = response.body;
+        });
+        it(`has been set up`, async () => {
+          expect(statusCheck.has_setup).to.be(true);
+        });
+
+        it(`has data`, async () => {
+          expect(statusCheck.has_data).to.be(true);
+        });
+
+        it(`does not have pre 8.9.1 data`, async () => {
+          expect(statusCheck.pre_8_9_1_data).to.be(false);
+        });
+
+        it(`is unauthorized to fully check profiling status `, async () => {
+          expect(statusCheck.unauthorized).to.be(true);
+        });
+      });
+    });
+
+    describe('Symbolizer integration is not installed', () => {
+      let symbolizerId: string | undefined;
+      before(async () => {
+        await setupProfiling(bettertest, logger);
+        const response = await getProfilingPackagePolicyIds(bettertest);
+        symbolizerId = response.symbolizerId;
+        if (symbolizerId) {
+          await deletePackagePolicy(bettertest, symbolizerId);
+        }
+      });
+
+      it('expectes a symbolizer integration to exist', () => {
+        expect(symbolizerId).not.to.be(undefined);
+      });
+
+      describe('Admin user', () => {
+        let statusCheck: ProfilingStatus;
+        before(async () => {
+          const response = await profilingApiClient.adminUser({
+            endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+          });
+          statusCheck = response.body;
+        });
+        it(`has not been set up`, async () => {
+          expect(statusCheck.has_setup).to.be(false);
+        });
+
+        it(`does not have data`, async () => {
+          expect(statusCheck.has_data).to.be(false);
+        });
+
+        it(`does not have pre 8.9.1 data`, async () => {
+          expect(statusCheck.pre_8_9_1_data).to.be(false);
+        });
+      });
+
+      describe('Viewer user', () => {
+        let statusCheck: ProfilingStatus;
+        before(async () => {
+          const response = await profilingApiClient.readUser({
+            endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+          });
+          statusCheck = response.body;
+        });
+        it(`has been set up`, async () => {
+          expect(statusCheck.has_setup).to.be(true);
+        });
+
+        it(`has data`, async () => {
+          expect(statusCheck.has_data).to.be(true);
+        });
+
+        it(`does not have pre 8.9.1 data`, async () => {
+          expect(statusCheck.pre_8_9_1_data).to.be(false);
+        });
+
+        it(`is unauthorized to fully check profiling status `, async () => {
+          expect(statusCheck.unauthorized).to.be(true);
+        });
+      });
+    });
+
+    describe('APM integration is not installed', () => {
+      before(async () => {
+        await setupProfiling(bettertest, logger);
+        await deletePackagePolicy(bettertest, 'elastic-cloud-apm');
+      });
+
+      describe('Admin user', () => {
+        let statusCheck: ProfilingStatus;
+        before(async () => {
+          const response = await profilingApiClient.adminUser({
+            endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+          });
+          statusCheck = response.body;
+        });
+        it(`has been set up`, async () => {
+          expect(statusCheck.has_setup).to.be(true);
+        });
+
+        it(`does not have data`, async () => {
+          expect(statusCheck.has_data).to.be(false);
+        });
+
+        it(`does not have pre 8.9.1 data`, async () => {
+          expect(statusCheck.pre_8_9_1_data).to.be(false);
+        });
+      });
+
+      describe('Viewer user', () => {
+        let statusCheck: ProfilingStatus;
+        before(async () => {
+          const response = await profilingApiClient.readUser({
+            endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+          });
+          statusCheck = response.body;
+        });
+        it(`has been set up`, async () => {
+          expect(statusCheck.has_setup).to.be(true);
+        });
+
+        it(`has data`, async () => {
+          expect(statusCheck.has_data).to.be(true);
+        });
+
+        it(`does not have pre 8.9.1 data`, async () => {
+          expect(statusCheck.pre_8_9_1_data).to.be(false);
+        });
+
+        it(`is unauthorized to fully check profiling status `, async () => {
+          expect(statusCheck.unauthorized).to.be(true);
+        });
+      });
+    });
+
+    describe('Profiling is set up', () => {
+      before(async () => {
+        await setupProfiling(bettertest, logger);
+      });
+
+      describe('without data', () => {
+        describe('Admin user', () => {
+          let statusCheck: ProfilingStatus;
+          before(async () => {
+            const response = await profilingApiClient.adminUser({
+              endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+            });
+            statusCheck = response.body;
+          });
+          it(`has been set up`, async () => {
+            expect(statusCheck.has_setup).to.be(true);
+          });
+
+          it(`does not have data`, async () => {
+            expect(statusCheck.has_data).to.be(false);
+          });
+
+          it(`does not have pre 8.9.1 data`, async () => {
+            expect(statusCheck.pre_8_9_1_data).to.be(false);
+          });
+        });
+
+        describe('Viewer user', () => {
+          let statusCheck: ProfilingStatus;
+          before(async () => {
+            const response = await profilingApiClient.readUser({
+              endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+            });
+            statusCheck = response.body;
+          });
+          it(`has been set up`, async () => {
+            expect(statusCheck.has_setup).to.be(true);
+          });
+
+          it(`has data`, async () => {
+            expect(statusCheck.has_data).to.be(true);
+          });
+
+          it(`does not have pre 8.9.1 data`, async () => {
+            expect(statusCheck.pre_8_9_1_data).to.be(false);
+          });
+
+          it(`is unauthorized to fully check profiling status `, async () => {
+            expect(statusCheck.unauthorized).to.be(true);
+          });
+        });
+      });
+
+      describe('with data', () => {
+        before(async () => {
+          await loadProfilingData(es, logger);
+        });
+        describe('Admin user', () => {
+          let statusCheck: ProfilingStatus;
+          before(async () => {
+            const response = await profilingApiClient.adminUser({
+              endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+            });
+            statusCheck = response.body;
+          });
+          it(`has been set up`, async () => {
+            expect(statusCheck.has_setup).to.be(true);
+          });
+
+          it(`does not have data`, async () => {
+            expect(statusCheck.has_data).to.be(true);
+          });
+
+          it(`does not have pre 8.9.1 data`, async () => {
+            expect(statusCheck.pre_8_9_1_data).to.be(false);
+          });
+        });
+
+        describe('Viewer user', () => {
+          let statusCheck: ProfilingStatus;
+          before(async () => {
+            const response = await profilingApiClient.readUser({
+              endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
+            });
+            statusCheck = response.body;
+          });
+          it(`has been set up`, async () => {
+            expect(statusCheck.has_setup).to.be(true);
+          });
+
+          it(`has data`, async () => {
+            expect(statusCheck.has_data).to.be(true);
+          });
+
+          it(`does not have pre 8.9.1 data`, async () => {
+            expect(statusCheck.pre_8_9_1_data).to.be(false);
+          });
+
+          it(`is unauthorized to fully check profiling status `, async () => {
+            expect(statusCheck.unauthorized).to.be(true);
+          });
+        });
+      });
+    });
+  });
+}

x-pack/test/profiling_api_integration/utils/fleet.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PackagePolicy } from '@kbn/fleet-plugin/common';
+import {
+  COLLECTOR_PACKAGE_POLICY_NAME,
+  SYMBOLIZER_PACKAGE_POLICY_NAME,
+} from '@kbn/profiling-data-access-plugin/common';
+import { BetterTest } from '../common/bettertest';
+
+export async function deletePackagePolicy(bettertest: BetterTest, packagePolicyId: string) {
+  return bettertest({
+    pathname: `/api/fleet/package_policies/delete`,
+    method: 'post',
+    body: { packagePolicyIds: [packagePolicyId] },
+  });
+}
+
+export async function getProfilingPackagePolicyIds(bettertest: BetterTest) {
+  const response = await bettertest<{ items: PackagePolicy[] }>({
+    pathname: '/api/fleet/package_policies',
+    method: 'get',
+  });
+
+  const collector = response.body.items.find((item) => item.name === COLLECTOR_PACKAGE_POLICY_NAME);
+  const symbolizer = response.body.items.find(
+    (item) => item.name === SYMBOLIZER_PACKAGE_POLICY_NAME
+  );
+
+  return {
+    collectorId: collector?.id,
+    symbolizerId: symbolizer?.id,
+  };
+}

x-pack/test/profiling_api_integration/utils/profiling_data.ts

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { Client } from '@elastic/elasticsearch';
+import { getRoutePaths } from '@kbn/profiling-plugin/common';
+import { ProfilingStatus } from '@kbn/profiling-utils';
+import { ToolingLog } from '@kbn/tooling-log';
+import fs from 'fs';
+import Path from 'path';
+import { BetterTest } from '../common/bettertest';
+import { deletePackagePolicy, getProfilingPackagePolicyIds } from './fleet';
+
+const profilingRoutePaths = getRoutePaths();
+
+const esArchiversPath = Path.posix.join(
+  __dirname,
+  '..',
+  'common',
+  'fixtures',
+  'es_archiver',
+  'profiling'
+);
+
+function logWithTimer(logger: ToolingLog) {
+  const start = process.hrtime();
+
+  return (message: string) => {
+    const diff = process.hrtime(start);
+    const time = `${Math.round(diff[0] * 1000 + diff[1] / 1e6)}ms`;
+    logger.info(`(${time}) ${message}`);
+  };
+}
+
+export async function cleanUpProfilingData({
+  es,
+  bettertest,
+  logger,
+}: {
+  es: Client;
+  bettertest: BetterTest;
+  logger: ToolingLog;
+}) {
+  const log = logWithTimer(logger);
+  log(`Unloading Profiling data`);
+
+  const [indices, { collectorId, symbolizerId }] = await Promise.all([
+    es.cat.indices({ format: 'json' }),
+    getProfilingPackagePolicyIds(bettertest),
+  ]);
+
+  const profilingIndices = indices
+    .filter((index) => index.index !== undefined)
+    .map((index) => index.index)
+    .filter((index) => {
+      return index!.startsWith('profiling') || index!.startsWith('.profiling');
+    }) as string[];
+
+  await Promise.all([
+    ...profilingIndices.map((index) => es.indices.delete({ index })),
+    es.indices.deleteDataStream({
+      name: 'profiling-events*',
+    }),
+    collectorId ? deletePackagePolicy(bettertest, collectorId) : Promise.resolve(),
+    symbolizerId ? deletePackagePolicy(bettertest, symbolizerId) : Promise.resolve(),
+  ]);
+  log('Unloaded Profiling data');
+}
+
+export async function setupProfiling(bettertest: BetterTest, logger: ToolingLog) {
+  const log = logWithTimer(logger);
+  const response = await bettertest<ProfilingStatus>({
+    method: 'get',
+    pathname: profilingRoutePaths.HasSetupESResources,
+  });
+
+  if (response.body.has_setup) {
+    log(`Skipping Universal Profiling set up, already set up`);
+  } else {
+    log(`Setting up Universal Profiling`);
+    await bettertest<ProfilingStatus>({
+      method: 'post',
+      pathname: profilingRoutePaths.HasSetupESResources,
+    });
+    log(`Universal Profiling set up`);
+  }
+}
+
+export async function loadProfilingData(es: Client, logger: ToolingLog) {
+  const log = logWithTimer(logger);
+  log(`Loading profiling data`);
+  const content = fs.readFileSync(`${esArchiversPath}/data.json`, 'utf8');
+  await es.bulk({ operations: content.split('\n'), refresh: 'wait_for' });
+  log('Loaded profiling data');
+}

x-pack/test/tsconfig.json

@@ -142,5 +142,6 @@
     "@kbn/stack-alerts-plugin",
     "@kbn/apm-data-access-plugin",
     "@kbn/profiling-utils",
+    "@kbn/profiling-data-access-plugin",
   ]
 }