[Profling] Single-click setup from Kibana (#148959)

Co-authored-by: inge4pres <francesco.gualazzi@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Tim Rühsen <tim.ruehsen@gmx.de>
Co-authored-by: Francesco Gualazzi <inge4pres@users.noreply.github.com>
Closes https://github.com/elastic/prodfiler/issues/2884

.eslintrc.js

@@ -902,7 +902,10 @@ module.exports = {
       files: ['x-pack/plugins/profiling/**/*.{js,mjs,ts,tsx}'],
       rules: {
         'react-hooks/rules-of-hooks': 'error', // Checks rules of Hooks
-        'react-hooks/exhaustive-deps': ['error', { additionalHooks: '^(useAsync)$' }],
+        'react-hooks/exhaustive-deps': [
+          'error',
+          { additionalHooks: '^(useAsync|useTimeRangeAsync|useAutoAbortedHttpClient)$' },
+        ],
       },
     },
     {

x-pack/plugins/profiling/common/index.ts

@@ -29,6 +29,9 @@ export function getRoutePaths() {
     Flamechart: `${BASE_ROUTE_PATH}/flamechart`,
     CacheExecutables: `${BASE_ROUTE_PATH}/cache/executables`,
     CacheStackFrames: `${BASE_ROUTE_PATH}/cache/stackframes`,
+    HasSetupESResources: `${BASE_ROUTE_PATH}/setup/es_resources`,
+    HasSetupDataCollection: `${BASE_ROUTE_PATH}/setup/has_data`,
+    SetupDataCollectionInstructions: `${BASE_ROUTE_PATH}/setup/instructions`,
   };
 }
 

x-pack/plugins/profiling/kibana.json

@@ -19,7 +19,10 @@
     "kibanaReact",
     "unifiedSearch",
     "dataViews",
-    "charts"
+    "charts",
+    "spaces",
+    "cloud",
+    "fleet"
   ],
   "optionalPlugins": [],
   "configPath": [

x-pack/plugins/profiling/public/app.tsx

@@ -6,21 +6,22 @@
  */
 
 import { AppMountParameters, CoreSetup, CoreStart } from '@kbn/core/public';
-import React, { useMemo } from 'react';
-import ReactDOM from 'react-dom';
-
 import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public';
 import { Storage } from '@kbn/kibana-utils-plugin/public';
-import { RouteRenderer, RouterProvider } from '@kbn/typed-react-router-config';
-
 import { RedirectAppLinks } from '@kbn/shared-ux-link-redirect-app';
+import { RouteRenderer, RouterProvider } from '@kbn/typed-react-router-config';
+import React, { useMemo } from 'react';
+import ReactDOM from 'react-dom';
+import { HeaderMenuPortal } from '@kbn/observability-plugin/public';
+import { CheckSetup } from './components/check_setup';
 import { ProfilingDependenciesContextProvider } from './components/contexts/profiling_dependencies/profiling_dependencies_context';
+import { RouteBreadcrumbsContextProvider } from './components/contexts/route_breadcrumbs_context';
+import { TimeRangeContextProvider } from './components/contexts/time_range_context';
 import { RedirectWithDefaultDateRange } from './components/redirect_with_default_date_range';
 import { profilingRouter } from './routing';
 import { Services } from './services';
 import { ProfilingPluginPublicSetupDeps, ProfilingPluginPublicStartDeps } from './types';
-import { RouteBreadcrumbsContextProvider } from './components/contexts/route_breadcrumbs_context';
-import { TimeRangeContextProvider } from './components/contexts/time_range_context';
+import { ProfilingHeaderActionMenu } from './components/profiling_header_action_menu';
 
 interface Props {
   profilingFetchServices: Services;
@@ -30,10 +31,25 @@ interface Props {
   pluginsSetup: ProfilingPluginPublicSetupDeps;
   theme$: AppMountParameters['theme$'];
   history: AppMountParameters['history'];
+  setHeaderActionMenu: AppMountParameters['setHeaderActionMenu'];
 }
 
 const storage = new Storage(localStorage);
 
+function MountProfilingActionMenu({
+  theme$,
+  setHeaderActionMenu,
+}: {
+  theme$: AppMountParameters['theme$'];
+  setHeaderActionMenu: AppMountParameters['setHeaderActionMenu'];
+}) {
+  return (
+    <HeaderMenuPortal setHeaderActionMenu={setHeaderActionMenu} theme$={theme$}>
+      <ProfilingHeaderActionMenu />
+    </HeaderMenuPortal>
+  );
+}
+
 function App({
   coreStart,
   coreSetup,
@@ -42,6 +58,7 @@ function App({
   profilingFetchServices,
   theme$,
   history,
+  setHeaderActionMenu,
 }: Props) {
   const i18nCore = coreStart.i18n;
 
@@ -67,11 +84,17 @@ function App({
             <RouterProvider router={profilingRouter as any} history={history}>
               <TimeRangeContextProvider>
                 <ProfilingDependenciesContextProvider value={profilingDependencies}>
-                  <RedirectWithDefaultDateRange>
-                    <RouteBreadcrumbsContextProvider>
-                      <RouteRenderer />
-                    </RouteBreadcrumbsContextProvider>
-                  </RedirectWithDefaultDateRange>
+                  <CheckSetup>
+                    <RedirectWithDefaultDateRange>
+                      <RouteBreadcrumbsContextProvider>
+                        <RouteRenderer />
+                      </RouteBreadcrumbsContextProvider>
+                    </RedirectWithDefaultDateRange>
+                  </CheckSetup>
+                  <MountProfilingActionMenu
+                    setHeaderActionMenu={setHeaderActionMenu}
+                    theme$={theme$}
+                  />
                 </ProfilingDependenciesContextProvider>
               </TimeRangeContextProvider>
             </RouterProvider>

x-pack/plugins/profiling/public/components/check_setup.tsx

@@ -0,0 +1,145 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiLoadingSpinner, EuiText } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import React, { useState } from 'react';
+import { AsyncStatus, useAsync } from '../hooks/use_async';
+import { useAutoAbortedHttpClient } from '../hooks/use_auto_aborted_http_client';
+import { useProfilingDependencies } from './contexts/profiling_dependencies/use_profiling_dependencies';
+import { NoDataPage } from './no_data_page';
+import { ProfilingAppPageTemplate } from './profiling_app_page_template';
+
+export function CheckSetup({ children }: { children: React.ReactElement }) {
+  const {
+    start: { core },
+    services: { fetchHasSetup, postSetupResources },
+  } = useProfilingDependencies();
+
+  const [postSetupLoading, setPostSetupLoading] = useState(false);
+
+  const { status, data, error, refresh } = useAsync(
+    ({ http }) => {
+      return fetchHasSetup({ http });
+    },
+    [fetchHasSetup]
+  );
+
+  const http = useAutoAbortedHttpClient([]);
+
+  const displaySetupScreen =
+    (status === AsyncStatus.Settled && data?.has_setup !== true) || !!error;
+
+  const displayNoDataScreen =
+    status === AsyncStatus.Settled && data?.has_setup === true && data?.has_data === false;
+
+  const displayUi = data?.has_data === true;
+
+  const docsLink = `https://elastic.github.io/universal-profiling-documentation`;
+
+  const displayLoadingScreen = status !== AsyncStatus.Settled;
+
+  if (displayLoadingScreen) {
+    return (
+      <ProfilingAppPageTemplate hideSearchBar tabs={[]}>
+        <EuiFlexGroup alignItems="center" justifyContent="center">
+          <EuiFlexItem grow={false}>
+            <EuiLoadingSpinner size="xxl" />
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </ProfilingAppPageTemplate>
+    );
+  }
+
+  if (displayUi) {
+    return children;
+  }
+
+  if (displayNoDataScreen) {
+    return (
+      <NoDataPage
+        subTitle={i18n.translate('xpack.profiling.noDataPage.introduction', {
+          defaultMessage: `You're almost there! Follow the instructions below to add data.`,
+        })}
+      />
+    );
+  }
+
+  if (displaySetupScreen) {
+    return (
+      <ProfilingAppPageTemplate
+        tabs={[]}
+        noDataConfig={{
+          docsLink,
+          logo: 'logoObservability',
+          pageTitle: i18n.translate('xpack.profiling.noDataConfig.pageTitle', {
+            defaultMessage: 'Universal Profiling (now in Beta)',
+          }),
+          action: {
+            elasticAgent: {
+              description: (
+                <EuiText>
+                  {i18n.translate('xpack.profiling.noDataConfig.action.title', {
+                    defaultMessage: `Universal Profiling provides fleet-wide, whole-system, continuous profiling with zero instrumentation.
+              Understand what lines of code are consuming compute resources, at all times, and across your entire infrastructure.`,
+                  })}
+                </EuiText>
+              ),
+              onClick: (event: React.MouseEvent<HTMLElement, MouseEvent>) => {
+                event.preventDefault();
+              },
+              button: (
+                <EuiButton
+                  disabled={postSetupLoading}
+                  onClick={(event) => {
+                    event.preventDefault();
+
+                    setPostSetupLoading(true);
+
+                    postSetupResources({ http })
+                      .then(() => refresh())
+                      .catch((err) => {
+                        const message = err?.body?.message ?? err.message ?? String(err);
+
+                        core.notifications.toasts.addError(err, {
+                          title: i18n.translate(
+                            'xpack.profiling.checkSetup.setupFailureToastTitle',
+                            { defaultMessage: 'Failed to complete setup' }
+                          ),
+                          toastMessage: message,
+                        });
+                      })
+                      .finally(() => {
+                        setPostSetupLoading(false);
+                      });
+                  }}
+                  fill
+                  isLoading={postSetupLoading}
+                >
+                  {!postSetupLoading
+                    ? i18n.translate('xpack.profiling.noDataConfig.action.buttonLabel', {
+                        defaultMessage: 'Setup Universal Profiling',
+                      })
+                    : i18n.translate('xpack.profiling.noDataConfig.action.buttonLoadingLabel', {
+                        defaultMessage: 'Setting up Universal Profiling...',
+                      })}
+                </EuiButton>
+              ),
+            },
+          },
+          solution: i18n.translate('xpack.profiling.noDataConfig.solutionName', {
+            defaultMessage: 'Universal Profiling',
+          }),
+        }}
+        hideSearchBar
+      >
+        <></>
+      </ProfilingAppPageTemplate>
+    );
+  }
+
+  throw new Error('Invalid state');
+}

x-pack/plugins/profiling/public/components/flame_graphs_view/index.tsx

@@ -116,7 +116,7 @@ export function FlameGraphsView({ children }: { children: React.ReactElement })
   }
 
   return (
-    <ProfilingAppPageTemplate tabs={tabs} hideSearchBar={isDifferentialView} fullHeight>
+    <ProfilingAppPageTemplate tabs={tabs} hideSearchBar={isDifferentialView}>
       <EuiFlexGroup direction="column">
         {isDifferentialView ? (
           <EuiFlexItem grow={false}>

x-pack/plugins/profiling/public/components/no_data_page.tsx

@@ -0,0 +1,327 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  EuiCodeBlock,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiIcon,
+  EuiLink,
+  EuiLoadingSpinner,
+  EuiPanel,
+  EuiSpacer,
+  EuiSplitPanel,
+  EuiSteps,
+  EuiTab,
+  EuiTabs,
+  EuiText,
+} from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import React, { useState } from 'react';
+import { AsyncStatus, useAsync } from '../hooks/use_async';
+import { useProfilingDependencies } from './contexts/profiling_dependencies/use_profiling_dependencies';
+import { ProfilingAppPageTemplate } from './profiling_app_page_template';
+
+export function NoDataPage({ subTitle }: { subTitle: string }) {
+  const {
+    services: { setupDataCollectionInstructions },
+  } = useProfilingDependencies();
+
+  const { data, status } = useAsync(
+    ({ http }) => {
+      return setupDataCollectionInstructions({ http });
+    },
+    [setupDataCollectionInstructions]
+  );
+
+  const secretToken = data?.variables.secretToken;
+  const collectionAgentHostPort = data?.variables.apmServerUrl.replace('https://', '');
+
+  const tabs = [
+    {
+      key: 'kubernetes',
+      title: i18n.translate('xpack.profiling.tabs.kubernetesTitle', {
+        defaultMessage: 'Kubernetes',
+      }),
+      steps: [
+        {
+          title: i18n.translate('xpack.profiling.tabs.kubernetesRepositoryStep', {
+            defaultMessage: 'Configure the Universal Profiling host-agent Helm repository:',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              helm repo add optimyze https://optimyze.cloud/helm-charts
+            </EuiCodeBlock>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.kubernetesInstallStep', {
+            defaultMessage: 'Install host-agent via Helm:',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              {`helm install --create-namespace -n=universal-profiling universal-profiling-agent \\
+--set "projectID=1,secretToken=${secretToken}" \\
+--set "collectionAgentHostPort=${collectionAgentHostPort}" \\
+--set "image.baseUrl=docker.elastic.co,image.repository=observability,image.name=profiling-agent" \\
+optimyze/pf-host-agent`}
+            </EuiCodeBlock>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.kubernetesValidationStep', {
+            defaultMessage: 'Validate the host-agent pods are running:',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              kubectl -n universal-profiling get pods
+            </EuiCodeBlock>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.postValidationStep', {
+            defaultMessage:
+              'Use the Helm install output to get host-agent logs and spot potential errors',
+          }),
+          content: <></>,
+        },
+      ],
+    },
+    {
+      key: 'docker',
+      title: i18n.translate('xpack.profiling.tabs.dockerTitle', {
+        defaultMessage: 'Docker',
+      }),
+      steps: [
+        {
+          title: i18n.translate('xpack.profiling.tabs.dockerRunContainerStep', {
+            defaultMessage: 'Run the Universal Profiling container:',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              {`docker run --name host-agent --privileged --pid=host -v /etc/machine-id:/etc/machine-id:ro \\
+-v /var/run/docker.sock:/var/run/docker.sock -v /sys/kernel/debug:/sys/kernel/debug:ro \\
+docker.elastic.co/observability/profiling-agent:stable /root/pf-host-agent \\
+-project-id=1 -secret-token=${secretToken} \\
+-collection-agent=${collectionAgentHostPort}`}
+            </EuiCodeBlock>
+          ),
+        },
+      ],
+    },
+    {
+      key: 'binary',
+      title: i18n.translate('xpack.profiling.tabs.binaryTitle', {
+        defaultMessage: 'Binary',
+      }),
+      steps: [
+        {
+          title: i18n.translate('xpack.profiling.tabs.binaryDownloadStep', {
+            defaultMessage: 'Download the latest binary:',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              wget -O- https://releases.prodfiler.com/stable/pf-host-agent_linux_amd64.tgz | tar xz
+            </EuiCodeBlock>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.binaryGrantPermissionStep', {
+            defaultMessage: 'Grant executable permissions:',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              chmod +x pf-host-agent/pf-host-agent
+            </EuiCodeBlock>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.binaryRunHostAgentStep', {
+            defaultMessage: 'Run the Universal Profiling host-agent (requires root privileges):',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              {`sudo pf-host-agent/pf-host-agent -project-id=1 -secret-token=${secretToken} -collection-agent=${collectionAgentHostPort}`}
+            </EuiCodeBlock>
+          ),
+        },
+      ],
+    },
+    {
+      key: 'deb',
+      title: i18n.translate('xpack.profiling.tabs.debTitle', {
+        defaultMessage: 'DEB Package',
+      }),
+      steps: [
+        {
+          title: i18n.translate('xpack.profiling.tabs.debDownloadPackageStep', {
+            defaultMessage:
+              'Open the URL below and download the right DEB package for your CPU architecture:',
+          }),
+          content: (
+            <EuiLink target="_blank" href={`https://releases.prodfiler.com/stable/index.html`}>
+              https://releases.prodfiler.com/stable/index.html
+            </EuiLink>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.debInstallPackageStep', {
+            defaultMessage: 'Install the DEB package (requires root privileges):',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              {`sudo dpkg -i pf-host-agent*.deb`}
+            </EuiCodeBlock>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.debEditConfigStep', {
+            defaultMessage: 'Edit the configuration (requires root privileges):',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              {`echo -e "project-id 1\nsecret-token ${secretToken}\ncollection-agent ${collectionAgentHostPort}" | sudo tee -a /etc/prodfiler/prodfiler.conf`}
+            </EuiCodeBlock>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.debStartSystemdServiceStep', {
+            defaultMessage:
+              'Start the Universal Profiling systemd service (requires root privileges):',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              {`sudo systemctl enable pf-host-agent && sudo systemctl restart pf-host-agent`}
+            </EuiCodeBlock>
+          ),
+        },
+      ],
+    },
+    {
+      key: 'rpm',
+      title: i18n.translate('xpack.profiling.tabs.rpmTitle', {
+        defaultMessage: 'RPM Package',
+      }),
+      steps: [
+        {
+          title: i18n.translate('xpack.profiling.tabs.rpmDownloadPackageStep', {
+            defaultMessage:
+              'Open the URL below and download the right RPM package for your CPU architecture:',
+          }),
+          content: (
+            <EuiLink target="_blank" href={`https://releases.prodfiler.com/stable/index.html`}>
+              https://releases.prodfiler.com/stable/index.html
+            </EuiLink>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.rpmInstallPackageStep', {
+            defaultMessage: 'Install the RPM package (requires root privileges):',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              {`sudo rpm -i pf-host-agent*.rpm`}
+            </EuiCodeBlock>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.rpmEditConfigStep', {
+            defaultMessage: 'Edit the configuration (requires root privileges):',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              {`echo -e "project-id 1\nsecret-token ${secretToken}\ncollection-agent ${collectionAgentHostPort}" | sudo tee -a /etc/prodfiler/prodfiler.conf`}
+            </EuiCodeBlock>
+          ),
+        },
+        {
+          title: i18n.translate('xpack.profiling.tabs.rpmStartSystemdServiceStep', {
+            defaultMessage:
+              'Start the Universal Profiling systemd service (requires root privileges):',
+          }),
+          content: (
+            <EuiCodeBlock paddingSize="s" isCopyable>
+              {`sudo systemctl enable pf-host-agent && sudo systemctl restart pf-host-agent`}
+            </EuiCodeBlock>
+          ),
+        },
+      ],
+    },
+  ];
+
+  const [selectedTab, setSelectedTab] = useState(tabs[0].key);
+
+  const displayedTab = tabs.find((tab) => tab.key === selectedTab)!;
+
+  const displayedSteps = displayedTab.steps ?? [];
+
+  const isLoading = status === AsyncStatus.Loading;
+
+  return (
+    <ProfilingAppPageTemplate
+      tabs={[]}
+      restrictWidth
+      hideSearchBar
+      pageTitle={
+        <EuiFlexGroup direction="row" alignItems="center">
+          <EuiFlexItem grow={false}>
+            <EuiIcon type="logoObservability" size="m" />
+          </EuiFlexItem>
+          <EuiFlexItem grow={false}>
+            {i18n.translate('xpack.profiling.noDataPage.pageTitle', {
+              defaultMessage: 'Add profiling data',
+            })}
+          </EuiFlexItem>
+          {isLoading ? (
+            <EuiFlexItem>
+              <EuiLoadingSpinner />
+            </EuiFlexItem>
+          ) : null}
+        </EuiFlexGroup>
+      }
+    >
+      {isLoading ? (
+        <></>
+      ) : (
+        <>
+          <EuiText>{subTitle}</EuiText>
+          <EuiSpacer />
+          <EuiSplitPanel.Outer>
+            <EuiPanel hasBorder={false} hasShadow={false} grow={false} paddingSize="none">
+              <EuiSplitPanel.Inner color="subdued" paddingSize="none">
+                <EuiTabs style={{ padding: '0 24px' }}>
+                  {tabs.map((tab) => {
+                    return (
+                      <EuiTab
+                        key={tab.key}
+                        onClick={() => setSelectedTab(tab.key)}
+                        isSelected={tab.key === selectedTab}
+                      >
+                        {tab.title}
+                      </EuiTab>
+                    );
+                  })}
+                </EuiTabs>
+              </EuiSplitPanel.Inner>
+              <EuiSplitPanel.Inner style={{ padding: '0 24px' }}>
+                <EuiSpacer size="xxl" />
+                <EuiSteps
+                  steps={displayedSteps.map((step) => {
+                    return {
+                      title: step.title,
+                      children: step.content,
+                      status: 'incomplete',
+                    };
+                  })}
+                />
+              </EuiSplitPanel.Inner>
+            </EuiPanel>
+          </EuiSplitPanel.Outer>
+        </>
+      )}
+    </ProfilingAppPageTemplate>
+  );
+}

x-pack/plugins/profiling/public/components/profiling_app_page_template/index.tsx

@@ -9,6 +9,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiPageHeaderContentProps } from '@elastic/e
 import { i18n } from '@kbn/i18n';
 import React, { useEffect } from 'react';
 import { useHistory } from 'react-router-dom';
+import { NoDataPageProps } from '@kbn/shared-ux-page-no-data-types';
 import { useProfilingDependencies } from '../contexts/profiling_dependencies/use_profiling_dependencies';
 import { PrimaryProfilingSearchBar } from './primary_profiling_search_bar';
 
@@ -16,12 +17,18 @@ export function ProfilingAppPageTemplate({
   children,
   tabs,
   hideSearchBar = false,
-  fullHeight = false,
+  noDataConfig,
+  restrictWidth = false,
+  pageTitle = i18n.translate('xpack.profiling.appPageTemplate.pageTitle', {
+    defaultMessage: 'Universal Profiling',
+  }),
 }: {
   children: React.ReactElement;
   tabs: EuiPageHeaderContentProps['tabs'];
   hideSearchBar?: boolean;
-  fullHeight?: boolean;
+  noDataConfig?: NoDataPageProps;
+  restrictWidth?: boolean;
+  pageTitle?: React.ReactNode;
 }) {
   const {
     start: { observability },
@@ -37,12 +44,12 @@ export function ProfilingAppPageTemplate({
 
   return (
     <ObservabilityPageTemplate
+      noDataConfig={noDataConfig}
       pageHeader={{
-        pageTitle: i18n.translate('xpack.profiling.appPageTemplate.pageTitle', {
-          defaultMessage: 'Universal Profiling',
-        }),
+        pageTitle,
         tabs,
       }}
+      restrictWidth={restrictWidth}
       pageSectionProps={{
         contentProps: {
           style: {

x-pack/plugins/profiling/public/components/profiling_app_page_template/primary_profiling_search_bar.tsx

@@ -21,11 +21,13 @@ export function PrimaryProfilingSearchBar({ showSubmitButton }: { showSubmitButt
   const profilingRouter = useProfilingRouter();
   const routePath = useProfilingRoutePath();
 
-  const {
-    path,
-    query,
-    query: { rangeFrom, rangeTo, kuery },
-  } = useProfilingParams('/*');
+  const { path, query } = useProfilingParams('/*');
+
+  if (!('rangeFrom' in query)) {
+    throw new Error('Range query parameters are missing');
+  }
+
+  const { rangeFrom, rangeTo, kuery } = query;
 
   const { refresh } = useTimeRangeContext();
 

x-pack/plugins/profiling/public/components/profiling_header_action_menu.tsx

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { EuiFlexGroup, EuiFlexItem, EuiHeaderLink, EuiHeaderLinks, EuiIcon } from '@elastic/eui';
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import { useProfilingRouter } from '../hooks/use_profiling_router';
+
+export function ProfilingHeaderActionMenu() {
+  const router = useProfilingRouter();
+  return (
+    <EuiHeaderLinks gutterSize="xs">
+      <EuiHeaderLink href={router.link('/add-data-instructions')} color="primary">
+        <EuiFlexGroup direction="row" gutterSize="s" alignItems="center">
+          <EuiFlexItem grow={false}>
+            <EuiIcon type="listAdd" />
+          </EuiFlexItem>
+          <EuiFlexItem grow={false}>
+            {i18n.translate('xpack.profiling.headerActionMenu.addData', {
+              defaultMessage: 'Add data',
+            })}
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </EuiHeaderLink>
+    </EuiHeaderLinks>
+  );
+}

x-pack/plugins/profiling/public/hooks/use_async.ts

@@ -4,11 +4,9 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { HttpFetchOptions, HttpHandler, HttpStart } from '@kbn/core-http-browser';
 import { AbortError } from '@kbn/kibana-utils-plugin/common';
-import { useEffect, useRef, useState } from 'react';
-import { Overwrite, ValuesType } from 'utility-types';
-import { useProfilingDependencies } from '../components/contexts/profiling_dependencies/use_profiling_dependencies';
+import { useCallback, useEffect, useState } from 'react';
+import { AutoAbortedHttpService, useAutoAbortedHttpClient } from './use_auto_aborted_http_client';
 
 export enum AsyncStatus {
   Loading = 'loading',
@@ -20,69 +18,39 @@ export interface AsyncState<T> {
   data?: T;
   error?: Error;
   status: AsyncStatus;
+  refresh: () => void;
 }
 
-const HTTP_METHODS = ['fetch', 'get', 'post', 'put', 'delete', 'patch'] as const;
-
-type HttpMethod = ValuesType<typeof HTTP_METHODS>;
-
-type AutoAbortedHttpMethod = (
-  path: string,
-  options: Omit<HttpFetchOptions, 'signal'>
-) => ReturnType<HttpHandler>;
-
-export type AutoAbortedHttpService = Overwrite<
-  HttpStart,
-  Record<HttpMethod, AutoAbortedHttpMethod>
->;
-
 export type UseAsync = <T>(
   fn: ({ http }: { http: AutoAbortedHttpService }) => Promise<T> | undefined,
   dependencies: any[]
 ) => AsyncState<T>;
 
 export const useAsync: UseAsync = (fn, dependencies) => {
-  const {
-    start: {
-      core: { http },
-    },
-  } = useProfilingDependencies();
+  const [refreshId, setRefreshId] = useState(0);
+
+  const refresh = useCallback(() => {
+    setRefreshId((id) => id + 1);
+  }, []);
+
   const [asyncState, setAsyncState] = useState<AsyncState<any>>({
     status: AsyncStatus.Init,
+    refresh,
   });
 
   const { data, error } = asyncState;
 
-  const controllerRef = useRef(new AbortController());
+  const httpClient = useAutoAbortedHttpClient(dependencies);
 
   useEffect(() => {
-    controllerRef.current.abort();
-
-    controllerRef.current = new AbortController();
-
-    const autoAbortedMethods = {} as Record<HttpMethod, AutoAbortedHttpMethod>;
-
-    for (const key of HTTP_METHODS) {
-      autoAbortedMethods[key] = (path, options) => {
-        return http[key](path, { ...options, signal: controllerRef.current.signal }).catch(
-          (err) => {
-            if (err.name === 'AbortError') {
-              // return never-resolving promise
-              return new Promise(() => {});
-            }
-            throw err;
-          }
-        );
-      };
-    }
-
-    const returnValue = fn({ http: { ...http, ...autoAbortedMethods } });
+    const returnValue = fn({ http: httpClient });
 
     if (returnValue === undefined) {
       setAsyncState({
         status: AsyncStatus.Init,
         data: undefined,
         error: undefined,
+        refresh,
       });
       return;
     }
@@ -91,12 +59,14 @@ export const useAsync: UseAsync = (fn, dependencies) => {
       status: AsyncStatus.Loading,
       data,
       error,
+      refresh,
     });
 
     returnValue.then((nextData) => {
       setAsyncState({
         status: AsyncStatus.Settled,
         data: nextData,
+        refresh,
       });
     });
 
@@ -107,17 +77,12 @@ export const useAsync: UseAsync = (fn, dependencies) => {
       setAsyncState({
         status: AsyncStatus.Settled,
         error: nextError,
+        refresh,
       });
       throw nextError;
     });
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [http, ...dependencies]);
-
-  useEffect(() => {
-    return () => {
-      controllerRef.current.abort();
-    };
-  }, []);
+  }, [httpClient, refreshId, ...dependencies]);
 
   return asyncState;
 };

x-pack/plugins/profiling/public/hooks/use_auto_aborted_http_client.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useEffect, useMemo, useRef } from 'react';
+import { Overwrite, ValuesType } from 'utility-types';
+import { HttpFetchOptions, HttpHandler, HttpStart } from '@kbn/core/public';
+import { useProfilingDependencies } from '../components/contexts/profiling_dependencies/use_profiling_dependencies';
+
+const HTTP_METHODS = ['fetch', 'get', 'post', 'put', 'delete', 'patch'] as const;
+
+type HttpMethod = ValuesType<typeof HTTP_METHODS>;
+
+type AutoAbortedHttpMethod = (
+  path: string,
+  options: Omit<HttpFetchOptions, 'signal'>
+) => ReturnType<HttpHandler>;
+
+export type AutoAbortedHttpService = Overwrite<
+  HttpStart,
+  Record<HttpMethod, AutoAbortedHttpMethod>
+>;
+
+export function useAutoAbortedHttpClient(dependencies: any[]): AutoAbortedHttpService {
+  const controller = useRef(new AbortController());
+
+  const {
+    start: {
+      core: { http },
+    },
+  } = useProfilingDependencies();
+
+  const httpClient = useMemo(() => {
+    controller.current.abort();
+
+    controller.current = new AbortController();
+
+    const autoAbortedMethods = {} as Record<HttpMethod, AutoAbortedHttpMethod>;
+
+    for (const key of HTTP_METHODS) {
+      autoAbortedMethods[key] = (path, options) => {
+        return http[key](path, { ...options, signal: controller.current.signal }).catch((err) => {
+          if (err.name === 'AbortError') {
+            // return never-resolving promise
+            return new Promise(() => {});
+          }
+          throw err;
+        });
+      };
+    }
+
+    return {
+      ...http,
+      ...autoAbortedMethods,
+    };
+
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [http, ...dependencies]);
+
+  useEffect(() => {
+    return () => {
+      controller.current.abort();
+    };
+  }, []);
+
+  return httpClient;
+}

x-pack/plugins/profiling/public/plugin.tsx

@@ -53,10 +53,10 @@ export class ProfilingPlugin implements Plugin {
       map(([_, kuery]) => {
         const sections: NavigationSection[] = [
           {
-            // TODO: add beta badge to section label, needs support in Observability plugin
             label: i18n.translate('xpack.profiling.navigation.sectionLabel', {
               defaultMessage: 'Universal Profiling',
             }),
+            isBetaFeature: true,
             entries: links.map((link) => {
               return {
                 app: 'profiling',
@@ -83,7 +83,7 @@ export class ProfilingPlugin implements Plugin {
       appRoute: '/app/profiling',
       category: DEFAULT_APP_CATEGORIES.observability,
       deepLinks: links,
-      async mount({ element, history, theme$ }: AppMountParameters) {
+      async mount({ element, history, theme$, setHeaderActionMenu }: AppMountParameters) {
         const [coreStart, pluginsStart] = (await coreSetup.getStartServices()) as [
           CoreStart,
           ProfilingPluginPublicStartDeps,
@@ -111,6 +111,7 @@ export class ProfilingPlugin implements Plugin {
             pluginsSetup,
             history,
             theme$,
+            setHeaderActionMenu,
           },
           element
         );

x-pack/plugins/profiling/public/routing/index.tsx

@@ -17,6 +17,7 @@ import { FunctionsView } from '../components/functions_view';
 import { RedirectTo } from '../components/redirect_to';
 import { RouteBreadcrumb } from '../components/route_breadcrumb';
 import { StackTracesView } from '../components/stack_traces_view';
+import { NoDataPage } from '../components/no_data_page';
 
 const routes = {
   '/': {
@@ -31,6 +32,15 @@ const routes = {
       </RouteBreadcrumb>
     ),
     children: {
+      '/add-data-instructions': {
+        element: (
+          <NoDataPage
+            subTitle={i18n.translate('xpack.profiling.addDataTitle', {
+              defaultMessage: 'Select an option below to deploy the host-agent.',
+            })}
+          />
+        ),
+      },
       '/': {
         children: {
           '/stacktraces/{topNType}': {

x-pack/plugins/profiling/public/services.ts

@@ -9,7 +9,8 @@ import { getRoutePaths } from '../common';
 import { BaseFlameGraph, createFlameGraph, ElasticFlameGraph } from '../common/flamegraph';
 import { TopNFunctions } from '../common/functions';
 import { TopNResponse } from '../common/topn';
-import { AutoAbortedHttpService } from './hooks/use_async';
+import type { SetupDataCollectionInstructions } from '../server/lib/setup/get_setup_instructions';
+import { AutoAbortedHttpService } from './hooks/use_auto_aborted_http_client';
 
 export interface Services {
   fetchTopN: (params: {
@@ -33,6 +34,13 @@ export interface Services {
     timeTo: number;
     kuery: string;
   }) => Promise<ElasticFlameGraph>;
+  fetchHasSetup: (params: {
+    http: AutoAbortedHttpService;
+  }) => Promise<{ has_setup: boolean; has_data: boolean }>;
+  postSetupResources: (params: { http: AutoAbortedHttpService }) => Promise<void>;
+  setupDataCollectionInstructions: (params: {
+    http: AutoAbortedHttpService;
+  }) => Promise<SetupDataCollectionInstructions>;
 }
 
 export function getServices(): Services {
@@ -40,45 +48,50 @@ export function getServices(): Services {
 
   return {
     fetchTopN: async ({ http, type, timeFrom, timeTo, kuery }) => {
-      try {
-        const query: HttpFetchQuery = {
-          timeFrom,
-          timeTo,
-          kuery,
-        };
-        return await http.get(`${paths.TopN}/${type}`, { query });
-      } catch (e) {
-        return e;
-      }
+      const query: HttpFetchQuery = {
+        timeFrom,
+        timeTo,
+        kuery,
+      };
+      return (await http.get(`${paths.TopN}/${type}`, { query })) as Promise<TopNResponse>;
     },
 
     fetchTopNFunctions: async ({ http, timeFrom, timeTo, startIndex, endIndex, kuery }) => {
-      try {
-        const query: HttpFetchQuery = {
-          timeFrom,
-          timeTo,
-          startIndex,
-          endIndex,
-          kuery,
-        };
-        return await http.get(paths.TopNFunctions, { query });
-      } catch (e) {
-        return e;
-      }
+      const query: HttpFetchQuery = {
+        timeFrom,
+        timeTo,
+        startIndex,
+        endIndex,
+        kuery,
+      };
+      return (await http.get(paths.TopNFunctions, { query })) as Promise<TopNFunctions>;
     },
 
     fetchElasticFlamechart: async ({ http, timeFrom, timeTo, kuery }) => {
-      try {
-        const query: HttpFetchQuery = {
-          timeFrom,
-          timeTo,
-          kuery,
-        };
-        const baseFlamegraph = (await http.get(paths.Flamechart, { query })) as BaseFlameGraph;
-        return createFlameGraph(baseFlamegraph);
-      } catch (e) {
-        return e;
-      }
+      const query: HttpFetchQuery = {
+        timeFrom,
+        timeTo,
+        kuery,
+      };
+      const baseFlamegraph = (await http.get(paths.Flamechart, { query })) as BaseFlameGraph;
+      return createFlameGraph(baseFlamegraph);
+    },
+    fetchHasSetup: async ({ http }) => {
+      const hasSetup = (await http.get(paths.HasSetupESResources, {})) as {
+        has_setup: boolean;
+        has_data: boolean;
+      };
+      return hasSetup;
+    },
+    postSetupResources: async ({ http }) => {
+      await http.post(paths.HasSetupESResources, {});
+    },
+    setupDataCollectionInstructions: async ({ http }) => {
+      const instructions = (await http.get(
+        paths.SetupDataCollectionInstructions,
+        {}
+      )) as SetupDataCollectionInstructions;
+      return instructions;
     },
   };
 }

x-pack/plugins/profiling/server/lib/setup/get_setup_instructions.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsClientContract } from '@kbn/core/server';
+import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
+import { getApmPolicy } from './steps/get_apm_policy';
+
+export interface SetupDataCollectionInstructions {
+  variables: {
+    apmServerUrl: string;
+    secretToken: string;
+  };
+}
+
+export async function getSetupInstructions({
+  packagePolicyClient,
+  soClient,
+}: {
+  packagePolicyClient: PackagePolicyClient;
+  soClient: SavedObjectsClientContract;
+}): Promise<SetupDataCollectionInstructions> {
+  const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
+
+  if (!apmPolicy) {
+    throw new Error('Could not find APM policy');
+  }
+
+  const apmServerVars = apmPolicy.inputs[0].vars;
+
+  return {
+    variables: {
+      apmServerUrl: apmServerVars!.url.value!,
+      secretToken: apmServerVars!.secret_token.value!,
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/has_profiling_data.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ProfilingESClient } from '../../utils/create_profiling_es_client';
+
+export async function hasProfilingData({
+  client,
+}: {
+  client: ProfilingESClient;
+}): Promise<boolean> {
+  const hasProfilingDataResponse = await client.search('has_any_profiling_data', {
+    index: 'profiling*',
+    size: 0,
+    track_total_hits: 1,
+    terminate_after: 1,
+  });
+
+  return hasProfilingDataResponse.hits.total.value > 0;
+}

x-pack/plugins/profiling/server/lib/setup/mappings/README.md

@@ -0,0 +1,53 @@
+## Universal Profiling mappings
+
+### Server routes
+
+* Check if ES setup is done
+
+      curl -H "content-type: application/json" -u <user:pass> \
+        -XGET "http://localhost:5601/api/profiling/v1/setup/es_resources"
+
+* Apply the ES setup (mappings + Fleet policy)
+
+      curl -H "content-type: application/json" -u <user:pass> -H "kbn-xsrf: reporting" \
+        -XPOST "http://localhost:5601/api/profiling/v1/setup/es_resources"
+
+* check data has been ingested
+
+      curl -H "content-type: application/json" -u <user:pass> \
+         -XGET "http://localhost:5601/api/profiling/v1/setup/has_data"
+    
+
+### Testing in Cloud
+
+Be sure to have configured `EC_API_KEY` env var with an API key for Cloud (ESS).
+
+Build and push a Kibana image with the latest changes.
+Choose a unique identifier for the build, then:
+
+```
+node scripts/build --docker-images --skip-docker-ubi --skip-docker-ubuntu
+docker tag docker.elastic.co/kibana-ci/kibana-cloud:8.7.0-SNAPSHOT docker.elastic.co/observability-ci/kibana:<UNIQUE_IDENTIFIER>
+docker push docker.elastic.co/observability-ci/kibana:<UNIQUE_IDENTIFIER>
+```
+
+Then, within `apm-server` repo:
+
+```
+cd testing/cloud
+make
+vim docker_image.auto.tfvars
+```
+
+Replace the `"kibana"` key in `docker_image_tag_override=` map with your unique identifier tag from previous step.
+Now you can run:
+
+```
+terraform init
+terraform apply -var-file docker_image.auto.tfvars
+```
+
+and once completed, you'll see the output with information on how to access the deployment.
+
+When changing code in Kibana, you don't need to tear down the Terraform deployment, simply update the `docker_image.auto.tfvars`
+with the new tag and run `terraform apply ...` as above: this will update Kibana.

x-pack/plugins/profiling/server/lib/setup/steps/catch_resource_already_exists_exception.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isResponseError } from '@kbn/es-errors';
+
+export function catchResourceAlreadyExistsException(error: any) {
+  if (isResponseError(error) && error.body?.error?.type === 'resource_already_exists_exception') {
+    return Promise.resolve();
+  }
+
+  return Promise.reject(error);
+}

x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_events.json

@@ -0,0 +1,76 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": "4",
+      "max_result_window": 150000,
+      "refresh_interval": "10s",
+      "sort": {
+        "field": [
+          "service.name",
+          "@timestamp",
+          "orchestrator.resource.name",
+          "container.name",
+          "process.thread.name",
+          "host.id"
+        ]
+      }
+    },
+    "codec": "best_compression"
+  },
+  "mappings": {
+    "_source": {
+      "enabled": false
+    },
+    "properties": {
+      "ecs.version": {
+        "type": "keyword",
+        "index": true
+      },
+      "service.name": {
+        "type": "keyword"
+      },
+      "@timestamp": {
+        "type": "date",
+        "format": "epoch_second"
+      },
+      "host.id": {
+        "type": "keyword"
+      },
+      "Stacktrace.id": {
+        "type": "keyword",
+        "index": false
+      },
+      "orchestrator.resource.name": {
+        "type": "keyword"
+      },
+      "container.name": {
+        "type": "keyword"
+      },
+      "process.thread.name": {
+        "type": "keyword"
+      },
+      "Stacktrace.count": {
+        "type": "short",
+        "index": false
+      },
+      "agent.version": {
+        "type": "keyword"
+      },
+      "host.ip": {
+        "type": "ip"
+      },
+      "host.ipstring": {
+        "type": "keyword"
+      },
+      "host.name": {
+        "type": "keyword"
+      },
+      "os.kernel": {
+        "type": "keyword"
+      },
+      "tags": {
+        "type": "keyword"
+      }
+    }
+  }
+}

x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_executables.json

@@ -0,0 +1,35 @@
+{
+  "settings": {
+    "index": {
+      "refresh_interval": "10s"
+    }
+  },
+  "mappings": {
+    "_source": {
+      "mode": "synthetic"
+    },
+    "properties": {
+      "ecs.version": {
+        "type": "keyword",
+        "index": true
+      },
+      "Executable.build.id": {
+        "type": "keyword",
+        "index": true
+      },
+      "Executable.file.name": {
+        "type": "keyword",
+        "index": true
+      },
+      "@timestamp": {
+        "type": "date",
+        "format": "epoch_second"
+      },
+      "Symbolization.lastprocessed": {
+        "type": "date",
+        "format": "epoch_second",
+        "index": false
+      }
+    }
+  }
+}

x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_ilm.json

@@ -0,0 +1,9 @@
+{
+  "settings": {
+    "index": {
+      "lifecycle": {
+        "name": "profiling"
+      }
+    }
+  }
+}

x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stackframes.json

@@ -0,0 +1,43 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 16,
+      "refresh_interval": "10s"
+    }
+  },
+  "mappings": {
+    "_source": {
+      "enabled": true
+    },
+    "properties": {
+      "ecs.version": {
+        "type": "keyword",
+        "index": true,
+        "doc_values": false,
+        "store": false
+      },
+      "Stackframe.line.number": {
+        "type": "integer",
+        "index": false,
+        "doc_values": false,
+        "store": false
+      },
+      "Stackframe.file.name": {
+        "type": "keyword",
+        "index": false,
+        "doc_values": false,
+        "store": false
+      },
+      "Stackframe.function.name": {
+        "type": "keyword",
+        "index": false
+      },
+      "Stackframe.function.offset": {
+        "type": "integer",
+        "index": false,
+        "doc_values": false,
+        "store": false
+      }
+    }
+  }
+}

x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stacktraces.json

@@ -0,0 +1,27 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 16,
+      "refresh_interval": "10s"
+    }
+  },
+  "mappings": {
+    "_source": {
+      "mode": "synthetic"
+    },
+    "properties": {
+      "ecs.version": {
+        "type": "keyword",
+        "index": true
+      },
+      "Stacktrace.frame.ids": {
+        "type": "keyword",
+        "index": false
+      },
+      "Stacktrace.frame.types": {
+        "type": "keyword",
+        "index": false
+      }
+    }
+  }
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_apm_package_step.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { installPackage, getInstallation } from '@kbn/fleet-plugin/server/services/epm/packages';
+import {
+  fetchFindLatestPackageOrThrow,
+  pkgToPkgKey,
+} from '@kbn/fleet-plugin/server/services/epm/registry';
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+
+export function getApmPackageStep({
+  client,
+  soClient,
+  spaceId,
+}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
+  const esClient = client.getEsClient();
+  return {
+    name: 'apm_package',
+    hasCompleted: async () => {
+      const installation = await getInstallation({
+        pkgName: 'apm',
+        savedObjectsClient: soClient,
+      });
+
+      return !!installation;
+    },
+    init: async () => {
+      const { name, version } = await fetchFindLatestPackageOrThrow('apm');
+
+      await installPackage({
+        installSource: 'registry',
+        esClient,
+        savedObjectsClient: soClient,
+        pkgkey: pkgToPkgKey({ name, version }),
+        spaceId,
+        force: true,
+      });
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_apm_policy.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsClientContract } from '@kbn/core/server';
+import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
+
+export const ELASTIC_CLOUD_APM_POLICY = 'elastic-cloud-apm';
+
+export async function getApmPolicy({
+  packagePolicyClient,
+  soClient,
+}: {
+  packagePolicyClient: PackagePolicyClient;
+  soClient: SavedObjectsClientContract;
+}) {
+  return await packagePolicyClient.get(soClient, ELASTIC_CLOUD_APM_POLICY);
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_cluster_settings_step.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+
+const MAX_BUCKETS = 150000;
+
+export function getClusterSettingsStep({
+  client,
+}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
+  return {
+    name: 'cluster_settings',
+    hasCompleted: async () => {
+      const settings = await client.getEsClient().cluster.getSettings({});
+
+      return settings.persistent.search?.max_buckets === MAX_BUCKETS.toString();
+    },
+    init: async () => {
+      await client.getEsClient().cluster.putSettings({
+        persistent: {
+          search: {
+            max_buckets: MAX_BUCKETS,
+          },
+        },
+      });
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_component_templates_step.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { IndicesIndexState } from '@elastic/elasticsearch/lib/api/types';
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+import componentTemplateProfilingIlm from './component_template_profiling_ilm.json';
+import componentTemplateProfilingEvents from './component_template_profiling_events.json';
+import componentTemplateProfilingExecutables from './component_template_profiling_executables.json';
+import componentTemplateProfilingStackframes from './component_template_profiling_stackframes.json';
+import componentTemplateProfilingStacktraces from './component_template_profiling_stacktraces.json';
+
+export enum ProfilingComponentTemplateName {
+  Ilm = 'profiling-ilm',
+  Events = 'profiling-events',
+  Executables = 'profiling-executables',
+  Stackframes = 'profiling-stackframes',
+  Stacktraces = 'profiling-stacktraces',
+}
+
+export function getComponentTemplatesStep({
+  client,
+  logger,
+}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
+  const esClient = client.getEsClient();
+
+  return {
+    name: 'component_templates',
+    hasCompleted: async () => {
+      return Promise.all(
+        [
+          ProfilingComponentTemplateName.Ilm,
+          ProfilingComponentTemplateName.Events,
+          ProfilingComponentTemplateName.Executables,
+          ProfilingComponentTemplateName.Stackframes,
+          ProfilingComponentTemplateName.Stacktraces,
+        ].map((componentTemplateName) =>
+          esClient.cluster.getComponentTemplate({
+            name: componentTemplateName,
+          })
+        )
+      ).then(
+        () => Promise.resolve(true),
+        (error) => {
+          logger.debug('Some component templates could not be fetched');
+          logger.debug(error);
+          return Promise.resolve(false);
+        }
+      );
+    },
+    init: async () => {
+      await Promise.all([
+        esClient.cluster.putComponentTemplate({
+          name: ProfilingComponentTemplateName.Ilm,
+          create: false,
+          template: componentTemplateProfilingIlm,
+        }),
+        esClient.cluster.putComponentTemplate({
+          name: ProfilingComponentTemplateName.Events,
+          create: false,
+          template: componentTemplateProfilingEvents as IndicesIndexState,
+          _meta: {
+            description: 'Mappings for profiling events data stream',
+          },
+        }),
+        esClient.cluster.putComponentTemplate({
+          name: ProfilingComponentTemplateName.Executables,
+          create: false,
+          template: componentTemplateProfilingExecutables as IndicesIndexState,
+        }),
+        esClient.cluster.putComponentTemplate({
+          name: ProfilingComponentTemplateName.Stackframes,
+          create: false,
+          template: componentTemplateProfilingStackframes as IndicesIndexState,
+        }),
+        esClient.cluster.putComponentTemplate({
+          name: ProfilingComponentTemplateName.Stacktraces,
+          create: false,
+          template: componentTemplateProfilingStacktraces as IndicesIndexState,
+        }),
+      ]);
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_create_events_data_streams.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception';
+
+function getEventDataStreamNames() {
+  const subSampledIndicesIdx = Array.from(Array(11).keys(), (item: number) => item + 1);
+  const subSampledIndexName = (pow: number): string => {
+    return `profiling-events-5pow${String(pow).padStart(2, '0')}`;
+  };
+  // Generate all the possible index template names
+  const eventsIndices = ['profiling-events-all'].concat(
+    subSampledIndicesIdx.map((pow) => subSampledIndexName(pow))
+  );
+
+  return eventsIndices;
+}
+
+export function getCreateEventsDataStreamsStep({
+  client,
+  logger,
+}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
+  const esClient = client.getEsClient();
+
+  const dataStreamNames = getEventDataStreamNames();
+
+  return {
+    name: 'create_events_data_streams',
+    hasCompleted: async () => {
+      const dataStreams = await esClient.indices.getDataStream({
+        name: 'profiling-events*',
+      });
+
+      const allDataStreams = dataStreams.data_streams.map((dataStream) => dataStream.name);
+
+      const missingDataStreams = dataStreamNames.filter(
+        (eventIndex) => !allDataStreams.includes(eventIndex)
+      );
+
+      if (missingDataStreams.length > 0) {
+        logger.debug(`Missing event indices: ${missingDataStreams.join(', ')}`);
+      }
+
+      return missingDataStreams.length === 0;
+    },
+    init: async () => {
+      await Promise.all(
+        dataStreamNames.map((dataStreamName) =>
+          esClient.indices
+            .createDataStream({
+              name: dataStreamName,
+            })
+            .catch(catchResourceAlreadyExistsException)
+        )
+      );
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts

@@ -0,0 +1,321 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MappingSourceField } from '@elastic/elasticsearch/lib/api/types';
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception';
+
+const SQ_EXECUTABLES_INDEX = 'profiling-sq-executables';
+const LEAFFRAMES_INDEX = 'profiling-sq-leafframes';
+const SYMBOLS_INDEX = 'profiling-symbols';
+const ILM_LOCK_INDEX = '.profiling-ilm-lock';
+
+const getKeyValueIndices = () => {
+  const kvIndices = ['profiling-stacktraces', 'profiling-stackframes', 'profiling-executables'];
+
+  const pairs: Array<{ index: string; alias: string }> = kvIndices.flatMap((index) => {
+    return [
+      { index: `${index}-000001`, alias: index },
+      { index: `${index}-000002`, alias: `${index}-next` },
+    ];
+  });
+
+  return pairs;
+};
+
+export function getCreateIndicesStep({
+  client,
+  logger,
+}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
+  const esClient = client.getEsClient();
+  const keyValueIndices = getKeyValueIndices();
+
+  return {
+    name: 'create_indices',
+    hasCompleted: async () => {
+      const nonKvIndices = [SQ_EXECUTABLES_INDEX, LEAFFRAMES_INDEX, SYMBOLS_INDEX, ILM_LOCK_INDEX];
+
+      const results = await Promise.all([
+        esClient.cat
+          .indices({
+            index: keyValueIndices
+              .map(({ index }) => index)
+              .concat(nonKvIndices)
+              .map((index) => index + '*')
+              .join(','),
+            format: 'json',
+          })
+          .then((response) => {
+            const allIndices = response.map((index) => index.index!);
+
+            const missingIndices = keyValueIndices
+              .map(({ index }) => index)
+              .concat(nonKvIndices)
+              .filter((index) => !allIndices.includes(index));
+
+            if (missingIndices.length) {
+              logger.debug(`Missing indices: ${missingIndices.join(',')}`);
+            }
+
+            return missingIndices.length === 0;
+          })
+          .catch((error) => {
+            logger.debug(`Failed fetching indices: ${error}`);
+            return Promise.resolve(false);
+          }),
+        esClient.cat
+          .aliases({
+            name: keyValueIndices.map(({ alias }) => alias + '*').join(','),
+            format: 'json',
+          })
+          .then((response) => {
+            const allAliases = response.map((index) => index.alias!);
+
+            const missingAliases = keyValueIndices
+              .map(({ alias }) => alias)
+              .filter((alias) => !allAliases.includes(alias));
+
+            if (missingAliases.length) {
+              logger.debug(`Missing aliases: ${missingAliases.join(',')}`);
+            }
+
+            return missingAliases.length === 0;
+          })
+          .catch((error) => {
+            logger.debug(`Failed fetching aliases: ${error}`);
+            return Promise.resolve(false);
+          }),
+      ]);
+
+      return results.every(Boolean);
+    },
+    init: async () => {
+      await Promise.all([
+        ...keyValueIndices.map(({ index, alias }) => {
+          return esClient.indices
+            .create({
+              index,
+              aliases: {
+                [alias]: {
+                  is_write_index: true,
+                },
+              },
+            })
+            .catch(catchResourceAlreadyExistsException);
+        }),
+        esClient.indices
+          .create({
+            index: SQ_EXECUTABLES_INDEX,
+            settings: {
+              index: {
+                refresh_interval: '10s',
+              },
+            },
+            mappings: {
+              _source: {
+                mode: 'synthetic',
+              } as MappingSourceField,
+              properties: {
+                'ecs.version': {
+                  type: 'keyword',
+                  index: true,
+                },
+                'Executable.file.id': {
+                  type: 'keyword',
+                  index: false,
+                },
+                'Time.created': {
+                  type: 'date',
+                  index: true,
+                },
+                'Symbolization.time.next': {
+                  type: 'date',
+                  index: true,
+                },
+                'Symbolization.retries': {
+                  type: 'short',
+                  index: true,
+                },
+              },
+            },
+          })
+          .catch(catchResourceAlreadyExistsException),
+        esClient.indices
+          .create({
+            index: LEAFFRAMES_INDEX,
+            settings: {
+              index: {
+                refresh_interval: '10s',
+              },
+            },
+            mappings: {
+              _source: {
+                mode: 'synthetic',
+              } as MappingSourceField,
+              properties: {
+                'ecs.version': {
+                  type: 'keyword',
+                  index: true,
+                },
+                'Stacktrace.frame.id': {
+                  type: 'keyword',
+                  index: false,
+                },
+                'Time.created': {
+                  type: 'date',
+                  index: true,
+                },
+                'Symbolization.time.next': {
+                  type: 'date',
+                  index: true,
+                },
+                'Symbolization.retries': {
+                  type: 'short',
+                  index: true,
+                },
+              },
+            },
+          })
+          .catch(catchResourceAlreadyExistsException),
+        esClient.indices
+          .create({
+            index: SYMBOLS_INDEX,
+            settings: {
+              index: {
+                number_of_shards: '16',
+                refresh_interval: '10s',
+              },
+            },
+            mappings: {
+              _source: {
+                enabled: true,
+              } as MappingSourceField,
+              properties: {
+                'ecs.version': {
+                  type: 'keyword',
+                  index: true,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.function.name': {
+                  // name of the function
+                  type: 'keyword',
+                  index: false,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.file.name': {
+                  // file path
+                  type: 'keyword',
+                  index: false,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.call.file.name': {
+                  // (for inlined functions) file path where inline function was called
+                  type: 'keyword',
+                  index: false,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.call.line': {
+                  // (for inlined functions) line where inline function was called
+                  type: 'integer',
+                  index: false,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.function.line': {
+                  // function start line (only available from DWARF). Currently unused.
+                  type: 'integer',
+                  index: false,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.depth': {
+                  // inline depth
+                  type: 'integer',
+                  index: false,
+                  doc_values: false,
+                  store: false,
+                },
+                // pairs of (32bit PC offset, 32bit line number) followed by 64bit PC range base at the end.
+                // To find line number for a given PC: find lowest offset such as offsetBase+PC >= offset, then read corresponding line number.
+                // offsetBase could seemingly be available from exec_pc_range (it's the first value of the pair), but it's not the case.
+                // Ranges are stored as points, which cannot be retrieve when disabling _source.
+                // See https://www.elastic.co/guide/en/elasticsearch/reference/current/point.html .
+                'Symbol.linetable.base': {
+                  // Linetable: base for offsets (64bit PC range base)
+                  type: 'unsigned_long',
+                  index: false,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.linetable.length': {
+                  // Linetable: length of range (PC range is [base, base+length))
+                  type: 'unsigned_long',
+                  index: false,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.linetable.offsets': {
+                  // Linetable: concatenated offsets (each value is ULEB128encoded)
+                  type: 'keyword',
+                  index: false,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.linetable.lines': {
+                  // Linetable: concatenated lines (each value is ULEB128 encoded)
+                  type: 'keyword',
+                  index: false,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.file.id': {
+                  // fileID. used for deletion and Symbol.exec.pcrange collision handling on symbolization
+                  type: 'keyword',
+                  index: true,
+                  doc_values: false,
+                  store: false,
+                },
+                'Symbol.exec.pcrange': {
+                  // PC ranges [begin, end)
+                  type: 'ip_range',
+                  index: true,
+                  doc_values: false,
+                  store: false,
+                },
+              },
+            },
+          })
+          .catch(catchResourceAlreadyExistsException),
+        esClient.indices
+          .create({
+            index: ILM_LOCK_INDEX,
+            settings: {
+              index: {
+                hidden: true,
+              },
+            },
+            mappings: {
+              properties: {
+                '@timestamp': {
+                  type: 'date',
+                  format: 'epoch_second',
+                },
+                phase: {
+                  type: 'keyword',
+                },
+              },
+            },
+          })
+          .catch(catchResourceAlreadyExistsException),
+      ]);
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_fleet_policy_step.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+import { merge, omit } from 'lodash';
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+import { getApmPolicy } from './get_apm_policy';
+
+async function createIngestAPIKey(esClient: ElasticsearchClient) {
+  const apiKeyResponse = await esClient.security.createApiKey({
+    name: 'profiling-manager',
+    role_descriptors: {
+      profiling_manager: {
+        indices: [
+          {
+            names: ['profiling-*', '.profiling-*'],
+            privileges: [
+              'read',
+              'create_doc',
+              'create',
+              'write',
+              'index',
+              'create_index',
+              'view_index_metadata',
+              'manage',
+            ],
+          },
+        ],
+        cluster: ['monitor'],
+      },
+    },
+  });
+
+  return atob(apiKeyResponse.encoded);
+}
+
+export function getFleetPolicyStep({
+  client,
+  soClient,
+  logger,
+  packagePolicyClient,
+}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
+  const esClient = client.getEsClient();
+  return {
+    name: 'fleet_policy',
+    hasCompleted: async () => {
+      try {
+        const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
+
+        return apmPolicy && apmPolicy?.inputs[0].config?.['apm-server'].value.profiling;
+      } catch (error) {
+        logger.debug('Could not fetch fleet policy');
+        logger.debug(error);
+        return false;
+      }
+    },
+    init: async () => {
+      const apmPolicyApiKey = await createIngestAPIKey(client.getEsClient());
+
+      const profilingApmConfig = {
+        profiling: {
+          enabled: true,
+          elasticsearch: {
+            api_key: apmPolicyApiKey,
+          },
+          metrics: {
+            elasticsearch: {
+              hosts: [
+                'https://1b6c02856ea642a6ac14499b01507233.us-east-2.aws.elastic-cloud.com:443',
+              ],
+              api_key: 'woq-IoMBRbbiEbPugtWW:_iBmc1PdSout7sf5FCkEpA',
+            },
+          },
+          keyvalue_retention: {
+            // 60 days
+            age: '1440h',
+            // 200 Gib
+            size_bytes: 200 * 1024 * 1024 * 1024,
+            execution_interval: '12h',
+          },
+        },
+      };
+
+      const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
+
+      if (!apmPolicy) {
+        throw new Error(`Could not find APM policy`);
+      }
+
+      const modifiedPolicyInputs = apmPolicy.inputs.map((input) => {
+        return input.type === 'apm'
+          ? merge({}, input, { config: { 'apm-server': { value: profilingApmConfig } } })
+          : input;
+      });
+
+      await packagePolicyClient.update(soClient, esClient, apmPolicy.id, {
+        ...omit(apmPolicy, 'id', 'revision', 'updated_at', 'updated_by'),
+        inputs: modifiedPolicyInputs,
+      });
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_ilm_step.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception';
+import ilmProfiling from './ilm_profiling.json';
+
+const LIFECYCLE_POLICY_NAME = 'profiling';
+
+export function getIlmStep({
+  client,
+  logger,
+}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
+  const esClient = client.getEsClient();
+
+  return {
+    name: 'ilm',
+    hasCompleted: () => {
+      return esClient.ilm.getLifecycle({ name: LIFECYCLE_POLICY_NAME }).then(
+        () => {
+          return Promise.resolve(true);
+        },
+        (error) => {
+          logger.debug('ILM policy not installed');
+          logger.debug(error);
+          return Promise.resolve(false);
+        }
+      );
+    },
+    init: async () => {
+      await esClient.ilm
+        .putLifecycle({
+          name: LIFECYCLE_POLICY_NAME,
+          policy: ilmProfiling,
+        })
+        .catch(catchResourceAlreadyExistsException);
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_index_templates_step.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+import { ProfilingComponentTemplateName } from './get_component_templates_step';
+
+enum ProfilingIndexTemplate {
+  Events = 'profiling-events',
+  Executables = 'profiling-executables',
+  Stacktraces = 'profiling-stacktraces',
+  Stackframes = 'profiling-stackframes',
+}
+
+export function getIndexTemplatesStep({
+  client,
+  logger,
+}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
+  const esClient = client.getEsClient();
+
+  return {
+    name: 'index_templates',
+    hasCompleted: async () => {
+      return Promise.all(
+        [
+          ProfilingIndexTemplate.Events,
+          ProfilingIndexTemplate.Executables,
+          ProfilingIndexTemplate.Stacktraces,
+          ProfilingIndexTemplate.Stackframes,
+        ].map((indexTemplateName) =>
+          esClient.indices.getIndexTemplate({
+            name: indexTemplateName,
+          })
+        )
+      ).then(
+        () => Promise.resolve(true),
+        (error) => {
+          logger.debug('Some index templates could not be fetched');
+          logger.debug(error);
+          return Promise.resolve(false);
+        }
+      );
+    },
+    init: async () => {
+      await Promise.all([
+        esClient.indices.putIndexTemplate({
+          name: ProfilingIndexTemplate.Events,
+          create: false,
+          index_patterns: [ProfilingIndexTemplate.Events + '*'],
+          data_stream: {
+            hidden: false,
+          },
+          composed_of: [ProfilingComponentTemplateName.Events, ProfilingComponentTemplateName.Ilm],
+          priority: 100,
+          _meta: {
+            description: `Index template for ${ProfilingIndexTemplate.Events}`,
+          },
+        }),
+        ...[
+          ProfilingIndexTemplate.Executables,
+          ProfilingIndexTemplate.Stacktraces,
+          ProfilingIndexTemplate.Stackframes,
+        ].map((indexTemplateName) => {
+          return esClient.indices.putIndexTemplate({
+            name: indexTemplateName,
+            // Don't fail if the index template already exists, simply overwrite the format
+            create: false,
+            index_patterns: [indexTemplateName + '*'],
+            composed_of: [indexTemplateName],
+            _meta: {
+              description: `Index template for ${indexTemplateName}`,
+            },
+          });
+        }),
+      ]);
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_is_cloud_enabled_step.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+
+export function getIsCloudEnabledStep({
+  isCloudEnabled,
+}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
+  return {
+    name: 'is_cloud',
+    hasCompleted: async () => {
+      return isCloudEnabled;
+    },
+    init: async () => {
+      if (!isCloudEnabled) {
+        throw new Error(`Universal Profiling is only available on Elastic Cloud.`);
+      }
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/steps/get_security_step.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+
+const PROFILING_READER_ROLE_NAME = 'profiling-reader';
+
+export function getSecurityStep({
+  client,
+  logger,
+}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
+  const esClient = client.getEsClient();
+
+  return {
+    name: 'security',
+    hasCompleted: () => {
+      return esClient.security
+        .getRole({
+          name: PROFILING_READER_ROLE_NAME,
+        })
+        .then(
+          () => {
+            return Promise.resolve(true);
+          },
+          (error) => {
+            logger.debug('Could not fetch profiling-reader role');
+            logger.debug(error);
+            return Promise.resolve(false);
+          }
+        );
+    },
+    init: async () => {
+      await esClient.security.putRole({
+        name: PROFILING_READER_ROLE_NAME,
+        indices: [
+          {
+            names: ['profiling-*'],
+            privileges: ['read', 'view_index_metadata'],
+          },
+        ],
+        cluster: ['monitor'],
+      });
+    },
+  };
+}

x-pack/plugins/profiling/server/lib/setup/steps/ilm_profiling.json

@@ -0,0 +1,35 @@
+{
+  "phases": {
+    "hot": {
+      "min_age": "0ms",
+      "actions": {
+        "rollover": {
+          "max_primary_shard_size": "50gb",
+          "max_age": "7d"
+        },
+        "set_priority": {
+          "priority": 100
+        }
+      }
+    },
+    "warm": {
+      "min_age": "30d",
+      "actions": {
+        "set_priority": {
+          "priority": 50
+        },
+        "shrink": {
+          "number_of_shards": 2
+        }
+      }
+    },
+    "delete": {
+      "min_age": "60d",
+      "actions": {
+        "delete": {
+          "delete_searchable_snapshot": true
+        }
+      }
+    }
+  }
+}

x-pack/plugins/profiling/server/lib/setup/steps/index.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getClusterSettingsStep } from './get_cluster_settings_step';
+import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
+import { getComponentTemplatesStep } from './get_component_templates_step';
+import { getIlmStep } from './get_ilm_step';
+import { getIndexTemplatesStep } from './get_index_templates_step';
+import { getFleetPolicyStep } from './get_fleet_policy_step';
+import { getSecurityStep } from './get_security_step';
+import { getApmPackageStep } from './get_apm_package_step';
+import { getCreateEventsDataStreamsStep } from './get_create_events_data_streams';
+import { getCreateIndicesStep } from './get_create_indices_step';
+import { getIsCloudEnabledStep } from './get_is_cloud_enabled_step';
+
+export function getProfilingSetupSteps(
+  options: ProfilingSetupStepFactoryOptions
+): ProfilingSetupStep[] {
+  return [
+    getIsCloudEnabledStep(options),
+    getApmPackageStep(options),
+    getClusterSettingsStep(options),
+    getIlmStep(options),
+    getComponentTemplatesStep(options),
+    getIndexTemplatesStep(options),
+    getCreateEventsDataStreamsStep(options),
+    getCreateIndicesStep(options),
+    getSecurityStep(options),
+    getFleetPolicyStep(options),
+  ];
+}

x-pack/plugins/profiling/server/lib/setup/types.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsClientContract } from '@kbn/core/server';
+import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
+import { Logger } from '@kbn/logging';
+import { ProfilingESClient } from '../../utils/create_profiling_es_client';
+
+export interface ProfilingSetupStep {
+  name: string;
+  init: () => Promise<void>;
+  hasCompleted: () => Promise<boolean>;
+}
+
+export interface ProfilingSetupStepFactoryOptions {
+  client: ProfilingESClient;
+  soClient: SavedObjectsClientContract;
+  packagePolicyClient: PackagePolicyClient;
+  logger: Logger;
+  spaceId: string;
+  isCloudEnabled: boolean;
+}

x-pack/plugins/profiling/server/plugin.ts

@@ -59,10 +59,15 @@ export class ProfilingPlugin
           setup: deps,
         },
         services: {
-          createProfilingEsClient: ({ request, esClient: defaultEsClient }) => {
-            const esClient = profilingSpecificEsClient
-              ? profilingSpecificEsClient.asScoped(request).asInternalUser
-              : defaultEsClient;
+          createProfilingEsClient: ({
+            request,
+            esClient: defaultEsClient,
+            useDefaultAuth = false,
+          }) => {
+            const esClient =
+              profilingSpecificEsClient && !useDefaultAuth
+                ? profilingSpecificEsClient.asScoped(request).asInternalUser
+                : defaultEsClient;
 
             return createProfilingEsClient({ request, esClient });
           },

x-pack/plugins/profiling/server/routes/index.ts

@@ -5,21 +5,19 @@
  * 2.0.
  */
 
-import type { IRouter, Logger } from '@kbn/core/server';
-import type { KibanaRequest } from '@kbn/core-http-server';
 import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import type { KibanaRequest } from '@kbn/core-http-server';
+import type { IRouter, Logger } from '@kbn/core/server';
 import {
   ProfilingPluginSetupDeps,
   ProfilingPluginStartDeps,
   ProfilingRequestHandlerContext,
 } from '../types';
 import { ProfilingESClient } from '../utils/create_profiling_es_client';
-
 import { registerCacheExecutablesRoute, registerCacheStackFramesRoute } from './cache';
-
 import { registerFlameChartSearchRoute } from './flamechart';
 import { registerTopNFunctionsSearchRoute } from './functions';
-
+import { registerSetupRoute } from './setup';
 import {
   registerTraceEventsTopNContainersSearchRoute,
   registerTraceEventsTopNDeploymentsSearchRoute,
@@ -39,6 +37,7 @@ export interface RouteRegisterParameters {
     createProfilingEsClient: (params: {
       request: KibanaRequest;
       esClient: ElasticsearchClient;
+      useDefaultAuth?: boolean;
     }) => ProfilingESClient;
   };
 }
@@ -53,4 +52,7 @@ export function registerRoutes(params: RouteRegisterParameters) {
   registerTraceEventsTopNHostsSearchRoute(params);
   registerTraceEventsTopNStackTracesSearchRoute(params);
   registerTraceEventsTopNThreadsSearchRoute(params);
+  // Setup of Profiling resources, automates the configuration of Universal Profiling
+  // and will show instructions on how to add data
+  registerSetupRoute(params);
 }

x-pack/plugins/profiling/server/routes/setup.ts

@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { eachSeries } from 'async';
+import { Logger } from '@kbn/logging';
+import { RouteRegisterParameters } from '.';
+import { getRoutePaths } from '../../common';
+import { getSetupInstructions } from '../lib/setup/get_setup_instructions';
+import { getProfilingSetupSteps } from '../lib/setup/steps';
+import { handleRouteHandlerError } from '../utils/handle_route_error_handler';
+import { hasProfilingData } from '../lib/setup/has_profiling_data';
+import { getClient } from './compat';
+import { ProfilingSetupStep } from '../lib/setup/types';
+
+function checkSteps({ steps, logger }: { steps: ProfilingSetupStep[]; logger: Logger }) {
+  return Promise.all(
+    steps.map(async (step) => {
+      try {
+        return { name: step.name, completed: await step.hasCompleted() };
+      } catch (error) {
+        logger.error(error);
+        return { name: step.name, completed: false, error: error.toString() };
+      }
+    })
+  );
+}
+
+export function registerSetupRoute({
+  router,
+  logger,
+  services: { createProfilingEsClient },
+  dependencies,
+}: RouteRegisterParameters) {
+  const paths = getRoutePaths();
+  // Check if ES resources needed for Universal Profiling to work exist
+  router.get(
+    {
+      path: paths.HasSetupESResources,
+      validate: false,
+    },
+    async (context, request, response) => {
+      try {
+        const esClient = await getClient(context);
+        logger.debug('checking if profiling ES configurations are installed');
+        const core = await context.core;
+
+        const steps = getProfilingSetupSteps({
+          client: createProfilingEsClient({
+            esClient,
+            request,
+            useDefaultAuth: true,
+          }),
+          logger,
+          packagePolicyClient: dependencies.start.fleet.packagePolicyService,
+          soClient: core.savedObjects.client,
+          spaceId: dependencies.setup.spaces.spacesService.getSpaceId(request),
+          isCloudEnabled: dependencies.setup.cloud.isCloudEnabled,
+        });
+
+        const hasDataPromise = hasProfilingData({
+          client: createProfilingEsClient({
+            esClient,
+            request,
+          }),
+        });
+
+        const stepCompletionResultsPromises = checkSteps({ steps, logger });
+
+        const hasData = await hasDataPromise;
+
+        if (hasData) {
+          return response.ok({
+            body: {
+              has_data: true,
+              has_setup: true,
+              steps: [],
+            },
+          });
+        }
+
+        const stepCompletionResults = await stepCompletionResultsPromises;
+
+        // Reply to clients if we have already created all 12 events template indices.
+        // This is kind of simplistic but can be a good first step to ensure
+        // Profiling resources will be created.
+        return response.ok({
+          body: {
+            has_setup: stepCompletionResults.every((step) => step.completed),
+            has_data: false,
+            steps: stepCompletionResults,
+          },
+        });
+      } catch (error) {
+        return handleRouteHandlerError({ error, logger, response });
+      }
+    }
+  );
+  // Configure ES resources needed by Universal Profiling using the mappings
+  router.post(
+    {
+      path: paths.HasSetupESResources,
+      validate: {},
+    },
+    async (context, request, response) => {
+      try {
+        const esClient = await getClient(context);
+        logger.info('Applying initial setup of Elasticsearch resources');
+        const steps = getProfilingSetupSteps({
+          client: createProfilingEsClient({ esClient, request, useDefaultAuth: true }),
+          logger,
+          packagePolicyClient: dependencies.start.fleet.packagePolicyService,
+          soClient: (await context.core).savedObjects.client,
+          spaceId: dependencies.setup.spaces.spacesService.getSpaceId(request),
+          isCloudEnabled: dependencies.setup.cloud.isCloudEnabled,
+        });
+
+        await eachSeries(steps, (step, cb) => {
+          logger.debug(`Executing step ${step.name}`);
+          step
+            .init()
+            .then(() => cb())
+            .catch(cb);
+        });
+
+        const checkedSteps = await checkSteps({ steps, logger });
+
+        if (checkedSteps.every((step) => step.completed)) {
+          return response.ok();
+        }
+
+        return response.custom({
+          statusCode: 500,
+          body: {
+            message: `Failed to complete all steps`,
+            steps: checkedSteps,
+          },
+        });
+      } catch (error) {
+        return handleRouteHandlerError({ error, logger, response });
+      }
+    }
+  );
+  // Show users the instructions on how to setup Universal Profiling agents
+  router.get(
+    {
+      path: paths.SetupDataCollectionInstructions,
+      validate: false,
+    },
+    async (context, request, response) => {
+      try {
+        const setupInstructions = await getSetupInstructions({
+          packagePolicyClient: dependencies.start.fleet.packagePolicyService,
+          soClient: (await context.core).savedObjects.client,
+        });
+
+        return response.ok({ body: setupInstructions });
+      } catch (error) {
+        return handleRouteHandlerError({ error, logger, response });
+      }
+    }
+  );
+}

x-pack/plugins/profiling/server/routes/topn.test.ts

@@ -53,6 +53,7 @@ describe('TopN data from Elasticsearch', () => {
           },
         }) as Promise<any>
     ),
+    getEsClient: jest.fn(() => context.elasticsearch.client.asCurrentUser),
   };
   const logger = loggerMock.create();
 

x-pack/plugins/profiling/server/types.ts

@@ -5,21 +5,32 @@
  * 2.0.
  */
 
-import { RequestHandlerContext } from '@kbn/core/server';
-import { PluginSetupContract as FeaturesPluginSetup } from '@kbn/features-plugin/server';
-import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server';
+import { CustomRequestHandlerContext } from '@kbn/core-http-request-handler-context-server';
+import type { PluginSetupContract as FeaturesPluginSetup } from '@kbn/features-plugin/server';
+import type { ObservabilityPluginSetup } from '@kbn/observability-plugin/server';
+import { SpacesPluginStart, SpacesPluginSetup } from '@kbn/spaces-plugin/server';
+import { CloudSetup, CloudStart } from '@kbn/cloud-plugin/server';
+import { FleetSetupContract, FleetStartContract } from '@kbn/fleet-plugin/server';
 
 export interface ProfilingPluginSetupDeps {
   observability: ObservabilityPluginSetup;
   features: FeaturesPluginSetup;
+  spaces: SpacesPluginSetup;
+  cloud: CloudSetup;
+  fleet: FleetSetupContract;
 }
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface ProfilingPluginStartDeps {}
+export interface ProfilingPluginStartDeps {
+  observability: {};
+  features: {};
+  spaces: SpacesPluginStart;
+  cloud: CloudStart;
+  fleet: FleetStartContract;
+}
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface ProfilingPluginSetup {}
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface ProfilingPluginStart {}
 
-export type ProfilingRequestHandlerContext = RequestHandlerContext;
+export type ProfilingRequestHandlerContext = CustomRequestHandlerContext<{}>;

x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts

@@ -40,6 +40,7 @@ export interface ProfilingESClient {
     query: QueryDslQueryContainer;
     sampleSize: number;
   }): Promise<StackTraceResponse>;
+  getEsClient(): ElasticsearchClient;
 }
 
 export function createProfilingEsClient({
@@ -116,5 +117,8 @@ export function createProfilingEsClient({
 
       return unwrapEsResponse(promise) as Promise<StackTraceResponse>;
     },
+    getEsClient() {
+      return esClient;
+    },
   };
 }

x-pack/plugins/profiling/tsconfig.json

@@ -9,7 +9,8 @@
     "common/**/*.ts",
     "public/**/*.ts",
     "public/**/*.tsx",
-    "server/**/*.ts"
+    "server/**/*.ts",
+    "server/**/*.json"
   ],
   "kbn_references": [
     "@kbn/core",
@@ -19,7 +20,6 @@
     "@kbn/observability-plugin",
     "@kbn/i18n",
     "@kbn/es-types",
-    "@kbn/core-http-browser",
     "@kbn/data-views-plugin",
     "@kbn/charts-plugin",
     "@kbn/typed-react-router-config",
@@ -34,6 +34,12 @@
     "@kbn/core-http-server",
     "@kbn/apm-utils",
     "@kbn/core-elasticsearch-server",
+    "@kbn/fleet-plugin",
+    "@kbn/shared-ux-page-no-data-types",
+    "@kbn/es-errors",
+    "@kbn/core-http-request-handler-context-server",
+    "@kbn/spaces-plugin",
+    "@kbn/cloud-plugin",
     // add references to other TypeScript projects the plugin depends on
 
     // requiredPlugins from ./kibana.json