[Logs Onboarding] Adds install shipper step for custom logs (#158386)

Closes #154937

This PR is an extension of
[#157802](https://github.com/elastic/kibana/pull/157802) which was
reverted because public Api endpoint versioning problems.

![Screenshot 2023-05-15 at 5 02 29
PM](62484fb3-e02f-410d-aa7a-86bcc4dc0b03)

---------

Co-authored-by: Oliver Gupte <oliver.gupte@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -2097,6 +2097,18 @@
       }
     }
   },
+  "observability-onboarding-state": {
+    "properties": {
+      "state": {
+        "type": "object",
+        "dynamic": false
+      },
+      "progress": {
+        "type": "object",
+        "dynamic": false
+      }
+    }
+  },
   "ml-job": {
     "properties": {
       "job_id": {

src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts

@@ -119,6 +119,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "ml-module": "2225cbb4bd508ea5f69db4b848be9d8a74b60198",
         "ml-trained-model": "482195cefd6b04920e539d34d7356d22cb68e4f3",
         "monitoring-telemetry": "5d91bf75787d9d4dd2fae954d0b3f76d33d2e559",
+        "observability-onboarding-state": "c2a7439293913d69cc286a8f8f9885bc2dd9682f",
         "osquery-manager-usage-metric": "983bcbc3b7dda0aad29b20907db233abba709bcc",
         "osquery-pack": "6ab4358ca4304a12dcfc1777c8135b75cffb4397",
         "osquery-pack-asset": "b14101d3172c4b60eb5404696881ce5275c84152",

src/core/server/integration_tests/saved_objects/migrations/group3/dot_kibana_split.test.ts

@@ -233,6 +233,7 @@ describe('split .kibana index into multiple system indices', () => {
             "ml-module",
             "ml-trained-model",
             "monitoring-telemetry",
+            "observability-onboarding-state",
             "osquery-manager-usage-metric",
             "osquery-pack",
             "osquery-pack-asset",

src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts

@@ -89,6 +89,7 @@ const previouslyRegisteredTypes = [
   'ml-module',
   'ml-telemetry',
   'monitoring-telemetry',
+  'observability-onboarding-state',
   'osquery-pack',
   'osquery-pack-asset',
   'osquery-saved-query',

x-pack/plugins/observability_onboarding/public/assets/standalone_agent_setup.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+API_KEY_ENCODED=$1
+API_ENDPOINT=$2
+
+updateStepProgress() {
+  echo "  GET $API_ENDPOINT/step/$1?status=$2"
+  curl --request GET \
+    --url "$API_ENDPOINT/step/$1?status=$2 2023-05-24" \
+    --header "Authorization: ApiKey $API_KEY_ENCODED" \
+    --header "Content-Type: application/json" \
+    --header "kbn-xsrf: true"
+  echo ""
+}
+
+echo "Downloading Elastic Agent"
+# https://www.elastic.co/guide/en/fleet/8.7/install-standalone-elastic-agent.html
+curl -L -O https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-8.7.1-linux-x86_64.tar.gz
+updateStepProgress "ea-download" "success"
+echo "Extracting Elastic Agent"
+tar xzvf elastic-agent-8.7.1-linux-x86_64.tar.gz
+updateStepProgress "ea-extract" "success"
+echo "Installing Elastic Agent"
+cd elastic-agent-8.7.1-linux-x86_64
+./elastic-agent install -f
+updateStepProgress "ea-install" "success"
+echo "Sending status to Kibana..."
+updateStepProgress "ea-status" "active"

x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx

@@ -5,20 +5,17 @@
  * 2.0.
  */
 
-import React, { PropsWithChildren, useState } from 'react';
+import { Buffer } from 'buffer';
+import { flatten, zip } from 'lodash';
+import React, { useState } from 'react';
 import {
-  EuiTitle,
   EuiText,
   EuiButton,
-  EuiFlexGroup,
-  EuiFlexItem,
-  EuiHorizontalRule,
   EuiSpacer,
-  EuiCard,
-  EuiIcon,
-  EuiIconProps,
   EuiButtonGroup,
   EuiCodeBlock,
+  EuiSteps,
+  EuiSkeletonRectangle,
 } from '@elastic/eui';
 import {
   StepPanel,
@@ -26,178 +23,221 @@ import {
   StepPanelFooter,
 } from '../../../shared/step_panel';
 import { useWizard } from '.';
+import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
 
+type ElasticAgentPlatform = 'linux-tar' | 'macos' | 'windows';
 export function InstallElasticAgent() {
-  const { goToStep, goBack, getState, setState } = useWizard();
+  const { goToStep, goBack, getState, CurrentStep } = useWizard();
   const wizardState = getState();
-  const [elasticAgentPlatform, setElasticAgentPlatform] = useState(
+  const [elasticAgentPlatform, setElasticAgentPlatform] =
-    wizardState.elasticAgentPlatform
+    useState<ElasticAgentPlatform>('linux-tar');
-  );
-  const [alternativeShippers, setAlternativeShippers] = useState(
-    wizardState.alternativeShippers
-  );
 
   function onContinue() {
-    setState({ ...getState(), elasticAgentPlatform, alternativeShippers });
     goToStep('collectLogs');
   }
 
-  function createAlternativeShipperToggle(
-    type: NonNullable<keyof typeof alternativeShippers>
-  ) {
-    return () => {
-      setAlternativeShippers({
-        ...alternativeShippers,
-        [type]: !alternativeShippers[type],
-      });
-    };
-  }
-
   function onBack() {
     goBack();
   }
 
-  return (
+  const { data: installShipperSetup, status: installShipperSetupStatus } =
-    <StepPanel
+    useFetcher((callApi) => {
-      title="Install the Elastic Agent"
+      if (CurrentStep === InstallElasticAgent) {
-      panelFooter={
+        return callApi(
-        <StepPanelFooter
+          'POST /internal/observability_onboarding/custom_logs/install_shipper_setup',
-          items={[
+          {
-            <EuiButton color="ghost" fill onClick={onBack}>
+            params: {
-              Back
+              body: {
-            </EuiButton>,
+                name: wizardState.datasetName,
-            <EuiButton color="primary" fill onClick={onContinue}>
+                state: {
-              Continue
+                  datasetName: wizardState.datasetName,
-            </EuiButton>,
+                  namespace: wizardState.namespace,
-          ]}
+                  customConfigurations: wizardState.customConfigurations,
-        />
+                  logFilePaths: wizardState.logFilePaths,
+                },
+              },
+            },
+          }
+        );
       }
-    >
+    }, []);
+
+  const { data: yamlConfig = '', status: yamlConfigStatus } = useFetcher(
+    (callApi) => {
+      if (CurrentStep === InstallElasticAgent && installShipperSetup) {
+        return callApi(
+          'GET /api/observability_onboarding/elastic_agent/config 2023-05-24',
+          {
+            headers: {
+              authorization: `ApiKey ${installShipperSetup.apiKeyEncoded}`,
+            },
+          }
+        );
+      }
+    },
+    [installShipperSetup?.apiKeyId, installShipperSetup?.apiKeyEncoded]
+  );
+
+  const apiKeyEncoded = installShipperSetup?.apiKeyEncoded;
+
+  return (
+    <StepPanel title="Install shipper to collect data">
       <StepPanelContent>
         <EuiText color="subdued">
           <p>
-            Select a platform and run the command to install, enroll, and start
+            Add Elastic Agent to your hosts to begin sending data to your
-            the Elastic Agent. Do this for each host. For other platforms, see
+            Elastic Cloud. Run standalone if you want to download and manage
-            our downloads page. Review host requirements and other installation
+            each agent configuration file on your own, or enroll in Fleet, for
-            options.
+            centralized management of all your agents through our Fleet managed
+            interface.
           </p>
         </EuiText>
         <EuiSpacer size="m" />
-        <EuiButtonGroup
+        <EuiSteps
-          isFullWidth
+          steps={[
-          legend="Choose platform"
+            {
-          options={[
+              title: 'Install the Elastic Agent',
-            { id: 'linux-tar', label: 'Linux Tar' },
+              status:
-            { id: 'macos', label: 'MacOs' },
+                installShipperSetupStatus === FETCH_STATUS.LOADING
-            { id: 'windows', label: 'Windows' },
+                  ? 'loading'
-            { id: 'deb', label: 'DEB' },
+                  : 'current',
-            { id: 'rpm', label: 'RPM' },
+              children: (
+                <>
+                  <EuiText color="subdued">
+                    <p>
+                      Select a platform and run the command to install in your
+                      Terminal, enroll, and start the Elastic Agent. Do this for
+                      each host. For other platforms, see our downloads page.
+                      Review host requirements and other installation options.
+                    </p>
+                  </EuiText>
+                  <EuiSpacer size="m" />
+                  <EuiButtonGroup
+                    isFullWidth
+                    legend="Choose platform"
+                    options={[
+                      { id: 'linux-tar', label: 'Linux' },
+                      { id: 'macos', label: 'MacOs', isDisabled: true },
+                      { id: 'windows', label: 'Windows', isDisabled: true },
+                    ]}
+                    type="single"
+                    idSelected={elasticAgentPlatform}
+                    onChange={(id: string) =>
+                      setElasticAgentPlatform(id as typeof elasticAgentPlatform)
+                    }
+                  />
+                  <EuiSpacer size="m" />
+                  <EuiSkeletonRectangle
+                    isLoading={
+                      installShipperSetupStatus === FETCH_STATUS.LOADING
+                    }
+                    contentAriaLabel="Command to install elastic agent"
+                    width="100%"
+                    height={80}
+                    borderRadius="s"
+                  >
+                    <EuiCodeBlock language="bash" isCopyable>
+                      {getInstallShipperCommand({
+                        elasticAgentPlatform,
+                        apiKeyEncoded,
+                        apiEndpoint: installShipperSetup?.apiEndpoint,
+                        scriptDownloadUrl:
+                          installShipperSetup?.scriptDownloadUrl,
+                      })}
+                    </EuiCodeBlock>
+                  </EuiSkeletonRectangle>
+                </>
+              ),
+            },
+            {
+              title: 'Configure the agent',
+              status:
+                yamlConfigStatus === FETCH_STATUS.LOADING
+                  ? 'loading'
+                  : 'incomplete',
+              children: (
+                <>
+                  <EuiText color="subdued">
+                    <p>
+                      Copy the config below to the elastic agent.yml on the host
+                      where the Elastic Agent is installed.
+                    </p>
+                  </EuiText>
+                  <EuiSpacer size="m" />
+                  <EuiSkeletonRectangle
+                    isLoading={yamlConfigStatus === FETCH_STATUS.LOADING}
+                    contentAriaLabel="Elastic agent yaml configuration"
+                    width="100%"
+                    height={300}
+                    borderRadius="s"
+                  >
+                    <EuiCodeBlock language="yaml" isCopyable>
+                      {yamlConfig}
+                    </EuiCodeBlock>
+                  </EuiSkeletonRectangle>
+                  <EuiSpacer size="m" />
+                  <EuiButton
+                    iconType="download"
+                    color="primary"
+                    href={`data:application/yaml;base64,${Buffer.from(
+                      yamlConfig,
+                      'utf8'
+                    ).toString('base64')}`}
+                    download="elastic-agent.yml"
+                    target="_blank"
+                  >
+                    Download config file
+                  </EuiButton>
+                </>
+              ),
+            },
           ]}
-          type="single"
-          idSelected={elasticAgentPlatform}
-          onChange={(id: string) =>
-            setElasticAgentPlatform(id as typeof elasticAgentPlatform)
-          }
         />
-        <EuiSpacer size="m" />
-        <EuiCodeBlock language="html" isCopyable>
-          {PLATFORM_COMMAND[elasticAgentPlatform]}
-        </EuiCodeBlock>
-        <EuiHorizontalRule margin="l" />
-        <LogsTypeSection title="Or select alternative shippers" description="">
-          <EuiFlexGroup>
-            <EuiFlexItem>
-              <OptionCard
-                title="Filebeat"
-                iconType="document"
-                onClick={createAlternativeShipperToggle('filebeat')}
-                isSelected={alternativeShippers.filebeat}
-              />
-            </EuiFlexItem>
-            <EuiFlexItem>
-              <OptionCard
-                title="fluentbit"
-                iconType="package"
-                onClick={createAlternativeShipperToggle('fluentbit')}
-                isSelected={alternativeShippers.fluentbit}
-              />
-            </EuiFlexItem>
-          </EuiFlexGroup>
-          <EuiSpacer size="m" />
-          <EuiFlexGroup>
-            <EuiFlexItem>
-              <OptionCard
-                title="Logstash"
-                iconType="logstashIf"
-                onClick={createAlternativeShipperToggle('logstash')}
-                isSelected={alternativeShippers.logstash}
-              />
-            </EuiFlexItem>
-            <EuiFlexItem>
-              <OptionCard
-                title="Fluentd"
-                iconType="package"
-                onClick={createAlternativeShipperToggle('fluentd')}
-                isSelected={alternativeShippers.fluentd}
-              />
-            </EuiFlexItem>
-          </EuiFlexGroup>
-        </LogsTypeSection>
       </StepPanelContent>
+      <StepPanelFooter
+        items={[
+          <EuiButton color="ghost" fill onClick={onBack}>
+            Back
+          </EuiButton>,
+          <EuiButton color="primary" fill onClick={onContinue}>
+            Continue
+          </EuiButton>,
+        ]}
+      />
     </StepPanel>
   );
 }
 
-function LogsTypeSection({
+function getInstallShipperCommand({
-  title,
+  elasticAgentPlatform,
-  description,
+  apiKeyEncoded = '$API_KEY',
-  children,
+  apiEndpoint = '$API_ENDPOINT',
-}: PropsWithChildren<{ title: string; description: string }>) {
+  scriptDownloadUrl = '$SCRIPT_DOWNLOAD_URL',
-  return (
-    <>
-      <EuiTitle size="s">
-        <h3>{title}</h3>
-      </EuiTitle>
-      <EuiSpacer size="m" />
-      <EuiText color="subdued">
-        <p>{description}</p>
-      </EuiText>
-      <EuiSpacer size="m" />
-      {children}
-    </>
-  );
-}
-
-function OptionCard({
-  title,
-  iconType,
-  onClick,
-  isSelected,
 }: {
-  title: string;
+  elasticAgentPlatform: ElasticAgentPlatform;
-  iconType: EuiIconProps['type'];
+  apiKeyEncoded: string | undefined;
-  onClick: () => void;
+  apiEndpoint: string | undefined;
-  isSelected: boolean;
+  scriptDownloadUrl: string | undefined;
 }) {
-  return (
+  const setupScriptFilename = 'standalone_agent_setup.sh';
-    <EuiCard
+  const PLATFORM_COMMAND: Record<ElasticAgentPlatform, string> = {
-      layout="horizontal"
+    'linux-tar': oneLine`
-      icon={<EuiIcon type={iconType} size="l" />}
+      curl ${scriptDownloadUrl} -o ${setupScriptFilename} &&
-      title={title}
+      sudo bash ${setupScriptFilename} ${apiKeyEncoded} ${apiEndpoint}
-      titleSize="xs"
+    `,
-      paddingSize="m"
+    macos: oneLine`
-      style={{ height: 56 }}
+      curl -O https://elastic.co/agent-setup.sh &&
-      onClick={onClick}
+      sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==
-      hasBorder={true}
+    `,
-      display={isSelected ? 'primary' : undefined}
+    windows: oneLine`
-    />
+      curl -O https://elastic.co/agent-setup.sh &&
-  );
+      sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==
+    `,
+  };
+  return PLATFORM_COMMAND[elasticAgentPlatform];
 }
 
-const PLATFORM_COMMAND = {
+function oneLine(parts: TemplateStringsArray, ...args: string[]) {
-  'linux-tar': `curl -O https://elastic.co/agent-setup.sh && sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==`,
+  const str = flatten(zip(parts, args)).join('');
-  macos: `curl -O https://elastic.co/agent-setup.sh && sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==`,
+  return str.replace(/\s+/g, ' ').trim();
-  windows: `curl -O https://elastic.co/agent-setup.sh && sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==`,
+}
-  deb: `curl -O https://elastic.co/agent-setup.sh && sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==`,
-  rpm: `curl -O https://elastic.co/agent-setup.sh && sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==`,
-} as const;

x-pack/plugins/observability_onboarding/public/context/create_wizard_context.tsx

@@ -14,7 +14,7 @@ import React, {
   useRef,
 } from 'react';
 
-interface WizardContext<T, StepKey extends string> {
+export interface WizardContext<T, StepKey extends string> {
   CurrentStep: ComponentType;
   goToStep: (step: StepKey) => void;
   goBack: () => void;
@@ -172,8 +172,9 @@ export function createWizardContext<
   }
 
   function useWizard() {
-    const { CurrentStep: _, ...rest } = useContext(context);
+    // const { CurrentStep: _, ...rest } = useContext(context);
-    return rest;
+    // return rest;
+    return useContext(context);
   }
 
   return { context, Provider, Step, useWizard };

x-pack/plugins/observability_onboarding/server/lib/get_authentication_api_key.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KibanaRequest } from '@kbn/core-http-server';
+import { HTTPAuthorizationHeader } from '@kbn/security-plugin/server';
+
+export const getAuthenticationAPIKey = (request: KibanaRequest) => {
+  const authorizationHeader = HTTPAuthorizationHeader.parseFromRequest(request);
+  if (authorizationHeader && authorizationHeader.credentials) {
+    const apiKey = Buffer.from(authorizationHeader.credentials, 'base64')
+      .toString()
+      .split(':');
+    return {
+      apiKeyId: apiKey[0],
+      apiKey: apiKey[1],
+    };
+  }
+  throw new Error('Authorization header is missing');
+};

x-pack/plugins/observability_onboarding/server/plugin.ts

@@ -23,6 +23,7 @@ import {
   ObservabilityOnboardingPluginStartDependencies,
 } from './types';
 import { ObservabilityOnboardingConfig } from '.';
+import { observabilityOnboardingState } from './saved_objects/observability_onboarding_status';
 
 export class ObservabilityOnboardingPlugin
   implements
@@ -47,6 +48,8 @@ export class ObservabilityOnboardingPlugin
   ) {
     this.logger.debug('observability_onboarding: Setup');
 
+    core.savedObjects.registerType(observabilityOnboardingState);
+
     const resourcePlugins = mapValues(plugins, (value, key) => {
       return {
         setup: value,

x-pack/plugins/observability_onboarding/server/routes/custom_logs/create_shipper_api_key.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+
+export function createShipperApiKey(
+  esClient: ElasticsearchClient,
+  name: string
+) {
+  // Based on https://www.elastic.co/guide/en/fleet/master/grant-access-to-elasticsearch.html#create-api-key-standalone-agent
+  return esClient.security.createApiKey({
+    body: {
+      name: `standalone_agent_custom_logs_${name}`,
+      metadata: { application: 'logs' },
+      role_descriptors: {
+        standalone_agent: {
+          cluster: ['monitor'],
+          indices: [
+            {
+              names: ['logs-*-*', 'metrics-*-*'],
+              privileges: ['auto_configure', 'create_doc'],
+            },
+          ],
+        },
+      },
+    },
+  });
+}

x-pack/plugins/observability_onboarding/server/routes/custom_logs/find_latest_observability_onboarding_state.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsClientContract } from '@kbn/core/server';
+import {
+  OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+  ObservabilityOnboardingState,
+  SavedObservabilityOnboardingState,
+} from '../../saved_objects/observability_onboarding_status';
+
+export async function findLatestObservabilityOnboardingState({
+  savedObjectsClient,
+}: {
+  savedObjectsClient: SavedObjectsClientContract;
+}): Promise<SavedObservabilityOnboardingState | undefined> {
+  const result = await savedObjectsClient.find<ObservabilityOnboardingState>({
+    type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+    page: 1,
+    perPage: 1,
+    sortField: `updated_at`,
+    sortOrder: 'desc',
+  });
+  if (result.total === 0) {
+    return undefined;
+  }
+  const { id, updated_at: updatedAt, attributes } = result.saved_objects[0];
+  return {
+    id,
+    updatedAt: updatedAt ? Date.parse(updatedAt) : 0,
+    ...attributes,
+  };
+}

x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_es_hosts.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Client } from '@elastic/elasticsearch';
+import { CloudSetup } from '@kbn/cloud-plugin/server';
+import { decodeCloudId } from '@kbn/fleet-plugin/common';
+
+const DEFAULT_ES_HOSTS = ['http://localhost:9200'];
+
+export function getESHosts({
+  cloudSetup,
+  esClient,
+}: {
+  cloudSetup: CloudSetup;
+  esClient: Client;
+}): string[] {
+  if (cloudSetup.cloudId) {
+    const cloudUrl = decodeCloudId(cloudSetup.cloudId)?.elasticsearchUrl;
+    if (cloudUrl) {
+      return [cloudUrl];
+    }
+  }
+
+  const aliveConnections = esClient.connectionPool.connections.filter(
+    ({ status }) => status === 'alive'
+  );
+  if (aliveConnections.length) {
+    return aliveConnections.map(({ url }) => {
+      const { protocol, host } = new URL(url);
+      return `${protocol}//${host}`;
+    });
+  }
+
+  return DEFAULT_ES_HOSTS;
+}

x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_kibana_url.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreStart } from '@kbn/core/server';
+
+export function getKibanaUrl({ http }: CoreStart, path = '') {
+  const basePath = http.basePath;
+  const { protocol, hostname, port } = http.getServerInfo();
+  return `${protocol}://${hostname}:${port}${basePath.prepend(path)}`;
+}

x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_observability_onboarding_state.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsClientContract } from '@kbn/core/server';
+import {
+  OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+  ObservabilityOnboardingState,
+  SavedObservabilityOnboardingState,
+} from '../../saved_objects/observability_onboarding_status';
+
+export async function getObservabilityOnboardingState({
+  savedObjectsClient,
+  apiKeyId,
+}: {
+  savedObjectsClient: SavedObjectsClientContract;
+  apiKeyId: string;
+}): Promise<SavedObservabilityOnboardingState | undefined> {
+  try {
+    const result = await savedObjectsClient.get<ObservabilityOnboardingState>(
+      OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+      apiKeyId
+    );
+    const { id, updated_at: updatedAt, attributes } = result;
+    return {
+      id,
+      updatedAt: updatedAt ? Date.parse(updatedAt) : 0,
+      ...attributes,
+    };
+  } catch (error) {
+    return undefined;
+  }
+}

x-pack/plugins/observability_onboarding/server/routes/custom_logs/route.ts

@@ -0,0 +1,258 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+import Boom from '@hapi/boom';
+import type { Client } from '@elastic/elasticsearch';
+import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
+import { getESHosts } from './get_es_hosts';
+import { getKibanaUrl } from './get_kibana_url';
+import { createShipperApiKey } from './create_shipper_api_key';
+import { saveObservabilityOnboardingState } from './save_observability_onboarding_state';
+import {
+  ObservabilityOnboardingState,
+  OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+  SavedObservabilityOnboardingState,
+} from '../../saved_objects/observability_onboarding_status';
+import { getObservabilityOnboardingState } from './get_observability_onboarding_state';
+import { findLatestObservabilityOnboardingState } from './find_latest_observability_onboarding_state';
+import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key';
+
+const createApiKeyRoute = createObservabilityOnboardingServerRoute({
+  endpoint:
+    'POST /internal/observability_onboarding/custom_logs/install_shipper_setup',
+  options: { tags: [] },
+  params: t.type({
+    body: t.type({
+      name: t.string,
+      state: t.record(t.string, t.unknown),
+    }),
+  }),
+  async handler(resources): Promise<{
+    apiKeyId: string;
+    apiKeyEncoded: string;
+    apiEndpoint: string;
+    scriptDownloadUrl: string;
+    esHost: string;
+  }> {
+    const {
+      context,
+      params: {
+        body: { name, state },
+      },
+      core,
+      plugins,
+      request,
+    } = resources;
+    const coreStart = await core.start();
+    const scriptDownloadUrl = getKibanaUrl(
+      coreStart,
+      '/plugins/observabilityOnboarding/assets/standalone_agent_setup.sh'
+    );
+    const apiEndpoint = getKibanaUrl(
+      coreStart,
+      '/api/observability_onboarding/custom_logs'
+    );
+    const {
+      elasticsearch: { client },
+    } = await context.core;
+    const { id: apiKeyId, encoded: apiKeyEncoded } = await createShipperApiKey(
+      client.asCurrentUser,
+      name
+    );
+    const [esHost] = getESHosts({
+      cloudSetup: plugins.cloud.setup,
+      esClient: coreStart.elasticsearch.client.asInternalUser as Client,
+    });
+
+    const savedObjectsClient = coreStart.savedObjects.getScopedClient(request);
+
+    await saveObservabilityOnboardingState({
+      savedObjectsClient,
+      apiKeyId,
+      observabilityOnboardingState: { state } as ObservabilityOnboardingState,
+    });
+
+    return {
+      apiKeyId, // key the status off this
+      apiKeyEncoded,
+      apiEndpoint,
+      scriptDownloadUrl,
+      esHost,
+    };
+  },
+});
+
+const stepProgressUpdateRoute = createObservabilityOnboardingServerRoute({
+  endpoint:
+    'GET /api/observability_onboarding/custom_logs/step/{name} 2023-05-24',
+  options: { tags: [] },
+  params: t.type({
+    path: t.type({
+      name: t.string,
+    }),
+    query: t.type({
+      status: t.string,
+    }),
+  }),
+  async handler(resources): Promise<object> {
+    const {
+      params: {
+        path: { name },
+        query: { status },
+      },
+      request,
+      core,
+    } = resources;
+    const { apiKeyId } = getAuthenticationAPIKey(request);
+    const coreStart = await core.start();
+    const savedObjectsClient =
+      coreStart.savedObjects.createInternalRepository();
+
+    const savedObservabilityOnboardingState =
+      await getObservabilityOnboardingState({
+        savedObjectsClient,
+        apiKeyId,
+      });
+
+    if (!savedObservabilityOnboardingState) {
+      return {
+        message:
+          'Unable to report setup progress - onboarding session not found.',
+      };
+    }
+
+    const { id, updatedAt, ...observabilityOnboardingState } =
+      savedObservabilityOnboardingState;
+    await saveObservabilityOnboardingState({
+      savedObjectsClient,
+      apiKeyId,
+      observabilityOnboardingState: {
+        ...observabilityOnboardingState,
+        progress: {
+          ...observabilityOnboardingState.progress,
+          [name]: status,
+        },
+      },
+    });
+    return { name, status };
+  },
+});
+
+const getStateRoute = createObservabilityOnboardingServerRoute({
+  endpoint: 'GET /internal/observability_onboarding/custom_logs/state',
+  options: { tags: [] },
+  params: t.type({
+    query: t.type({
+      apiKeyId: t.string,
+    }),
+  }),
+  async handler(resources): Promise<{
+    savedObservabilityOnboardingState: SavedObservabilityOnboardingState | null;
+  }> {
+    const {
+      params: {
+        query: { apiKeyId },
+      },
+      core,
+    } = resources;
+    const coreStart = await core.start();
+    const savedObjectsClient =
+      coreStart.savedObjects.createInternalRepository();
+    const savedObservabilityOnboardingState =
+      (await getObservabilityOnboardingState({
+        savedObjectsClient,
+        apiKeyId,
+      })) || null;
+    return { savedObservabilityOnboardingState };
+  },
+});
+
+const getLatestStateRoute = createObservabilityOnboardingServerRoute({
+  endpoint: 'GET /internal/observability_onboarding/custom_logs/state/latest',
+  options: { tags: [] },
+  async handler(resources): Promise<{
+    savedObservabilityOnboardingState: SavedObservabilityOnboardingState | null;
+  }> {
+    const { core } = resources;
+    const coreStart = await core.start();
+    const savedObjectsClient =
+      coreStart.savedObjects.createInternalRepository();
+    const savedObservabilityOnboardingState =
+      (await findLatestObservabilityOnboardingState({ savedObjectsClient })) ||
+      null;
+    return { savedObservabilityOnboardingState };
+  },
+});
+
+const customLogsExistsRoute = createObservabilityOnboardingServerRoute({
+  endpoint: 'GET /internal/observability_onboarding/custom_logs/exists',
+  options: { tags: [] },
+  params: t.type({
+    query: t.type({
+      dataset: t.string,
+      namespace: t.string,
+    }),
+  }),
+  async handler(resources): Promise<{ exists: boolean }> {
+    const {
+      core,
+      request,
+      params: {
+        query: { dataset, namespace },
+      },
+    } = resources;
+    const coreStart = await core.start();
+    const esClient =
+      coreStart.elasticsearch.client.asScoped(request).asCurrentUser;
+    try {
+      const { hits } = await esClient.search({
+        index: `logs-${dataset}-${namespace}`,
+        terminate_after: 1,
+      });
+      const total = hits.total as { value: number };
+      return { exists: total.value > 0 };
+    } catch (error) {
+      if (error.statusCode === 404) {
+        return { exists: false };
+      }
+      throw Boom.boomify(error, {
+        statusCode: error.statusCode,
+        message: error.message,
+        data: error.body,
+      });
+    }
+  },
+});
+
+const deleteStatesRoute = createObservabilityOnboardingServerRoute({
+  endpoint: 'DELETE /internal/observability_onboarding/custom_logs/states',
+  options: { tags: [] },
+  async handler(resources): Promise<object> {
+    const { core } = resources;
+    const coreStart = await core.start();
+    const savedObjectsClient =
+      coreStart.savedObjects.createInternalRepository();
+    const findStatesResult =
+      await savedObjectsClient.find<ObservabilityOnboardingState>({
+        type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+      });
+    const bulkDeleteResult = await savedObjectsClient.bulkDelete(
+      findStatesResult.saved_objects
+    );
+    return { bulkDeleteResult };
+  },
+});
+
+export const customLogsRouteRepository = {
+  ...createApiKeyRoute,
+  ...stepProgressUpdateRoute,
+  ...getStateRoute,
+  ...getLatestStateRoute,
+  ...customLogsExistsRoute,
+  ...deleteStatesRoute,
+};

x-pack/plugins/observability_onboarding/server/routes/custom_logs/save_observability_onboarding_state.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsClientContract } from '@kbn/core/server';
+import {
+  OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+  ObservabilityOnboardingState,
+  SavedObservabilityOnboardingState,
+} from '../../saved_objects/observability_onboarding_status';
+
+interface Options {
+  savedObjectsClient: SavedObjectsClientContract;
+  observabilityOnboardingState: ObservabilityOnboardingState;
+  apiKeyId: string;
+}
+export async function saveObservabilityOnboardingState({
+  savedObjectsClient,
+  observabilityOnboardingState,
+  apiKeyId,
+}: Options): Promise<SavedObservabilityOnboardingState> {
+  const {
+    id,
+    attributes,
+    updated_at: updatedAt,
+  } = await savedObjectsClient.update<ObservabilityOnboardingState>(
+    OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+    apiKeyId,
+    observabilityOnboardingState,
+    { upsert: observabilityOnboardingState }
+  );
+  return {
+    id,
+    ...(attributes as ObservabilityOnboardingState),
+    updatedAt: updatedAt ? Date.parse(updatedAt) : 0,
+  };
+}

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { dump, load } from 'js-yaml';
+
+export const generateYml = ({
+  datasetName = '',
+  namespace = '',
+  customConfigurations,
+  logFilePaths = [],
+  apiKey,
+  esHost,
+  logfileId,
+}: {
+  datasetName?: string;
+  namespace?: string;
+  customConfigurations?: string;
+  logFilePaths?: string[];
+  apiKey: string;
+  esHost: string[];
+  logfileId: string;
+}) => {
+  const customConfigYaml = load(customConfigurations ?? '');
+
+  return dump({
+    ...{
+      outputs: {
+        default: {
+          type: 'elasticsearch',
+          hosts: esHost,
+          api_key: apiKey,
+        },
+      },
+      inputs: [
+        {
+          id: logfileId,
+          type: 'logfile',
+          data_stream: {
+            namespace,
+          },
+          streams: [
+            {
+              id: `logs-onboarding-${datasetName}`,
+              data_stream: {
+                dataset: datasetName,
+              },
+              paths: logFilePaths,
+            },
+          ],
+        },
+      ],
+    },
+    ...customConfigYaml,
+  });
+};

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Client } from '@elastic/elasticsearch';
+import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key';
+import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
+import { findLatestObservabilityOnboardingState } from '../custom_logs/find_latest_observability_onboarding_state';
+import { getESHosts } from '../custom_logs/get_es_hosts';
+import { generateYml } from './generate_yml';
+
+const generateConfig = createObservabilityOnboardingServerRoute({
+  endpoint: 'GET /api/observability_onboarding/elastic_agent/config 2023-05-24',
+  options: { tags: [] },
+  async handler(resources): Promise<string> {
+    const { core, plugins, request } = resources;
+    const { apiKeyId, apiKey } = getAuthenticationAPIKey(request);
+
+    const coreStart = await core.start();
+    const savedObjectsClient =
+      coreStart.savedObjects.createInternalRepository();
+
+    const esHost = getESHosts({
+      cloudSetup: plugins.cloud.setup,
+      esClient: coreStart.elasticsearch.client.asInternalUser as Client,
+    });
+
+    const savedState = await findLatestObservabilityOnboardingState({
+      savedObjectsClient,
+    });
+
+    const yaml = generateYml({
+      datasetName: savedState?.state.datasetName,
+      customConfigurations: savedState?.state.customConfigurations,
+      logFilePaths: savedState?.state.logFilePaths,
+      namespace: savedState?.state.namespace,
+      apiKey: `${apiKeyId}:${apiKey}`,
+      esHost,
+      logfileId: `custom-logs-${Date.now()}`,
+    });
+
+    return yaml;
+  },
+});
+
+export const elasticAgentRouteRepository = {
+  ...generateConfig,
+};

x-pack/plugins/observability_onboarding/server/routes/index.ts

@@ -9,10 +9,14 @@ import type {
   ServerRouteRepository,
 } from '@kbn/server-route-repository';
 import { statusRouteRepository } from './status/route';
+import { customLogsRouteRepository } from './custom_logs/route';
+import { elasticAgentRouteRepository } from './elastic_agent/route';
 
 function getTypedObservabilityOnboardingServerRouteRepository() {
   const repository = {
     ...statusRouteRepository,
+    ...customLogsRouteRepository,
+    ...elasticAgentRouteRepository,
   };
 
   return repository;

x-pack/plugins/observability_onboarding/server/routes/register_routes.ts

@@ -66,6 +66,13 @@ export function registerRoutes({
             logger,
             params: decodedParams,
             plugins,
+            core: {
+              setup: core,
+              start: async () => {
+                const [coreStart] = await core.getStartServices();
+                return coreStart;
+              },
+            },
           })) as any;
 
           if (data === undefined) {

x-pack/plugins/observability_onboarding/server/routes/types.ts

@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { KibanaRequest, Logger } from '@kbn/core/server';
+import { CoreSetup, CoreStart, KibanaRequest, Logger } from '@kbn/core/server';
 import { ObservabilityOnboardingServerRouteRepository } from '.';
 import {
   ObservabilityOnboardingPluginSetupDependencies,
@@ -26,6 +26,10 @@ export interface ObservabilityOnboardingRouteHandlerResources {
       >;
     };
   };
+  core: {
+    setup: CoreSetup;
+    start: () => Promise<CoreStart>;
+  };
 }
 
 export interface ObservabilityOnboardingRouteCreateOptions {

x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SavedObjectsType } from '@kbn/core/server';
+
+export const OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE =
+  'observability-onboarding-state';
+
+export interface ObservabilityOnboardingState {
+  state: {
+    datasetName: string;
+    customConfigurations: string;
+    logFilePaths: string[];
+    namespace: string;
+  };
+  progress: Record<string, string>;
+}
+
+export interface SavedObservabilityOnboardingState
+  extends ObservabilityOnboardingState {
+  id: string;
+  updatedAt: number;
+}
+
+export const observabilityOnboardingState: SavedObjectsType = {
+  name: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+  hidden: false,
+  namespaceType: 'multiple',
+  mappings: {
+    properties: {
+      state: { type: 'object', dynamic: false },
+      progress: { type: 'object', dynamic: false },
+    },
+  },
+};

x-pack/plugins/observability_onboarding/server/types.ts

@@ -5,21 +5,27 @@
  * 2.0.
  */
 
+import { CloudSetup, CloudStart } from '@kbn/cloud-plugin/server';
 import { CustomRequestHandlerContext } from '@kbn/core/server';
 import {
   PluginSetup as DataPluginSetup,
   PluginStart as DataPluginStart,
 } from '@kbn/data-plugin/server';
 import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
 
 export interface ObservabilityOnboardingPluginSetupDependencies {
   data: DataPluginSetup;
   observability: ObservabilityPluginSetup;
+  cloud: CloudSetup;
+  usageCollection: UsageCollectionSetup;
 }
 
 export interface ObservabilityOnboardingPluginStartDependencies {
   data: DataPluginStart;
   observability: undefined;
+  cloud: CloudStart;
+  usageCollection: undefined;
 }
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface

x-pack/plugins/observability_onboarding/tsconfig.json

@@ -23,7 +23,12 @@
     "@kbn/config-schema",
     "@kbn/shared-ux-router",
     "@kbn/i18n-react",
+    "@kbn/cloud-plugin",
+    "@kbn/fleet-plugin",
+    "@kbn/usage-collection-plugin",
     "@kbn/observability-shared-plugin",
+    "@kbn/core-http-server",
+    "@kbn/security-plugin",
   ],
   "exclude": [
     "target/**/*",