[Onboarding] Save installed integrations to the onboarding SO (#184717)

Closes https://github.com/elastic/kibana/issues/183363

## Summary

Extends the POST/GET /progress endpoints with an `install-integrations`
step and it's payload that contains all the installed integrations.

## How to test

1. Run Kibana and ES locally
2. Generate Install API Key in Kibana
3. Go to System logs quick onboarding flow and copy generated ingest API
key and onboarding ID
4. Open Network tab in the dev console to track `/progress` endpoint
calls
5. Run the auto-detect CLI and go through the flow
```bash
sudo ./x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh --install-key="{{INSTALL_API_KEY}}" --ingest-key="{{INGEST_API_KEY}}" --kibana-url="http://localhost:5601/ftw" --id="{{ONBOARDING_ID}}" --ea-version="8.13.4"
```
6. Check response from the `/progress` endpoint In the dev console. Make
sure there is an `install-integrations` step and it has all of the
selected integrations.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -129,7 +129,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "ml-module": "94f210e60504fe0cc8241d867af97f8130250359",
         "ml-trained-model": "482195cefd6b04920e539d34d7356d22cb68e4f3",
         "monitoring-telemetry": "5d91bf75787d9d4dd2fae954d0b3f76d33d2e559",
-        "observability-onboarding-state": "b16064c516aac64ae699c737d7d10b6e199bfded",
+        "observability-onboarding-state": "34eef666124a9e4ad1c607d0097cc25128764681",
         "osquery-manager-usage-metric": "983bcbc3b7dda0aad29b20907db233abba709bcc",
         "osquery-pack": "702e86b1a936153b39f65b0781bdc136e186e123",
         "osquery-pack-asset": "cd140bc2e4b092e93692b587bf6e38051ef94c75",

x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/get_has_logs.ts

@@ -9,11 +9,11 @@ import { ElasticsearchClient } from '@kbn/core/server';
 import { termQuery } from '@kbn/observability-plugin/server';
 import { AGENT_ID } from '../../../common/es_fields';
 import {
-  ElasticAgentStepPayload,
   LogFilesState,
   ObservabilityOnboardingType,
   SystemLogsState,
 } from '../../saved_objects/observability_onboarding_status';
+import { ElasticAgentStepPayload } from '../types';
 
 export async function getHasLogs({
   type,

x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts

@@ -15,14 +15,12 @@ import {
 import type { TemplateAgentPolicyInput } from '@kbn/fleet-plugin/common';
 import { dump } from 'js-yaml';
 import { getObservabilityOnboardingFlow, saveObservabilityOnboardingFlow } from '../../lib/state';
-import {
-  ElasticAgentStepPayload,
-  ObservabilityOnboardingFlow,
-} from '../../saved_objects/observability_onboarding_status';
+import { ObservabilityOnboardingFlow } from '../../saved_objects/observability_onboarding_status';
 import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
 import { getHasLogs } from './get_has_logs';
 
 import { getFallbackESUrl } from '../../lib/get_fallback_urls';
+import { ElasticAgentStepPayload, Integration, StepProgressPayloadRT } from '../types';
 
 const updateOnboardingFlowRoute = createObservabilityOnboardingServerRoute({
   endpoint: 'PUT /internal/observability_onboarding/flow/{onboardingId}',
@@ -72,7 +70,9 @@ const stepProgressUpdateRoute = createObservabilityOnboardingServerRoute({
         status: t.string,
       }),
       t.partial({ message: t.string }),
-      t.partial({ payload: t.record(t.string, t.unknown) }),
+      t.partial({
+        payload: StepProgressPayloadRT,
+      }),
     ]),
   }),
   async handler(resources) {
@@ -112,7 +112,7 @@ const stepProgressUpdateRoute = createObservabilityOnboardingServerRoute({
           [name]: {
             status,
             message,
-            payload: payload as unknown as ElasticAgentStepPayload,
+            payload,
           },
         },
       },
@@ -162,7 +162,7 @@ const getProgressRoute = createObservabilityOnboardingServerRoute({
           type,
           state: savedObservabilityOnboardingState.state,
           esClient,
-          payload: progress['ea-status']?.payload,
+          payload: progress['ea-status']?.payload as ElasticAgentStepPayload,
         });
         if (hasLogs) {
           progress['logs-ingest'] = { status: 'complete' };
@@ -239,14 +239,6 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
       });
     }
 
-    await saveObservabilityOnboardingFlow({
-      savedObjectsClient,
-      savedObjectId: params.path.onboardingId,
-      observabilityOnboardingState: {
-        ...savedObservabilityOnboardingState,
-      } as ObservabilityOnboardingFlow,
-    });
-
     let agentPolicyInputs: TemplateAgentPolicyInput[] = [];
     try {
       agentPolicyInputs = await ensureInstalledIntegrations(integrationsToInstall, packageClient);
@@ -261,6 +253,21 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
       throw error;
     }
 
+    await saveObservabilityOnboardingFlow({
+      savedObjectsClient,
+      savedObjectId: params.path.onboardingId,
+      observabilityOnboardingState: {
+        ...savedObservabilityOnboardingState,
+        progress: {
+          ...savedObservabilityOnboardingState.progress,
+          'install-integrations': {
+            status: 'complete',
+            payload: integrationsToInstall,
+          },
+        },
+      } as ObservabilityOnboardingFlow,
+    });
+
     const elasticsearchUrl = plugins.cloud?.setup?.elasticsearchUrl
       ? [plugins.cloud?.setup?.elasticsearchUrl]
       : await getFallbackESUrl(services.esLegacyConfigService);
@@ -277,17 +284,6 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
   },
 });
 
-type Integration =
-  | {
-      pkgName: string;
-      installSource: 'registry';
-    }
-  | {
-      pkgName: string;
-      installSource: 'custom';
-      logFilePaths: string[];
-    };
-
 async function ensureInstalledIntegrations(
   integrationsToInstall: Integration[],
   packageClient: PackageClient

x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts

@@ -11,6 +11,7 @@ import {
   KibanaResponseFactory,
   Logger,
 } from '@kbn/core/server';
+import * as t from 'io-ts';
 import { ObservabilityOnboardingServerRouteRepository } from '.';
 import { ObservabilityOnboardingConfig } from '..';
 import { EsLegacyConfigService } from '../services/es_legacy_config_service';
@@ -50,3 +51,34 @@ export interface ObservabilityOnboardingRouteCreateOptions {
     xsrfRequired?: boolean;
   };
 }
+
+export const IntegrationRT = t.union([
+  t.type({
+    pkgName: t.string,
+    installSource: t.literal('registry'),
+  }),
+  t.type({
+    pkgName: t.string,
+    installSource: t.literal('custom'),
+    logFilePaths: t.array(t.string),
+  }),
+]);
+
+export type Integration = t.TypeOf<typeof IntegrationRT>;
+
+export const ElasticAgentStepPayloadRT = t.type({
+  agentId: t.string,
+});
+
+export type ElasticAgentStepPayload = t.TypeOf<typeof ElasticAgentStepPayloadRT>;
+
+export const InstallIntegrationsStepPayloadRT = t.array(IntegrationRT);
+
+export type InstallIntegrationsStepPayload = t.TypeOf<typeof InstallIntegrationsStepPayloadRT>;
+
+export const StepProgressPayloadRT = t.union([
+  ElasticAgentStepPayloadRT,
+  InstallIntegrationsStepPayloadRT,
+]);
+
+export type StepProgressPayload = t.TypeOf<typeof StepProgressPayloadRT>;

x-pack/plugins/observability_solution/observability_onboarding/server/saved_objects/observability_onboarding_status.ts

@@ -7,8 +7,10 @@
 
 import { SavedObjectsType } from '@kbn/core/server';
 import { schema } from '@kbn/config-schema';
+import { StepProgressPayload } from '../routes/types';
 
 export const OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE = 'observability-onboarding-state';
+
 export interface LogFilesState {
   datasetName: string;
   serviceName?: string;
@@ -21,10 +23,6 @@ export interface SystemLogsState {
   namespace: string;
 }
 
-export interface ElasticAgentStepPayload {
-  agentId: string;
-}
-
 export type ObservabilityOnboardingType = 'logFiles' | 'systemLogs';
 
 type ObservabilityOnboardingFlowState = LogFilesState | SystemLogsState | undefined;
@@ -37,7 +35,7 @@ export interface ObservabilityOnboardingFlow {
     {
       status: string;
       message?: string;
-      payload?: ElasticAgentStepPayload;
+      payload?: StepProgressPayload;
     }
   >;
 }
@@ -63,6 +61,14 @@ const ElasticAgentStepPayloadSchema = schema.object({
   agentId: schema.string(),
 });
 
+export const InstallIntegrationsStepPayloadSchema = schema.arrayOf(
+  schema.object({
+    pkgName: schema.string(),
+    installSource: schema.string(),
+    logFilePaths: schema.maybe(schema.arrayOf(schema.string())),
+  })
+);
+
 export const observabilityOnboardingFlow: SavedObjectsType = {
   name: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
   hidden: false,
@@ -92,5 +98,26 @@ export const observabilityOnboardingFlow: SavedObjectsType = {
         }),
       },
     },
+    '2': {
+      changes: [],
+      schemas: {
+        create: schema.object({
+          type: schema.string(),
+          state: schema.maybe(
+            schema.oneOf([LogFilesStateSchema, SystemLogsStateSchema, schema.never()])
+          ),
+          progress: schema.mapOf(
+            schema.string(),
+            schema.object({
+              status: schema.string(),
+              message: schema.maybe(schema.string()),
+              payload: schema.maybe(
+                schema.oneOf([ElasticAgentStepPayloadSchema, InstallIntegrationsStepPayloadSchema])
+              ),
+            })
+          ),
+        }),
+      },
+    },
   },
 };