[Logs onboarding] Generate elastic-agent.yml file for system logs (#162972)

Closes https://github.com/elastic/kibana/issues/154929.

This PR along with https://github.com/elastic/kibana/pull/162654,
https://github.com/elastic/kibana/pull/162706 and
https://github.com/elastic/kibana/pull/162600 completes the work
required for collect system logs.

### Changes
- `ObservabilityOnboardingType` now could be `logFiles | systemLogs`.
This help us to identify (without changing the script) whether we need
to retrieve the yaml configuration for customLogs or for systemLogs.
- Added `generateSystemLogsYml` which generates a specific configuration
for system logs.
- `get_has_logs.ts` was modified so we are querying the proper index
depending on the type of logs.

#### Demo


47eca890-37b2-401e-9e41-67c978ab50ad

x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx

@@ -91,6 +91,7 @@ export function InstallElasticAgent() {
           params: {
             body: {
               name: datasetName,
+              type: 'logFiles',
               state: {
                 datasetName,
                 serviceName,

x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx

@@ -40,8 +40,7 @@ export function InstallElasticAgent() {
   const [elasticAgentPlatform, setElasticAgentPlatform] =
     useState<ElasticAgentPlatform>('linux-tar');
 
-  const datasetName = 'elastic-agent';
+  const datasetName = 'system-logs';
-  const namespace = 'default';
 
   function onBack() {
     navigateToKibanaUrl('/app/observabilityOnboarding');
@@ -83,10 +82,7 @@ export function InstallElasticAgent() {
           params: {
             body: {
               name: datasetName,
-              state: {
+              type: 'systemLogs',
-                datasetName,
-                namespace,
-              },
             },
           },
         });
@@ -95,26 +91,6 @@ export function InstallElasticAgent() {
     [monitoringRole?.hasPrivileges]
   );
 
-  const { status: saveOnboardingStateDataStatus } = useFetcher((callApi) => {
-    const { onboardingId } = getState();
-    if (onboardingId) {
-      return callApi(
-        'PUT /internal/observability_onboarding/flow/{onboardingId}',
-        {
-          params: {
-            path: { onboardingId },
-            body: {
-              state: {
-                datasetName,
-                namespace,
-              },
-            },
-          },
-        }
-      );
-    }
-  }, []);
-
   const { apiKeyEncoded, onboardingId } = installShipperSetup ?? getState();
 
   const { data: yamlConfig = '', status: yamlConfigStatus } = useFetcher(
@@ -132,7 +108,7 @@ export function InstallElasticAgent() {
     [
       apiKeyEncoded,
       onboardingId,
-      saveOnboardingStateDataStatus === FETCH_STATUS.SUCCESS,
+      installShipperSetupStatus === FETCH_STATUS.SUCCESS,
     ]
   );
 

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/__snapshots__/generate_custom_logs_yml.test.ts.snap

@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`generateYml should return a basic yml configuration 1`] = `
+exports[`generateCustomLogsYml should return a basic yml configuration 1`] = `
 "outputs:
   default:
     type: elasticsearch
@@ -21,7 +21,7 @@ inputs:
 "
 `;
 
-exports[`generateYml should return a yml configuration with customConfigurations 1`] = `
+exports[`generateCustomLogsYml should return a yml configuration with customConfigurations 1`] = `
 "outputs:
   default:
     type: elasticsearch
@@ -47,7 +47,7 @@ agent.monitoring:
 "
 `;
 
-exports[`generateYml should return a yml configuration with multiple logFilePaths 1`] = `
+exports[`generateCustomLogsYml should return a yml configuration with multiple logFilePaths 1`] = `
 "outputs:
   default:
     type: elasticsearch
@@ -69,7 +69,7 @@ inputs:
 "
 `;
 
-exports[`generateYml should return a yml configuration with service name 1`] = `
+exports[`generateCustomLogsYml should return a yml configuration with service name 1`] = `
 "outputs:
   default:
     type: elasticsearch

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { dump } from 'js-yaml';
-import { generateYml } from './generate_yml';
+import { generateCustomLogsYml } from './generate_custom_logs_yml';
 
 const baseMockConfig = {
   datasetName: 'my-dataset',
@@ -17,9 +17,9 @@ const baseMockConfig = {
   logfileId: 'my-logs-id',
 };
 
-describe('generateYml', () => {
+describe('generateCustomLogsYml', () => {
   it('should return a basic yml configuration', () => {
-    const result = generateYml(baseMockConfig);
+    const result = generateCustomLogsYml(baseMockConfig);
     expect(result).toMatchSnapshot();
   });
 
@@ -29,7 +29,7 @@ describe('generateYml', () => {
       logFilePaths: ['/my-service-1.logs', '/my-service-2.logs'],
     };
 
-    const result = generateYml(mockConfig);
+    const result = generateCustomLogsYml(mockConfig);
     expect(result).toMatchSnapshot();
   });
 
@@ -39,7 +39,7 @@ describe('generateYml', () => {
       serviceName: 'my-service',
     };
 
-    const result = generateYml(mockConfig);
+    const result = generateCustomLogsYml(mockConfig);
     expect(result).toMatchSnapshot();
   });
 
@@ -57,7 +57,7 @@ describe('generateYml', () => {
       }),
     };
 
-    const result = generateYml(mockConfig);
+    const result = generateCustomLogsYml(mockConfig);
     expect(result).toMatchSnapshot();
   });
 });

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.ts

@@ -7,7 +7,7 @@
 
 import { dump, load } from 'js-yaml';
 
-export const generateYml = ({
+export const generateCustomLogsYml = ({
   datasetName = '',
   serviceName,
   namespace = '',

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts

@@ -6,11 +6,13 @@
  */
 
 import * as t from 'io-ts';
+import { v4 as uuidv4 } from 'uuid';
 import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key';
-import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
-import { generateYml } from './generate_yml';
 import { getFallbackESUrl } from '../../lib/get_fallback_urls';
 import { getObservabilityOnboardingFlow } from '../../lib/state';
+import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
+import { generateCustomLogsYml } from './custom_logs/generate_custom_logs_yml';
+import { generateSystemLogsYml } from './system_logs/generate_system_logs_yml';
 
 const generateConfig = createObservabilityOnboardingServerRoute({
   endpoint: 'GET /internal/observability_onboarding/elastic_agent/config',
@@ -43,18 +45,24 @@ const generateConfig = createObservabilityOnboardingServerRoute({
       savedObjectId: onboardingId,
     });
 
-    const yaml = generateYml({
+    const yaml =
-      datasetName: savedState?.state?.datasetName,
+      savedState?.type === 'systemLogs'
-      customConfigurations: savedState?.state?.customConfigurations,
+        ? generateSystemLogsYml({
-      logFilePaths: savedState?.state?.logFilePaths,
+            ...savedState?.state,
-      namespace: savedState?.state?.namespace,
+            apiKey: authApiKey
-      apiKey: authApiKey
+              ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
-        ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
+              : '$API_KEY',
-        : '$API_KEY',
+            esHost: elasticsearchUrl,
-      esHost: elasticsearchUrl,
+            uuid: uuidv4(),
-      logfileId: `custom-logs-${Date.now()}`,
+          })
-      serviceName: savedState?.state?.serviceName,
+        : generateCustomLogsYml({
-    });
+            ...savedState?.state,
+            apiKey: authApiKey
+              ? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
+              : '$API_KEY',
+            esHost: elasticsearchUrl,
+            logfileId: `custom-logs-${uuidv4()}`,
+          });
 
     return yaml;
   },

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/__snapshots__/generate_system_logs_yml.test.ts.snap

@@ -0,0 +1,48 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`generateSystemLogsYml should return system logs oriented yml configuration 1`] = `
+"outputs:
+  default:
+    type: elasticsearch
+    hosts:
+      - 'http://localhost:9200'
+    api_key: 'elastic:changeme'
+inputs:
+  - id: system-logs-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1
+    type: logfile
+    data_stream:
+      namespace: default
+    streams:
+      - id: logfile-system.auth-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1
+        data_stream:
+          dataset: system.auth
+          type: logs
+        paths:
+          - /var/log/auth.log*
+          - /var/log/secure*
+        exclude_files:
+          - .gz$
+        multiline:
+          pattern: ^s
+          match: after
+        tags:
+          - system-auth
+        processors:
+          - add_locale: null
+      - id: logfile-system.syslog-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1
+        data_stream:
+          dataset: system.syslog
+          type: logs
+        paths:
+          - /var/log/messages*
+          - /var/log/syslog*
+          - /var/log/system*
+        exclude_files:
+          - .gz$
+        multiline:
+          pattern: ^s
+          match: after
+        processors:
+          - add_locale: null
+"
+`;

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.test.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { generateSystemLogsYml } from './generate_system_logs_yml';
+
+const baseMockConfig = {
+  namespace: 'default',
+  apiKey: 'elastic:changeme',
+  esHost: ['http://localhost:9200'],
+  uuid: '8df0ff52-6f3b-4b5a-a2da-f06c55d111d1',
+};
+
+describe('generateSystemLogsYml', () => {
+  it('should return system logs oriented yml configuration', () => {
+    const result = generateSystemLogsYml(baseMockConfig);
+    expect(result).toMatchSnapshot();
+  });
+});

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { dump } from 'js-yaml';
+
+export const generateSystemLogsYml = ({
+  namespace = 'default',
+  apiKey,
+  esHost,
+  uuid,
+}: {
+  namespace?: string;
+  apiKey: string;
+  esHost: string[];
+  uuid: string;
+}) => {
+  return dump({
+    outputs: {
+      default: {
+        type: 'elasticsearch',
+        hosts: esHost,
+        api_key: apiKey,
+      },
+    },
+    inputs: [
+      {
+        id: `system-logs-${uuid}`,
+        type: 'logfile',
+        data_stream: {
+          namespace,
+        },
+        streams: [
+          {
+            id: `logfile-system.auth-${uuid}`,
+            data_stream: {
+              dataset: 'system.auth',
+              type: 'logs',
+            },
+            paths: ['/var/log/auth.log*', '/var/log/secure*'],
+            exclude_files: ['.gz$'],
+            multiline: {
+              pattern: '^s',
+              match: 'after',
+            },
+            tags: ['system-auth'],
+            processors: [
+              {
+                add_locale: null,
+              },
+            ],
+          },
+          {
+            id: `logfile-system.syslog-${uuid}`,
+            data_stream: {
+              dataset: 'system.syslog',
+              type: 'logs',
+            },
+            paths: [
+              '/var/log/messages*',
+              '/var/log/syslog*',
+              '/var/log/system*',
+            ],
+            exclude_files: ['.gz$'],
+            multiline: {
+              pattern: '^s',
+              match: 'after',
+            },
+            processors: [
+              {
+                add_locale: null,
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+};

x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts

@@ -6,19 +6,34 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core/server';
+import {
+  LogFilesState,
+  ObservabilityOnboardingType,
+  SystemLogsState,
+} from '../../saved_objects/observability_onboarding_status';
 
 export async function getHasLogs({
-  dataset,
+  type,
-  namespace,
+  state,
   esClient,
 }: {
-  dataset: string;
+  type: ObservabilityOnboardingType;
-  namespace: string;
+  state?: LogFilesState | SystemLogsState;
   esClient: ElasticsearchClient;
 }) {
+  if (!state) {
+    return false;
+  }
+
   try {
+    const { namespace } = state;
+    const index =
+      type === 'logFiles'
+        ? `logs-${(state as LogFilesState).datasetName}-${namespace}`
+        : `logs-system.syslog-${namespace}`;
+
     const { hits } = await esClient.search({
-      index: `logs-${dataset}-${namespace}`,
+      index,
       terminate_after: 1,
     });
     const total = hits.total as { value: number };

x-pack/plugins/observability_onboarding/server/routes/flow/route.ts

@@ -148,16 +148,13 @@ const getProgressRoute = createObservabilityOnboardingServerRoute({
     const esClient =
       coreStart.elasticsearch.client.asScoped(request).asCurrentUser;
 
-    const dataset = savedObservabilityOnboardingState.state
+    const type = savedObservabilityOnboardingState.type;
-      ?.datasetName as string;
-    const namespace = savedObservabilityOnboardingState.state
-      ?.namespace as string;
 
     if (progress['ea-status']?.status === 'complete') {
       try {
         const hasLogs = await getHasLogs({
-          dataset,
+          type,
-          namespace,
+          state: savedObservabilityOnboardingState.state,
           esClient,
         });
         if (hasLogs) {

x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts

@@ -15,7 +15,7 @@ export function createShipperApiKey(
   // Based on https://www.elastic.co/guide/en/fleet/master/grant-access-to-elasticsearch.html#create-api-key-standalone-agent
   return esClient.security.createApiKey({
     body: {
-      name: `standalone_agent_custom_logs_${name}`,
+      name: `standalone_agent_logs_onboarding_${name}`,
       metadata: { application: 'logs' },
       role_descriptors: {
         standalone_agent: {

x-pack/plugins/observability_onboarding/server/routes/logs/route.ts

@@ -66,6 +66,9 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
       t.type({
         name: t.string,
       }),
+      t.type({
+        type: t.union([t.literal('logFiles'), t.literal('systemLogs')]),
+      }),
       t.partial({
         state: t.record(t.string, t.unknown),
       }),
@@ -77,7 +80,7 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
     const {
       context,
       params: {
-        body: { name, state },
+        body: { name, type, state },
       },
       core,
       request,
@@ -91,13 +94,15 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
       name
     );
 
+    const generatedState =
+      type === 'systemLogs' ? { namespace: 'default' } : state;
     const savedObjectsClient = coreStart.savedObjects.getScopedClient(request);
 
     const { id } = await saveObservabilityOnboardingFlow({
       savedObjectsClient,
       observabilityOnboardingState: {
-        type: 'logFiles',
+        type,
-        state: state as ObservabilityOnboardingFlow['state'],
+        state: generatedState as ObservabilityOnboardingFlow['state'],
         progress: {},
       },
     });

x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts

@@ -17,10 +17,19 @@ export interface LogFilesState {
   namespace: string;
 }
 
-type ObservabilityOnboardingFlowState = LogFilesState | undefined;
+export interface SystemLogsState {
+  namespace: string;
+}
+
+export type ObservabilityOnboardingType = 'logFiles' | 'systemLogs';
+
+type ObservabilityOnboardingFlowState =
+  | LogFilesState
+  | SystemLogsState
+  | undefined;
 
 export interface ObservabilityOnboardingFlow {
-  type: 'logFiles';
+  type: ObservabilityOnboardingType;
   state: ObservabilityOnboardingFlowState;
   progress: Record<
     string,

x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts

@@ -31,25 +31,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     const logFilepath = '/my-logs.log';
     const serviceName = 'my-service';
 
-    before(async () => {
-      const req = await observabilityOnboardingApiClient.logMonitoringUser({
-        endpoint: 'POST /internal/observability_onboarding/logs/flow',
-        params: {
-          body: {
-            name: 'name',
-            state: {
-              datasetName,
-              namespace,
-              logFilePaths: [logFilepath],
-              serviceName,
-            },
-          },
-        },
-      });
-
-      onboardingId = req.body.onboardingId;
-    });
-
     describe("when onboardingId doesn't exists", () => {
       it('should return input properties empty', async () => {
         const req = await callApi({
@@ -66,17 +47,72 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     });
 
     describe('when onboardingId exists', () => {
-      it('should return input properties configured', async () => {
+      describe('and onboarding type is logFiles', () => {
-        const req = await callApi({
+        before(async () => {
-          onboardingId,
+          const req = await observabilityOnboardingApiClient.logMonitoringUser({
+            endpoint: 'POST /internal/observability_onboarding/logs/flow',
+            params: {
+              body: {
+                type: 'logFiles',
+                name: 'name',
+                state: {
+                  datasetName,
+                  namespace,
+                  logFilePaths: [logFilepath],
+                  serviceName,
+                },
+              },
+            },
+          });
+
+          onboardingId = req.body.onboardingId;
         });
 
-        expect(req.status).to.be(200);
+        it('should return input properties configured', async () => {
+          const req = await callApi({
+            onboardingId,
+          });
 
-        const ymlConfig = load(req.text);
+          expect(req.status).to.be(200);
-        expect(ymlConfig.inputs[0].data_stream.namespace).to.be(namespace);
+
-        expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be(datasetName);
+          const ymlConfig = load(req.text);
-        expect(ymlConfig.inputs[0].streams[0].paths).to.be.eql([logFilepath]);
+          expect(ymlConfig.inputs[0].data_stream.namespace).to.be(namespace);
+          expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be(datasetName);
+          expect(ymlConfig.inputs[0].streams[0].paths).to.be.eql([logFilepath]);
+          expect(ymlConfig.inputs[0].streams[0].processors[0].add_fields.fields.name).to.be.eql(
+            serviceName
+          );
+        });
+      });
+
+      describe('and onboarding type is systemLogs', () => {
+        before(async () => {
+          const req = await observabilityOnboardingApiClient.logMonitoringUser({
+            endpoint: 'POST /internal/observability_onboarding/logs/flow',
+            params: {
+              body: {
+                type: 'systemLogs',
+                name: 'name',
+              },
+            },
+          });
+
+          onboardingId = req.body.onboardingId;
+        });
+
+        it('should return input properties configured', async () => {
+          const req = await callApi({
+            onboardingId,
+          });
+
+          expect(req.status).to.be(200);
+
+          const ymlConfig = load(req.text);
+          expect(ymlConfig.inputs[0].data_stream.namespace).to.be('default');
+          expect(ymlConfig.inputs[0].streams.length).to.be(2);
+          expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be('system.auth');
+          expect(ymlConfig.inputs[0].streams[1].data_stream.dataset).to.be('system.syslog');
+        });
       });
     });
   });

x-pack/test/observability_onboarding_api_integration/tests/flow/progress/es_utils.ts

@@ -15,7 +15,7 @@ export function createLogDoc({
 }: {
   time: number;
   logFilepath: string;
-  serviceName: string;
+  serviceName?: string;
   namespace: string;
   datasetName: string;
   message: string;
@@ -30,9 +30,13 @@ export function createLogDoc({
         path: logFilepath,
       },
     },
-    service: {
+    ...(serviceName
-      name: serviceName,
+      ? {
-    },
+          service: {
+            name: serviceName,
+          },
+        }
+      : {}),
     data_stream: {
       namespace,
       type: 'logs',

x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts

@@ -44,6 +44,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         endpoint: 'POST /internal/observability_onboarding/logs/flow',
         params: {
           body: {
+            type: 'logFiles',
             name: 'name',
             state: {
               datasetName,
@@ -131,40 +132,107 @@ export default function ApiTest({ getService }: FtrProviderContext) {
             });
 
             describe('when logs have been ingested', () => {
-              before(async () => {
+              describe('and onboarding type is logFiles', () => {
-                await es.indices.createDataStream({
+                before(async () => {
-                  name: `logs-${datasetName}-${namespace}`,
+                  await es.indices.createDataStream({
+                    name: `logs-${datasetName}-${namespace}`,
+                  });
+
+                  const doc = createLogDoc({
+                    time: new Date('06/28/2023').getTime(),
+                    logFilepath: '/my-service.log',
+                    serviceName: 'my-service',
+                    namespace,
+                    datasetName,
+                    message: 'This is a log message',
+                  });
+
+                  await es.bulk({
+                    body: [{ create: { _index: `logs-${datasetName}-${namespace}` } }, doc],
+                    refresh: 'wait_for',
+                  });
                 });
 
-                const doc = createLogDoc({
+                it('should return log-ingest as complete', async () => {
-                  time: new Date('06/28/2023').getTime(),
+                  const request = await callApi({
-                  logFilepath: '/my-service.log',
+                    onboardingId,
-                  serviceName: 'my-service',
+                  });
-                  namespace,
+
-                  datasetName,
+                  expect(request.status).to.be(200);
-                  message: 'This is a log message',
+
+                  const logsIngestProgress = request.body.progress['logs-ingest'];
+                  expect(logsIngestProgress).to.have.property('status', 'complete');
                 });
 
-                await es.bulk({
+                after(async () => {
-                  body: [{ create: { _index: `logs-${datasetName}-${namespace}` } }, doc],
+                  await es.indices.deleteDataStream({
-                  refresh: 'wait_for',
+                    name: `logs-${datasetName}-${namespace}`,
+                  });
                 });
               });
 
-              it('should return log-ingest as complete', async () => {
+              describe('and onboarding type is systemLogs', () => {
-                const request = await callApi({
+                let systemLogsOnboardingId: string;
-                  onboardingId,
+
+                before(async () => {
+                  const req = await observabilityOnboardingApiClient.logMonitoringUser({
+                    endpoint: 'POST /internal/observability_onboarding/logs/flow',
+                    params: {
+                      body: {
+                        type: 'systemLogs',
+                        name: 'name',
+                      },
+                    },
+                  });
+
+                  systemLogsOnboardingId = req.body.onboardingId;
+
+                  await observabilityOnboardingApiClient.logMonitoringUser({
+                    endpoint: 'POST /internal/observability_onboarding/flow/{id}/step/{name}',
+                    params: {
+                      path: {
+                        id: systemLogsOnboardingId,
+                        name: 'ea-status',
+                      },
+                      body: {
+                        status: 'complete',
+                      },
+                    },
+                  });
+
+                  await es.indices.createDataStream({
+                    name: `logs-system.syslog-${namespace}`,
+                  });
+
+                  const doc = createLogDoc({
+                    time: new Date('06/28/2023').getTime(),
+                    logFilepath: '/var/log/system.log',
+                    namespace,
+                    datasetName: 'system.syslog',
+                    message: 'This is a system log message',
+                  });
+
+                  await es.bulk({
+                    body: [{ create: { _index: `logs-system.syslog-${namespace}` } }, doc],
+                    refresh: 'wait_for',
+                  });
                 });
 
-                expect(request.status).to.be(200);
+                it('should return log-ingest as complete', async () => {
+                  const request = await callApi({
+                    onboardingId: systemLogsOnboardingId,
+                  });
 
-                const logsIngestProgress = request.body.progress['logs-ingest'];
+                  expect(request.status).to.be(200);
-                expect(logsIngestProgress).to.have.property('status', 'complete');
-              });
 
-              after(async () => {
+                  const logsIngestProgress = request.body.progress['logs-ingest'];
-                await es.indices.deleteDataStream({
+                  expect(logsIngestProgress).to.have.property('status', 'complete');
-                  name: `logs-${datasetName}-${namespace}`,
+                });
+
+                after(async () => {
+                  await es.indices.deleteDataStream({
+                    name: `logs-system.syslog-${namespace}`,
+                  });
                 });
               });
             });

x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts

@@ -21,6 +21,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       endpoint: 'POST /internal/observability_onboarding/logs/flow',
       params: {
         body: {
+          type: 'logFiles',
           name: 'name',
           state,
         },
@@ -28,11 +29,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     });
   }
 
-  async function callApiWithPrivileges(state = {}) {
+  async function callApiWithPrivileges(type: 'logFiles' | 'systemLogs', state = {}) {
     return await observabilityOnboardingApiClient.logMonitoringUser({
       endpoint: 'POST /internal/observability_onboarding/logs/flow',
       params: {
         body: {
+          type,
           name: 'name',
           state,
         },
@@ -54,14 +56,14 @@ export default function ApiTest({ getService }: FtrProviderContext) {
 
     describe('when required privileges are set', () => {
       it('returns a flow id and apiKey encoded', async () => {
-        const request = await callApiWithPrivileges();
+        const request = await callApiWithPrivileges('logFiles');
 
         expect(request.status).to.be(200);
         expect(request.body.apiKeyEncoded).to.not.empty();
         expect(request.body.onboardingId).to.not.empty();
       });
 
-      it('saves the expected state', async () => {
+      it('saves the expected state for logFiles', async () => {
         const state = {
           datasetName: 'my-dataset',
           serviceName: 'my-service',
@@ -69,7 +71,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
           logFilePaths: 'my-service-logs.log',
         };
 
-        const request = await callApiWithPrivileges(state);
+        const request = await callApiWithPrivileges('logFiles', state);
 
         const savedState = await kibanaServer.savedObjects.get({
           type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
@@ -78,6 +80,21 @@ export default function ApiTest({ getService }: FtrProviderContext) {
 
         expect(savedState.attributes).to.be.eql({ type: 'logFiles', state, progress: {} });
       });
+
+      it('saves the expected state for systemLogs', async () => {
+        const state = {
+          namespace: 'default',
+        };
+
+        const request = await callApiWithPrivileges('systemLogs');
+
+        const savedState = await kibanaServer.savedObjects.get({
+          type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+          id: request.body.onboardingId,
+        });
+
+        expect(savedState.attributes).to.be.eql({ type: 'systemLogs', state, progress: {} });
+      });
     });
   });
 }

x-pack/test/observability_onboarding_api_integration/tests/logs/update_step_progress.spec.ts

@@ -67,6 +67,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
           endpoint: 'POST /internal/observability_onboarding/logs/flow',
           params: {
             body: {
+              type: 'logFiles',
               name: 'name',
               state: {},
             },