[8.x] [Automatic Import] Refactor useEffect to reduce complexity (#194150) (#194194)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Automatic Import] Refactor useEffect to reduce complexity
(#194150)](https://github.com/elastic/kibana/pull/194150)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Bharat
Pasupula","email":"123897612+bhapas@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-09-26T16:50:28Z","message":"[Automatic
Import] Refactor useEffect to reduce complexity
(#194150)","sha":"bcdb0d8ede90a99317bddd99cc122dc98107d1a7","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor","Team:Security-Scalability","Feature:AutomaticImport"],"title":"[Automatic
Import] Refactor useEffect to reduce
complexity","number":194150,"url":"https://github.com/elastic/kibana/pull/194150","mergeCommit":{"message":"[Automatic
Import] Refactor useEffect to reduce complexity
(#194150)","sha":"bcdb0d8ede90a99317bddd99cc122dc98107d1a7"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/194150","number":194150,"mergeCommit":{"message":"[Automatic
Import] Refactor useEffect to reduce complexity
(#194150)","sha":"bcdb0d8ede90a99317bddd99cc122dc98107d1a7"}}]}]
BACKPORT-->

Co-authored-by: Bharat Pasupula <123897612+bhapas@users.noreply.github.com>

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/data_stream_step.tsx

@@ -20,7 +20,7 @@ import type { InputType } from '../../../../../../common';
 import { useActions, type State } from '../../state';
 import type { IntegrationSettings } from '../../types';
 import { StepContentWrapper } from '../step_content_wrapper';
-import type { OnComplete } from './generation_modal';
+import type { OnComplete } from './use_generation';
 import { GenerationModal } from './generation_modal';
 import { SampleLogsInput } from './sample_logs_input';
 import * as i18n from './translations';

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx

@@ -21,33 +21,13 @@ import {
   EuiText,
   useEuiTheme,
 } from '@elastic/eui';
-import { isEmpty } from 'lodash/fp';
-import React, { useCallback, useEffect, useMemo, useState } from 'react';
+import React, { useMemo } from 'react';
 import { css } from '@emotion/react';
-import { getLangSmithOptions } from '../../../../../common/lib/lang_smith';
-import type { ESProcessorItem } from '../../../../../../common';
-import {
-  type AnalyzeLogsRequestBody,
-  type CategorizationRequestBody,
-  type EcsMappingRequestBody,
-  type RelatedRequestBody,
-} from '../../../../../../common';
-import {
-  runCategorizationGraph,
-  runEcsGraph,
-  runRelatedGraph,
-  runAnalyzeLogsGraph,
-} from '../../../../../common/lib/api';
-import { useKibana } from '../../../../../common/hooks/use_kibana';
 import type { State } from '../../state';
 import * as i18n from './translations';
-import { useTelemetry } from '../../../telemetry';
-import type { ErrorCode } from '../../../../../../common/constants';
 
-export type OnComplete = (result: State['result']) => void;
-
-const ProgressOrder = ['ecs', 'categorization', 'related'];
-type ProgressItem = (typeof ProgressOrder)[number];
+import type { OnComplete, ProgressItem } from './use_generation';
+import { ProgressOrder, useGeneration } from './use_generation';
 
 const progressText: Record<ProgressItem, string> = {
   analyzeLogs: i18n.PROGRESS_ANALYZE_LOGS,
@@ -56,165 +36,6 @@ const progressText: Record<ProgressItem, string> = {
   related: i18n.PROGRESS_RELATED_GRAPH,
 };
 
-interface UseGenerationProps {
-  integrationSettings: State['integrationSettings'];
-  connector: State['connector'];
-  onComplete: OnComplete;
-}
-export const useGeneration = ({
-  integrationSettings,
-  connector,
-  onComplete,
-}: UseGenerationProps) => {
-  const { reportGenerationComplete } = useTelemetry();
-  const { http, notifications } = useKibana().services;
-  const [progress, setProgress] = useState<ProgressItem>();
-  const [error, setError] = useState<null | string>(null);
-  const [isRequesting, setIsRequesting] = useState<boolean>(true);
-
-  useEffect(() => {
-    if (
-      !isRequesting ||
-      http == null ||
-      connector == null ||
-      integrationSettings == null ||
-      notifications?.toasts == null
-    ) {
-      return;
-    }
-    const generationStartedAt = Date.now();
-    const abortController = new AbortController();
-    const deps = { http, abortSignal: abortController.signal };
-
-    (async () => {
-      try {
-        let additionalProcessors: ESProcessorItem[] | undefined;
-
-        // logSamples may be modified to JSON format if they are in different formats
-        // Keeping originalLogSamples for running pipeline and generating docs
-        const originalLogSamples = integrationSettings.logSamples;
-        let logSamples = integrationSettings.logSamples;
-        let samplesFormat = integrationSettings.samplesFormat;
-
-        if (integrationSettings.samplesFormat === undefined) {
-          const analyzeLogsRequest: AnalyzeLogsRequestBody = {
-            packageName: integrationSettings.name ?? '',
-            dataStreamName: integrationSettings.dataStreamName ?? '',
-            logSamples: integrationSettings.logSamples ?? [],
-            connectorId: connector.id,
-            langSmithOptions: getLangSmithOptions(),
-          };
-
-          setProgress('analyzeLogs');
-          const analyzeLogsResult = await runAnalyzeLogsGraph(analyzeLogsRequest, deps);
-          if (abortController.signal.aborted) return;
-          if (isEmpty(analyzeLogsResult?.results)) {
-            setError('No results from Analyze Logs Graph');
-            return;
-          }
-          logSamples = analyzeLogsResult.results.parsedSamples;
-          samplesFormat = analyzeLogsResult.results.samplesFormat;
-          additionalProcessors = analyzeLogsResult.additionalProcessors;
-        }
-
-        const ecsRequest: EcsMappingRequestBody = {
-          packageName: integrationSettings.name ?? '',
-          dataStreamName: integrationSettings.dataStreamName ?? '',
-          rawSamples: logSamples ?? [],
-          samplesFormat: samplesFormat ?? { name: 'json' },
-          additionalProcessors: additionalProcessors ?? [],
-          connectorId: connector.id,
-          langSmithOptions: getLangSmithOptions(),
-        };
-
-        setProgress('ecs');
-        const ecsGraphResult = await runEcsGraph(ecsRequest, deps);
-        if (abortController.signal.aborted) return;
-        if (isEmpty(ecsGraphResult?.results)) {
-          setError('No results from ECS graph');
-          return;
-        }
-        const categorizationRequest: CategorizationRequestBody = {
-          ...ecsRequest,
-          rawSamples: originalLogSamples ?? [],
-          samplesFormat: samplesFormat ?? { name: 'json' },
-          currentPipeline: ecsGraphResult.results.pipeline,
-        };
-
-        setProgress('categorization');
-        const categorizationResult = await runCategorizationGraph(categorizationRequest, deps);
-        if (abortController.signal.aborted) return;
-        const relatedRequest: RelatedRequestBody = {
-          ...categorizationRequest,
-          currentPipeline: categorizationResult.results.pipeline,
-        };
-
-        setProgress('related');
-        const relatedGraphResult = await runRelatedGraph(relatedRequest, deps);
-        if (abortController.signal.aborted) return;
-
-        if (isEmpty(relatedGraphResult?.results)) {
-          throw new Error('Results not found in response');
-        }
-
-        reportGenerationComplete({
-          connector,
-          integrationSettings,
-          durationMs: Date.now() - generationStartedAt,
-        });
-
-        const result = {
-          pipeline: relatedGraphResult.results.pipeline,
-          docs: relatedGraphResult.results.docs,
-          samplesFormat,
-        };
-
-        onComplete(result);
-      } catch (e) {
-        if (abortController.signal.aborted) return;
-        const originalErrorMessage = `${e.message}${
-          e.body ? ` (${e.body.statusCode}): ${e.body.message}` : ''
-        }`;
-
-        reportGenerationComplete({
-          connector,
-          integrationSettings,
-          durationMs: Date.now() - generationStartedAt,
-          error: originalErrorMessage,
-        });
-
-        let errorMessage = originalErrorMessage;
-        const errorCode = e.body?.attributes?.errorCode as ErrorCode | undefined;
-        if (errorCode != null) {
-          errorMessage = i18n.ERROR_TRANSLATION[errorCode];
-        }
-        setError(errorMessage);
-      } finally {
-        setIsRequesting(false);
-      }
-    })();
-    return () => {
-      abortController.abort();
-    };
-  }, [
-    isRequesting,
-    onComplete,
-    setProgress,
-    connector,
-    http,
-    integrationSettings,
-    reportGenerationComplete,
-    notifications?.toasts,
-  ]);
-
-  const retry = useCallback(() => {
-    setError(null);
-    setIsRequesting(true);
-  }, []);
-
-  return { progress, error, retry };
-};
-
 const useModalCss = () => {
   const { euiTheme } = useEuiTheme();
   return {

x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/use_generation.tsx

@@ -0,0 +1,203 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isEmpty } from 'lodash/fp';
+import { useCallback, useEffect, useState } from 'react';
+import type { HttpSetup } from '@kbn/core-http-browser';
+import { getLangSmithOptions } from '../../../../../common/lib/lang_smith';
+import type { Docs, ESProcessorItem, Pipeline, SamplesFormat } from '../../../../../../common';
+import {
+  type AnalyzeLogsRequestBody,
+  type CategorizationRequestBody,
+  type EcsMappingRequestBody,
+  type RelatedRequestBody,
+} from '../../../../../../common';
+import {
+  runCategorizationGraph,
+  runEcsGraph,
+  runRelatedGraph,
+  runAnalyzeLogsGraph,
+} from '../../../../../common/lib/api';
+import { useKibana } from '../../../../../common/hooks/use_kibana';
+import type { State } from '../../state';
+import * as i18n from './translations';
+import { useTelemetry } from '../../../telemetry';
+import type { ErrorCode } from '../../../../../../common/constants';
+import type { AIConnector, IntegrationSettings } from '../../types';
+
+export type OnComplete = (result: State['result']) => void;
+export const ProgressOrder = ['analyzeLogs', 'ecs', 'categorization', 'related'] as const;
+export type ProgressItem = (typeof ProgressOrder)[number];
+
+interface UseGenerationProps {
+  integrationSettings: State['integrationSettings'];
+  connector: State['connector'];
+  onComplete: OnComplete;
+}
+
+interface RunGenerationProps {
+  integrationSettings: IntegrationSettings;
+  connector: AIConnector;
+  deps: { http: HttpSetup; abortSignal: AbortSignal };
+  setProgress: (progress: ProgressItem) => void;
+}
+
+interface GenerationResults {
+  pipeline: Pipeline;
+  docs: Docs;
+  samplesFormat?: SamplesFormat;
+}
+
+export const useGeneration = ({
+  integrationSettings,
+  connector,
+  onComplete,
+}: UseGenerationProps) => {
+  const { reportGenerationComplete } = useTelemetry();
+  const { http, notifications } = useKibana().services;
+  const [progress, setProgress] = useState<ProgressItem>();
+  const [error, setError] = useState<null | string>(null);
+  const [isRequesting, setIsRequesting] = useState<boolean>(true);
+
+  useEffect(() => {
+    if (
+      !isRequesting ||
+      http == null ||
+      connector == null ||
+      integrationSettings == null ||
+      notifications?.toasts == null
+    ) {
+      return;
+    }
+    const generationStartedAt = Date.now();
+    const abortController = new AbortController();
+    const deps = { http, abortSignal: abortController.signal };
+
+    (async () => {
+      try {
+        const result = await runGeneration({ integrationSettings, connector, deps, setProgress });
+        const durationMs = Date.now() - generationStartedAt;
+        reportGenerationComplete({ connector, integrationSettings, durationMs });
+        onComplete(result);
+      } catch (e) {
+        if (abortController.signal.aborted) return;
+        const originalErrorMessage = `${e.message}${
+          e.body ? ` (${e.body.statusCode}): ${e.body.message}` : ''
+        }`;
+
+        reportGenerationComplete({
+          connector,
+          integrationSettings,
+          durationMs: Date.now() - generationStartedAt,
+          error: originalErrorMessage,
+        });
+
+        let errorMessage = originalErrorMessage;
+        const errorCode = e.body?.attributes?.errorCode as ErrorCode | undefined;
+        if (errorCode != null) {
+          errorMessage = i18n.ERROR_TRANSLATION[errorCode];
+        }
+        setError(errorMessage);
+      } finally {
+        setIsRequesting(false);
+      }
+    })();
+    return () => {
+      abortController.abort();
+    };
+  }, [
+    isRequesting,
+    onComplete,
+    setProgress,
+    connector,
+    http,
+    integrationSettings,
+    reportGenerationComplete,
+    notifications?.toasts,
+  ]);
+
+  const retry = useCallback(() => {
+    setError(null);
+    setIsRequesting(true);
+  }, []);
+
+  return { progress, error, retry };
+};
+
+async function runGeneration({
+  integrationSettings,
+  connector,
+  deps,
+  setProgress,
+}: RunGenerationProps): Promise<GenerationResults> {
+  let additionalProcessors: ESProcessorItem[] | undefined;
+  // logSamples may be modified to JSON format if they are in different formats
+  // Keeping originalLogSamples for running pipeline and generating docs
+  const originalLogSamples = integrationSettings.logSamples;
+  let logSamples = integrationSettings.logSamples;
+  let samplesFormat: SamplesFormat | undefined = integrationSettings.samplesFormat;
+
+  if (integrationSettings.samplesFormat === undefined) {
+    const analyzeLogsRequest: AnalyzeLogsRequestBody = {
+      packageName: integrationSettings.name ?? '',
+      dataStreamName: integrationSettings.dataStreamName ?? '',
+      logSamples: integrationSettings.logSamples ?? [],
+      connectorId: connector.id,
+      langSmithOptions: getLangSmithOptions(),
+    };
+
+    setProgress('analyzeLogs');
+    const analyzeLogsResult = await runAnalyzeLogsGraph(analyzeLogsRequest, deps);
+    if (isEmpty(analyzeLogsResult?.results)) {
+      throw new Error('No results from Analyze Logs Graph');
+    }
+    logSamples = analyzeLogsResult.results.parsedSamples;
+    samplesFormat = analyzeLogsResult.results.samplesFormat;
+    additionalProcessors = analyzeLogsResult.additionalProcessors;
+  }
+
+  const ecsRequest: EcsMappingRequestBody = {
+    packageName: integrationSettings.name ?? '',
+    dataStreamName: integrationSettings.dataStreamName ?? '',
+    rawSamples: logSamples ?? [],
+    samplesFormat: samplesFormat ?? { name: 'json' },
+    additionalProcessors: additionalProcessors ?? [],
+    connectorId: connector.id,
+    langSmithOptions: getLangSmithOptions(),
+  };
+
+  setProgress('ecs');
+  const ecsGraphResult = await runEcsGraph(ecsRequest, deps);
+  if (isEmpty(ecsGraphResult?.results)) {
+    throw new Error('No results from ECS graph');
+  }
+  const categorizationRequest: CategorizationRequestBody = {
+    ...ecsRequest,
+    rawSamples: originalLogSamples ?? [],
+    samplesFormat: samplesFormat ?? { name: 'json' },
+    currentPipeline: ecsGraphResult.results.pipeline,
+  };
+
+  setProgress('categorization');
+  const categorizationResult = await runCategorizationGraph(categorizationRequest, deps);
+  const relatedRequest: RelatedRequestBody = {
+    ...categorizationRequest,
+    currentPipeline: categorizationResult.results.pipeline,
+  };
+
+  setProgress('related');
+  const relatedGraphResult = await runRelatedGraph(relatedRequest, deps);
+  if (isEmpty(relatedGraphResult?.results)) {
+    throw new Error('Results not found in response');
+  }
+
+  return {
+    pipeline: relatedGraphResult.results.pipeline,
+    docs: relatedGraphResult.results.docs,
+    samplesFormat,
+  };
+}