[8.x] [Onboarding] AWS Service detection re-design for Firehose flow (#192860) (#193278)

# Backport This will backport the following commits from `main` to `8.x`: - [[Onboarding] AWS Service detection re-design for Firehose flow (#192860)](https://github.com/elastic/kibana/pull/192860)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Mykola Harmash <mykola.harmash@gmail.com>
2025-04-24 09:48:58 -04:00 · 2024-09-18 23:28:38 +10:00 · 2024-09-18 23:28:38 +10:00 · 78b577f901
commit 78b577f901
parent c8043c9a9c
12 changed files with 536 additions and 79 deletions
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/pages/firehose.tsx
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/pages/firehose.tsx
@ -19,7 +19,7 @@ export const FirehosePage = () => (
        headlineCopy={i18n.translate(
          'xpack.observability_onboarding.experimentalOnboardingFlow.customHeader.firehose.text',
          {
-            defaultMessage: 'Setting up Amazon Data Firehose',
+            defaultMessage: 'Set up Amazon Data Firehose',
          }
        )}
        captionCopy={i18n.translate(
@ -29,6 +29,7 @@ export const FirehosePage = () => (
              'This installation is tailored for setting up Firehose in your Observability project with minimal configuration.',
          }
        )}
+        isTechnicalPreview={true}
      />
    }
  >
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/auto_refresh_callout.tsx
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/auto_refresh_callout.tsx
@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import {
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiIcon,
+  EuiText,
+  useEuiTheme,
+  useGeneratedHtmlId,
+} from '@elastic/eui';
+import { css } from '@emotion/react';
+import { HAS_DATA_FETCH_INTERVAL } from './utils';
+
+export function AutoRefreshCallout() {
+  const { euiTheme } = useEuiTheme();
+  const messageId = useGeneratedHtmlId();
+
+  return (
+    <EuiFlexGroup>
+      <EuiFlexItem
+        role="status"
+        aria-labelledby={messageId}
+        grow={false}
+        css={css`
+          background-color: ${euiTheme.colors.lightestShade};
+          padding: ${euiTheme.size.m} ${euiTheme.size.base};
+          border-radius: ${euiTheme.border.radius.medium};
+        `}
+      >
+        <EuiFlexGroup gutterSize="s" alignItems="center">
+          <EuiIcon type="timeRefresh" size="m" />
+          <EuiText size="s">
+            <p id={messageId}>
+              {i18n.translate(
+                'xpack.observability_onboarding.firehosePanel.autorefreshCalloutLabel',
+                {
+                  defaultMessage: 'Auto-refreshing every {intervalSeconds} s',
+                  values: { intervalSeconds: Math.round(HAS_DATA_FETCH_INTERVAL / 1000) },
+                }
+              )}
+            </p>
+          </EuiText>
+        </EuiFlexGroup>
+      </EuiFlexItem>
+    </EuiFlexGroup>
+  );
+}
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx
@ -5,7 +5,6 @@
 * 2.0.
 */

-import React from 'react';
 import {
  EuiAccordion,
  EuiCodeBlock,
@ -14,19 +13,20 @@ import {
  EuiText,
  useGeneratedHtmlId,
 } from '@elastic/eui';
-import { FormattedMessage } from '@kbn/i18n-react';
 import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n-react';
+import React from 'react';
 import {
  FIREHOSE_CLOUDFORMATION_STACK_NAME,
  FIREHOSE_LOGS_STREAM_NAME,
  FIREHOSE_METRICS_STREAM_NAME,
 } from '../../../../common/aws_firehose';
 import { CopyToClipboardButton } from '../shared/copy_to_clipboard_button';
+import { DownloadTemplateCallout } from './download_template_callout';
 import { buildCreateStackCommand, buildStackStatusCommand } from './utils';

 interface Props {
  encodedApiKey: string;
-  onboardingId: string;
  elasticsearchUrl: string;
  templateUrl: string;
  isCopyPrimaryAction: boolean;
@ -57,7 +57,7 @@ export function CreateStackCommandSnippet({
        <p>
          <FormattedMessage
            id="xpack.observability_onboarding.firehosePanel.createFirehoseStreamDescription"
-            defaultMessage="Run the command bellow in your terminal where you have {awsCLIInstallGuideLink} configured. The command will create a CloudFormation stack that includes a Firehose delivery, backup S3 bucket, CloudWatch subscription filter and metrics stream along with required IAM roles."
+            defaultMessage="Run the command bellow in your terminal where you have {awsCLIInstallGuideLink} configured. The command will create a CloudFormation stack from our template that includes a Firehose delivery, backup S3 bucket, CloudWatch subscription filter and metrics stream along with required IAM roles."
            values={{
              awsCLIInstallGuideLink: (
                <EuiLink
@ -75,6 +75,10 @@ export function CreateStackCommandSnippet({
            }}
          />
        </p>
+
+        <p>
+          <DownloadTemplateCallout />
+        </p>
      </EuiText>

      <EuiSpacer />
@ -94,7 +98,15 @@ export function CreateStackCommandSnippet({

      <EuiSpacer />

-      <EuiAccordion id={stackStatusAccordionId} buttonContent="Check stack status">
+      <EuiAccordion
+        id={stackStatusAccordionId}
+        buttonContent={i18n.translate(
+          'xpack.observability_onboarding.firehosePanel.stackStatusAccordionButtonLabel',
+          {
+            defaultMessage: 'Check status of the CloudFormation stack',
+          }
+        )}
+      >
        <EuiSpacer size="xs" />
        <EuiCodeBlock language="text" paddingSize="m" fontSize="m" isCopyable>
          {stackStatusCommand}
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx
@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiButton, EuiSpacer, EuiText } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n-react';
+import React from 'react';
+import {
+  FIREHOSE_CLOUDFORMATION_STACK_NAME,
+  FIREHOSE_LOGS_STREAM_NAME,
+  FIREHOSE_METRICS_STREAM_NAME,
+} from '../../../../common/aws_firehose';
+import { DownloadTemplateCallout } from './download_template_callout';
+import { buildCreateStackAWSConsoleURL } from './utils';
+
+interface Props {
+  encodedApiKey: string;
+  elasticsearchUrl: string;
+  templateUrl: string;
+  isPrimaryAction: boolean;
+}
+
+export function CreateStackInAWSConsole({
+  encodedApiKey,
+  elasticsearchUrl,
+  templateUrl,
+  isPrimaryAction,
+}: Props) {
+  const awsConsoleURL = buildCreateStackAWSConsoleURL({
+    templateUrl,
+    stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
+    logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
+    metricsStreamName: FIREHOSE_METRICS_STREAM_NAME,
+    elasticsearchUrl,
+    encodedApiKey,
+  });
+
+  return (
+    <>
+      <EuiText>
+        <p>
+          <FormattedMessage
+            id="xpack.observability_onboarding.firehosePanel.createFirehoseStreamInAWSConsoleDescription"
+            defaultMessage="Click the button below to create a CloudFormation stack from our template. The stack will include a Firehose delivery stream, backup S3 bucket, CloudWatch subscription filter, metrics stream, and necessary IAM roles. Keep this page open, and return once you've submitted the form in AWS Console"
+          />
+        </p>
+        <p>
+          <DownloadTemplateCallout />
+        </p>
+      </EuiText>
+
+      <EuiSpacer size="m" />
+
+      <EuiButton
+        data-test-subj="observabilityOnboardingCreateStackInAWSConsoleButton"
+        href={awsConsoleURL}
+        target="_blank"
+        iconSide="right"
+        iconType="popout"
+        fill={isPrimaryAction}
+      >
+        {i18n.translate(
+          'xpack.observability_onboarding.createStackInAWSConsole.createFirehoseStreamInAWSConsoleButtonLabel',
+          { defaultMessage: 'Create Firehose Stream in AWS' }
+        )}
+      </EuiButton>
+    </>
+  );
+}
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/download_template_callout.tsx
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/download_template_callout.tsx
@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { EuiLink } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { FIREHOSE_CLOUDFORMATION_TEMPLATE_URL } from '../../../../common/aws_firehose';
+
+export function DownloadTemplateCallout() {
+  return (
+    <FormattedMessage
+      id="xpack.observability_onboarding.firehosePanel.downloadTemplateDescription"
+      defaultMessage="If needed, you can {downloadLink} to use it as part of an existing IaC setup."
+      values={{
+        downloadLink: (
+          <EuiLink
+            data-test-subj="observabilityOnboardingFirehosePanelDownloadCloudFormationTemplateLink"
+            href={FIREHOSE_CLOUDFORMATION_TEMPLATE_URL}
+            download={true}
+          >
+            {i18n.translate(
+              'xpack.observability_onboarding.firehosePanel.downloadCloudFormationTemplateButtonLabel',
+              { defaultMessage: 'download and modify the CloudFormation template' }
+            )}
+          </EuiLink>
+        ),
+      }}
+    />
+  );
+}
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx
@ -5,23 +5,57 @@
 * 2.0.
 */

-import React, { useState } from 'react';
+import React, { useCallback, useState } from 'react';
+import { i18n } from '@kbn/i18n';
 import {
+  EuiButtonGroup,
+  EuiLink,
  EuiPanel,
  EuiSkeletonRectangle,
  EuiSkeletonText,
  EuiSpacer,
  EuiSteps,
  EuiStepStatus,
+  EuiText,
 } from '@elastic/eui';
 import useEvent from 'react-use/lib/useEvent';
+import { FormattedMessage } from '@kbn/i18n-react';
 import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
 import { EmptyPrompt } from '../shared/empty_prompt';
 import { CreateStackCommandSnippet } from './create_stack_command_snippet';
 import { VisualizeData } from './visualize_data';
+import { CreateStackInAWSConsole } from './create_stack_in_aws_console';
+import { FeedbackButtons } from '../shared/feedback_buttons';
+
+enum CreateStackOption {
+  AWS_CONSOLE_UI = 'createCloudFormationOptionAWSConsoleUI',
+  AWS_CLI = 'createCloudFormationOptionAWSCLI',
+}
+
+const OPTIONS = [
+  {
+    id: CreateStackOption.AWS_CONSOLE_UI,
+    label: i18n.translate(
+      'xpack.observability_onboarding.firehosePanel.createStackAWSConsoleOptionLabel',
+      {
+        defaultMessage: 'Via AWS Console',
+      }
+    ),
+  },
+  {
+    id: CreateStackOption.AWS_CLI,
+    label: i18n.translate(
+      'xpack.observability_onboarding.firehosePanel.createStackAWSCLIOptionLabel',
+      { defaultMessage: 'Via AWS CLI' }
+    ),
+  },
+];

 export function FirehosePanel() {
  const [windowLostFocus, setWindowLostFocus] = useState(false);
+  const [selectedOptionId, setSelectedOptionId] = useState<CreateStackOption>(
+    CreateStackOption.AWS_CONSOLE_UI
+  );
  const { data, status, error, refetch } = useFetcher(
    (callApi) => {
      return callApi('POST /internal/observability_onboarding/firehose/flow');
@ -32,6 +66,10 @@ export function FirehosePanel() {

  useEvent('blur', () => setWindowLostFocus(true), window);

+  const onOptionChange = useCallback((id: string) => {
+    setSelectedOptionId(id as CreateStackOption);
+  }, []);
+
  if (error !== undefined) {
    return <EmptyPrompt error={error} onRetryClick={refetch} />;
  }
@ -41,7 +79,45 @@ export function FirehosePanel() {

  const steps = [
    {
-      title: 'Create a Firehose delivery stream and ingest CloudWatch logs',
+      title: i18n.translate('xpack.observability_onboarding.firehosePanel.prerequisitesTitle', {
+        defaultMessage: 'Prerequisites',
+      }),
+      children: (
+        <>
+          <EuiText>
+            <p>
+              <FormattedMessage
+                id="xpack.observability_onboarding.firehosePanel.prerequisitesDescription"
+                defaultMessage="You must have an active AWS account and the necessary permissions to create delivery streams."
+              />
+            </p>
+            <p>
+              <FormattedMessage
+                id="xpack.observability_onboarding.firehosePanel.prerequisitesDocumentation"
+                defaultMessage="{documentationLink} for more info."
+                values={{
+                  documentationLink: (
+                    <EuiLink
+                      data-test-subj="observabilityOnboardingFirehosePanelCheckTheDocumentationLink"
+                      href="https://www.elastic.co/docs/current/integrations/awsfirehose"
+                      external
+                      target="_blank"
+                    >
+                      {i18n.translate(
+                        'xpack.observability_onboarding.firehosePanel.documentationLinkLabel',
+                        { defaultMessage: 'Check the documentation' }
+                      )}
+                    </EuiLink>
+                  ),
+                }}
+              />
+            </p>
+          </EuiText>
+        </>
+      ),
+    },
+    {
+      title: 'Create a Firehose delivery stream to ingest CloudWatch logs and metrics',
      children: (
        <>
          {status !== FETCH_STATUS.SUCCESS && (
@ -52,13 +128,41 @@ export function FirehosePanel() {
            </>
          )}
          {status === FETCH_STATUS.SUCCESS && data !== undefined && (
-            <CreateStackCommandSnippet
-              templateUrl={data.templateUrl}
-              encodedApiKey={data.apiKeyEncoded}
-              onboardingId={data.onboardingId}
-              elasticsearchUrl={data.elasticsearchUrl}
-              isCopyPrimaryAction={!isVisualizeStepActive}
-            />
+            <>
+              <EuiButtonGroup
+                legend={i18n.translate(
+                  'xpack.observability_onboarding.firehosePanel.createStackOptionsLegend',
+                  {
+                    defaultMessage: 'Select a preferred option to create a CloudFormation stack',
+                  }
+                )}
+                type="single"
+                buttonSize="m"
+                idSelected={selectedOptionId}
+                onChange={onOptionChange}
+                options={OPTIONS}
+              />
+
+              <EuiSpacer size="l" />
+
+              {selectedOptionId === CreateStackOption.AWS_CONSOLE_UI && (
+                <CreateStackInAWSConsole
+                  templateUrl={data.templateUrl}
+                  encodedApiKey={data.apiKeyEncoded}
+                  elasticsearchUrl={data.elasticsearchUrl}
+                  isPrimaryAction={!isVisualizeStepActive}
+                />
+              )}
+
+              {selectedOptionId === CreateStackOption.AWS_CLI && (
+                <CreateStackCommandSnippet
+                  templateUrl={data.templateUrl}
+                  encodedApiKey={data.apiKeyEncoded}
+                  elasticsearchUrl={data.elasticsearchUrl}
+                  isCopyPrimaryAction={!isVisualizeStepActive}
+                />
+              )}
+            </>
          )}
        </>
      ),
@ -73,6 +177,7 @@ export function FirehosePanel() {
  return (
    <EuiPanel hasBorder paddingSize="xl">
      <EuiSteps steps={steps} />
+      <FeedbackButtons flow="firehose" />
    </EuiPanel>
  );
 }
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/progress_callout.tsx
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/progress_callout.tsx
@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiFlexGroup, EuiText, EuiIconTip, EuiHorizontalRule } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import React from 'react';
+import { css } from '@emotion/react';
+import { ProgressIndicator } from '../shared/progress_indicator';
+
+const SUPPORTED_SERVICES = [
+  'API Gateway',
+  'AWS Usage',
+  'CloudTrail',
+  'DynamoDB',
+  'EBS',
+  'EC2',
+  'ECS',
+  'ELB',
+  'EMR',
+  'Kinesis Data Stream',
+  'Lambda',
+  'MSK',
+  'NAT Gateway',
+  'RDS',
+  'Route53',
+  'S3',
+  'SNS',
+  'SQS',
+  'VPC',
+  'VPN',
+];
+
+export function ProgressCallout() {
+  return (
+    <ProgressIndicator
+      data-test-subj="observabilityOnboardingFirehoseProgressCallout"
+      title={
+        <EuiFlexGroup alignItems="center" gutterSize="s">
+          <EuiText>
+            <p>
+              {i18n.translate('xpack.observability_onboarding.firehosePanel.waitingForDataTitle', {
+                defaultMessage: 'Retrieving data from Amazon Data Firehose',
+              })}
+            </p>
+          </EuiText>
+          <EuiIconTip
+            content={
+              <EuiText size="s">
+                <strong>
+                  {i18n.translate(
+                    'xpack.observability_onboarding.progressCallout.strong.allServicesWeCanLabel',
+                    { defaultMessage: 'All services we can detect' }
+                  )}
+                </strong>
+                <EuiHorizontalRule margin="xs" />
+                <ul>
+                  {SUPPORTED_SERVICES.map((service) => (
+                    <li key={service}>{service}</li>
+                  ))}
+                  <li>
+                    {i18n.translate(
+                      'xpack.observability_onboarding.progressCallout.li.otherLabel',
+                      {
+                        defaultMessage:
+                          'Other (Unsupported logs will be stored in a generic Firehose index).',
+                      }
+                    )}
+                  </li>
+                </ul>
+              </EuiText>
+            }
+            position="top"
+            type="iInCircle"
+          />
+        </EuiFlexGroup>
+      }
+      isLoading={true}
+      css={css`
+        display: inline-block;
+      `}
+    />
+  );
+}
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_aws_service_get_started_list.ts
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_aws_service_get_started_list.ts
@ -98,7 +98,7 @@ export function useAWSServiceGetStartedList(): AWSServiceGetStartedConfig[] {

  const generateMetricsDiscoverActionLink = useCallback(
    (namespace: string, name: string) => ({
-      id: `logs-explorer-${namespace}`,
+      id: `discover-${namespace}`,
      title: i18n.translate('xpack.observability_onboarding.firehosePanel.exploreDataTitle', {
        defaultMessage: 'See {name} metrics data in Discover',
        values: { name },
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts
@ -5,6 +5,8 @@
 * 2.0.
 */

+export const HAS_DATA_FETCH_INTERVAL = 5000;
+
 export function buildCreateStackCommand({
  templateUrl,
  stackName,
@ -48,3 +50,41 @@ export function buildStackStatusCommand({ stackName }: { stackName: string }) {
    .replace(/\n/g, ' ')
    .replace(/\s\s+/g, ' ');
 }
+
+export function buildCreateStackAWSConsoleURL({
+  templateUrl,
+  stackName,
+  logsStreamName,
+  metricsStreamName,
+  elasticsearchUrl,
+  encodedApiKey,
+}: {
+  templateUrl: string;
+  stackName: string;
+  logsStreamName: string;
+  metricsStreamName: string;
+  elasticsearchUrl: string;
+  encodedApiKey: string;
+}): string {
+  const url = new URL('https://console.aws.amazon.com');
+  const params = new URLSearchParams({
+    templateURL: templateUrl,
+    stackName,
+    /**
+     * 'param_' format is enforced by AWS
+     * but template parameters are in CamelCase
+     * which triggers the eslint rule.
+     */
+    /* eslint-disable @typescript-eslint/naming-convention */
+    param_FirehoseStreamNameForLogs: logsStreamName,
+    param_FirehoseStreamNameForMetrics: metricsStreamName,
+    param_ElasticEndpointURL: elasticsearchUrl,
+    param_ElasticAPIKey: encodedApiKey,
+    /* eslint-enable @typescript-eslint/naming-convention */
+  });
+
+  url.pathname = '/cloudformation/home';
+  url.hash = `/stacks/quickcreate?${params.toString()}`;
+
+  return url.toString();
+}
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx
@ -5,27 +5,35 @@
 * 2.0.
 */

-import { EuiIcon, EuiSpacer, useEuiTheme, useGeneratedHtmlId } from '@elastic/eui';
-import { css } from '@emotion/react';
+import { EuiIcon, EuiSpacer, EuiText, useGeneratedHtmlId } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import React from 'react';
+import React, { useEffect, useState } from 'react';
 import useInterval from 'react-use/lib/useInterval';
+import { union } from 'lodash';
+import { useKibana } from '@kbn/kibana-react-plugin/public';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { ObservabilityOnboardingAppServices } from '../../..';
 import {
  FIREHOSE_CLOUDFORMATION_STACK_NAME,
  FIREHOSE_LOGS_STREAM_NAME,
+  type AWSIndexName,
 } from '../../../../common/aws_firehose';
 import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
 import { AccordionWithIcon } from '../shared/accordion_with_icon';
 import { GetStartedPanel } from '../shared/get_started_panel';
-import { ProgressIndicator } from '../shared/progress_indicator';
 import { useAWSServiceGetStartedList } from './use_aws_service_get_started_list';
+import { AutoRefreshCallout } from './auto_refresh_callout';
+import { ProgressCallout } from './progress_callout';
+import { HAS_DATA_FETCH_INTERVAL } from './utils';

-const FETCH_INTERVAL = 2000;
 const REQUEST_PENDING_STATUS_LIST = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED];

 export function VisualizeData() {
  const accordionId = useGeneratedHtmlId({ prefix: 'accordion' });
-  const { euiTheme } = useEuiTheme();
+  const [orderedPopulatedAWSLogsIndexList, setOrderedPopulatedAWSLogsIndexList] = useState<
+    AWSIndexName[]
+  >([]);
+  const [shouldShowDataReceivedToast, setShouldShowDataReceivedToast] = useState<boolean>(true);
  const {
    data: populatedAWSLogsIndexList,
    status,
@ -40,6 +48,49 @@ export function VisualizeData() {
      },
    });
  }, []);
+  const {
+    services: { notifications },
+  } = useKibana<ObservabilityOnboardingAppServices>();
+
+  useEffect(() => {
+    if (
+      shouldShowDataReceivedToast &&
+      Array.isArray(populatedAWSLogsIndexList) &&
+      populatedAWSLogsIndexList.length > 0
+    ) {
+      notifications?.toasts.addSuccess(
+        {
+          title: i18n.translate(
+            'xpack.observability_onboarding.firehosePanel.dataReceivedToastTitle',
+            {
+              defaultMessage: 'Your data is on its way',
+            }
+          ),
+          text: i18n.translate(
+            'xpack.observability_onboarding.firehosePanel.dataReceivedToastText',
+            {
+              defaultMessage:
+                'We’ve begun processing your data. In the background, we automatically refresh every few seconds to capture more incoming data.',
+            }
+          ),
+        },
+        {
+          toastLifeTimeMs: 10000,
+        }
+      );
+      setShouldShowDataReceivedToast(false);
+    }
+
+    setOrderedPopulatedAWSLogsIndexList((currentList) =>
+      /**
+       * Using union() to ensure items in the array are unique
+       * add stay in the insertion order to keep the order of
+       * the AWS services in the UI.
+       */
+      union(currentList, populatedAWSLogsIndexList)
+    );
+  }, [notifications?.toasts, populatedAWSLogsIndexList, shouldShowDataReceivedToast]);
+
  const awsServiceGetStartedConfigList = useAWSServiceGetStartedList();

  useInterval(() => {
@ -48,7 +99,7 @@ export function VisualizeData() {
    }

    refetch();
-  }, FETCH_INTERVAL);
+  }, HAS_DATA_FETCH_INTERVAL);

  if (populatedAWSLogsIndexList === undefined) {
    return null;
@ -56,58 +107,58 @@ export function VisualizeData() {

  return (
    <>
-      <ProgressIndicator
-        title={i18n.translate('xpack.observability_onboarding.firehosePanel.waitingForDataTitle', {
-          defaultMessage: 'Waiting for data from the Firehose stream',
-        })}
-        isLoading={true}
-        css={css`
-          max-width: 40%;
-        `}
-      />
+      <EuiText>
+        <p>
+          <FormattedMessage
+            id="xpack.observability_onboarding.firehosePanel.visualizeDataDescription"
+            defaultMessage="Once the Firehose stream is created, data capture will begin automatically, and the incoming data will be displayed below."
+          />
+        </p>
+      </EuiText>

-      <EuiSpacer size="xl" />
+      <EuiSpacer size="m" />
+
+      {orderedPopulatedAWSLogsIndexList.length === 0 && <ProgressCallout />}
+      {orderedPopulatedAWSLogsIndexList.length > 0 && <AutoRefreshCallout />}
+
+      <EuiSpacer size="m" />

      <div data-test-subj="observabilityOnboardingAWSServiceList">
-        {awsServiceGetStartedConfigList.map(
-          ({ id, indexNameList, actionLinks, title, logoURL, previewImage }) => {
-            const isEnabled = indexNameList.some((indexName) =>
-              populatedAWSLogsIndexList.includes(indexName)
-            );
+        {orderedPopulatedAWSLogsIndexList.map((indexName, index) => {
+          const getStartedConfig = awsServiceGetStartedConfigList.find(({ indexNameList }) =>
+            indexNameList.includes(indexName)
+          );

-            return (
-              <AccordionWithIcon
-                data-test-subj={`observabilityOnboardingAWSService-${id}`}
-                key={id}
-                id={`${accordionId}_${id}`}
-                icon={<EuiIcon type={logoURL} size="l" />}
-                title={i18n.translate(
-                  'xpack.observability_onboarding.firehosePanel.awsServiceDataFoundTitle',
-                  {
-                    defaultMessage: '{title}',
-                    values: { title },
-                  }
-                )}
-                extraAction={
-                  isEnabled ? <EuiIcon type="checkInCircleFilled" color="success" /> : null
-                }
-                isDisabled={!isEnabled}
-                css={{
-                  paddingRight: euiTheme.size.s,
-                  filter: `grayscale(${isEnabled ? 0 : 1})`,
-                }}
-              >
-                <GetStartedPanel
-                  integration="aws"
-                  newTab
-                  isLoading={false}
-                  actionLinks={actionLinks}
-                  previewImage={previewImage}
-                />
-              </AccordionWithIcon>
-            );
+          if (!getStartedConfig) {
+            return null;
          }
-        )}
+
+          const { id, actionLinks, title, logoURL, previewImage } = getStartedConfig;
+
+          return (
+            <AccordionWithIcon
+              data-test-subj={`observabilityOnboardingAWSService-${id}`}
+              key={id}
+              id={`${accordionId}_${id}`}
+              icon={<EuiIcon type={logoURL} size="l" />}
+              title={title}
+              initialIsOpen={true}
+              borders={
+                index === 0 || index === orderedPopulatedAWSLogsIndexList.length - 1
+                  ? 'none'
+                  : 'horizontal'
+              }
+            >
+              <GetStartedPanel
+                integration="aws"
+                newTab
+                isLoading={false}
+                actionLinks={actionLinks}
+                previewImage={previewImage}
+              />
+            </AccordionWithIcon>
+          );
+        })}
      </div>
    </>
  );
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/accordion_with_icon.tsx
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/accordion_with_icon.tsx
@ -15,13 +15,14 @@ import {
 } from '@elastic/eui';

 interface AccordionWithIconProps
-  extends Omit<EuiAccordionProps, 'buttonContent' | 'buttonProps' | 'borders' | 'paddingSize'> {
+  extends Omit<EuiAccordionProps, 'buttonContent' | 'buttonProps' | 'paddingSize'> {
  title: string;
  icon: React.ReactNode;
 }
 export const AccordionWithIcon: FunctionComponent<AccordionWithIconProps> = ({
  title,
  icon,
+  borders = 'horizontal',
  children,
  ...rest
 }) => {
@ -39,7 +40,7 @@ export const AccordionWithIcon: FunctionComponent<AccordionWithIconProps> = ({
        </EuiFlexGroup>
      }
      buttonProps={{ paddingSize: 'l' }}
-      borders="horizontal"
+      borders={borders}
      paddingSize="none"
    >
      <div css={{ paddingLeft: 36, paddingBottom: 24 }}>{children}</div>
--- a/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts
+++ b/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts
@ -38,6 +38,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
    });

    beforeEach(async () => {
+      await (await testSubjects.find('createCloudFormationOptionAWSCLI')).click();
      await testSubjects.existOrFail('observabilityOnboardingFirehoseCreateStackCommand');
    });

@ -51,10 +52,10 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
    it('starts to monitor for incoming data after user leaves the page', async () => {
      await browser.execute(`window.dispatchEvent(new Event("blur"))`);

-      await testSubjects.isDisplayed('observabilityOnboardingAWSServiceList');
+      await testSubjects.isDisplayed('observabilityOnboardingFirehoseProgressCallout');
    });

-    it('highlights an AWS service when data is detected', async () => {
+    it('shows an AWS service when data is detected', async () => {
      const DATASET = 'aws.vpcflow';
      const AWS_SERVICE_ID = 'vpc-flow';
      await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton');
@ -79,11 +80,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
          })
      );

-      // Checking that an AWS service item is enabled after data is detected
-      await testSubjects
-        .find(`observabilityOnboardingAWSService-${AWS_SERVICE_ID}`)
-        .then((el) => el.findByTagName('button'))
-        .then((el) => el.isEnabled());
+      // Checking that an AWS service item is visible after data is detected
+      await testSubjects.isDisplayed(`observabilityOnboardingAWSService-${AWS_SERVICE_ID}`);
    });
  });
 }