[Oblt Onboarding] Expand e2e test coverage for k8s OTel flow (#217958)

Closes https://github.com/elastic/kibana/issues/207736

This change expands the current e2e ensemble test for K8S OTel flow to
also check application instrumentation. The test opens the APM service
inventory, finds and click on the instrumented application and checks
that at least one transaction is visible on the service details screen.

[🔒 Corresponding PR in Ensemble
repo](https://github.com/elastic/ensemble/pull/536).

> [!NOTE]
> This change needs to be merged and deployed on edge before we can
enable the test on Ensemble side because a couple of test IDs were added
to the UI

### How to test

1. Create a local K8S cluster (e.g. using minikube)
2. Add a java application to the cluster which later be instrumented
using these commands:
```
kubectl create namespace java
kubectl apply -f - <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: java-app
  name: java-app
  namespace: java
spec:
  replicas: 1
  selector:
    matchLabels:
      app: java-app
  template:
    metadata:
      labels:
        app: java-app
    spec:
      containers:
      - name: java-app
        image: docker.elastic.co/demos/apm/k8s-webhook-test
        env:
        - name: OTEL_INSTRUMENTATION_METHODS_INCLUDE
          value: "test.Testing[methodB]"
EOF
```
4. Run kibana locally and navigate to the K8S OTel flow
5. Run the Playwright test:
```
npx playwright test -c ./x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/playwright.config.ts --project stateful --reporter list --headed x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts
```
6. Once the test pauses, go through the onboarding instructions,
including the instrumentation part (use `java-app` as application name
and `java` as the namespace in the instrumentation command)
7. Wait for the test to un-pause and continue

x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/README.md

@@ -7,7 +7,7 @@ Playwright tests are only responsible for UI checks and do not automate onboardi
 ## Running The Tests Locally
 
 1. Run ES and Kibana
-2. Create a `.env` file in the `./x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/` directory with the following content (adjust the values like Kibana URL according yo your local setup):
+2. Create a `.env` file in the `./x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/` directory with the following content (adjust the values according to your local setup):
 ```bash
 KIBANA_BASE_URL = "http://localhost:5601/ftw"
 ELASTICSEARCH_HOST = "http://localhost:9200"
@@ -16,10 +16,10 @@ KIBANA_PASSWORD = "changeme"
 CLUSTER_ENVIRONMENT = local
 ARTIFACTS_FOLDER = ./.playwright
 ```
-3. Run the `playwright test`
+1. Run the `playwright test`
 ```bash
 # Assuming the working directory is the root of the Kibana repo
 npx playwright test -c ./x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/playwright.config.ts --project stateful --reporter list --headed
 ```
-4. Once the test reaches one of the required manual steps, like executing auto-detect command snippet, do the step manually.
-5. The test will proceed once the manual step is done.
+1. Once the test reaches one of the required manual steps, like executing auto-detect command snippet, do the step manually.
+2. The test will proceed once the manual step is done.

x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/fixtures/base_page.ts

@@ -49,8 +49,8 @@ export const test = base.extend<{
     await use(new KubernetesEAFlowPage(page));
   },
 
-  otelKubernetesFlowPage: async ({ page }, use) => {
-    await use(new OtelKubernetesFlowPage(page));
+  otelKubernetesFlowPage: async ({ page, context }, use) => {
+    await use(new OtelKubernetesFlowPage(page, context));
   },
 
   kubernetesOverviewDashboardPage: async ({ page }, use) => {

x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts

@@ -9,17 +9,22 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { test } from './fixtures/base_page';
 import { assertEnv } from '../lib/assert_env';
+import { OtelKubernetesOverviewDashboardPage } from './pom/pages/otel_kubernetes_overview_dashboard.page';
+import { ApmServiceInventoryPage } from './pom/pages/apm_service_inventory.page';
 
 test.beforeEach(async ({ page }) => {
   await page.goto(`${process.env.KIBANA_BASE_URL}/app/observabilityOnboarding`);
 });
 
-test('Otel Kubernetes', async ({
-  page,
-  onboardingHomePage,
-  otelKubernetesFlowPage,
-  otelKubernetesOverviewDashboardPage,
-}) => {
+/**
+ * These constants are used by Ensemble test
+ * when creating the app container. They should
+ * be kept in sync.
+ */
+const INSTRUMENTED_APP_CONTAINER_NAMESPACE = 'java';
+const INSTRUMENTED_APP_NAME = 'java-app';
+
+test('Otel Kubernetes', async ({ page, onboardingHomePage, otelKubernetesFlowPage }) => {
   assertEnv(process.env.ARTIFACTS_FOLDER, 'ARTIFACTS_FOLDER is not defined.');
 
   const fileName = 'code_snippet_otel_kubernetes.sh';
@@ -34,7 +39,20 @@ test('Otel Kubernetes', async ({
   await otelKubernetesFlowPage.copyInstallStackSnippetToClipboard();
   const installStackSnippet = (await page.evaluate('navigator.clipboard.readText()')) as string;
 
-  const codeSnippet = `${helmRepoSnippet}\n${installStackSnippet}`;
+  /**
+   * Getting the snippets and replacing placeholder
+   * with the values used by Ensemble
+   */
+  await otelKubernetesFlowPage.switchInstrumentationInstructions('java');
+  const annotateAllResourceSnippet = (
+    await otelKubernetesFlowPage.getAnnotateAllResourceSnippet()
+  )?.replace('my-namespace', INSTRUMENTED_APP_CONTAINER_NAMESPACE);
+  const restartDeploymentSnippet = (await otelKubernetesFlowPage.getRestartDeploymentSnippet())
+    ?.split('\n')[0]
+    ?.replace('myapp', INSTRUMENTED_APP_NAME)
+    ?.replace('my-namespace', INSTRUMENTED_APP_CONTAINER_NAMESPACE);
+
+  const codeSnippet = `${helmRepoSnippet}\n${installStackSnippet}\n${annotateAllResourceSnippet}\n${restartDeploymentSnippet}`;
 
   /**
    * Ensemble story watches for the code snippet file
@@ -45,11 +63,19 @@ test('Otel Kubernetes', async ({
   /**
    * There is no explicit data ingest indication
    * in the flow, so we need to rely on a timeout.
-   * 3 minutes should be enough for the stack to be
+   * 5 minutes should be enough for the stack to be
    * created and to start pushing data.
    */
-  await page.waitForTimeout(3 * 60000);
+  await page.waitForTimeout(5 * 60000);
 
-  await otelKubernetesFlowPage.clickClusterOverviewDashboardCTA();
+  const otelKubernetesOverviewDashboardPage = new OtelKubernetesOverviewDashboardPage(
+    await otelKubernetesFlowPage.openClusterOverviewDashboardInNewTab()
+  );
   await otelKubernetesOverviewDashboardPage.assertNodesPanelNotEmpty();
+
+  const apmServiceInventoryPage = new ApmServiceInventoryPage(
+    await otelKubernetesFlowPage.openServiceInventoryInNewTab()
+  );
+  await apmServiceInventoryPage.page.getByTestId('serviceLink_opentelemetry/java/elastic').click();
+  await apmServiceInventoryPage.assertTransactionExists();
 });

x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/apm_service_inventory.page.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { expect, type Page } from '@playwright/test';
+
+export class ApmServiceInventoryPage {
+  page: Page;
+
+  constructor(page: Page) {
+    this.page = page;
+  }
+
+  public async assertTransactionExists() {
+    await expect(this.page.getByTestId('apmTransactionDetailLinkLink')).toBeVisible();
+  }
+}

x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts

@@ -5,13 +5,15 @@
  * 2.0.
  */
 
-import { Page } from '@playwright/test';
+import { Page, BrowserContext } from '@playwright/test';
 
 export class OtelKubernetesFlowPage {
   page: Page;
+  context: BrowserContext;
 
-  constructor(page: Page) {
+  constructor(page: Page, context: BrowserContext) {
     this.page = page;
+    this.context = context;
   }
 
   public async copyHelmRepositorySnippetToClipboard() {
@@ -26,11 +28,53 @@ export class OtelKubernetesFlowPage {
       .click();
   }
 
-  public async clickClusterOverviewDashboardCTA() {
-    await this.page
+  public async switchInstrumentationInstructions(language: 'nodejs' | 'java' | 'python' | 'go') {
+    await this.page.getByTestId(language).click();
+  }
+
+  public async getAnnotateAllResourceSnippet() {
+    return await this.page
+      .getByTestId('observabilityOnboardingOtelKubernetesPanelAnnotateAllResourcesSnippet')
+      .textContent();
+  }
+
+  public async getRestartDeploymentSnippet() {
+    return await this.page
+      .getByTestId('observabilityOnboardingOtelKubernetesPanelRestartDeploymentSnippet')
+      .textContent();
+  }
+
+  public async openClusterOverviewDashboardInNewTab(): Promise<Page> {
+    const dashboardURL = await this.page
       .getByTestId(
         'observabilityOnboardingDataIngestStatusActionLink-kubernetes_otel-cluster-overview'
       )
-      .click();
+      .getAttribute('href');
+
+    if (dashboardURL) {
+      const newPage = await this.context.newPage();
+
+      await newPage.goto(dashboardURL);
+
+      return newPage;
+    } else {
+      throw new Error('Dashboard URL not found');
+    }
+  }
+
+  public async openServiceInventoryInNewTab(): Promise<Page> {
+    const serviceInventoryURL = await this.page
+      .getByTestId('observabilityOnboardingDataIngestStatusActionLink-services')
+      .getAttribute('href');
+
+    if (serviceInventoryURL) {
+      const newPage = await this.context.newPage();
+
+      await newPage.goto(serviceInventoryURL);
+
+      return newPage;
+    } else {
+      throw new Error('Service Inventory URL not found');
+    }
   }
 }

x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx

@@ -306,7 +306,12 @@ spec:
                     { defaultMessage: 'Annotate all resources in a namespace' }
                   )}
                 >
-                  <EuiCodeBlock paddingSize="m" language="bash" isCopyable={true}>
+                  <EuiCodeBlock
+                    paddingSize="m"
+                    language="bash"
+                    isCopyable={true}
+                    data-test-subj="observabilityOnboardingOtelKubernetesPanelAnnotateAllResourcesSnippet"
+                  >
                     {`kubectl annotate namespace my-namespace instrumentation.opentelemetry.io/inject-${idSelected}="${namespace}/elastic-instrumentation"`}
                   </EuiCodeBlock>
                 </EuiAccordion>
@@ -325,7 +330,12 @@ spec:
                   )}
                 </p>
                 <EuiSpacer />
-                <EuiCodeBlock paddingSize="m" language="bash" isCopyable={true}>
+                <EuiCodeBlock
+                  paddingSize="m"
+                  language="bash"
+                  isCopyable={true}
+                  data-test-subj="observabilityOnboardingOtelKubernetesPanelRestartDeploymentSnippet"
+                >
                   {`kubectl rollout restart deployment myapp -n my-namespace
 
 kubectl describe pod <myapp-pod-name> -n my-namespace`}