[Obs AI Assistant] E2E test for contextual insights (#182715)

This adds an e2e test for the AI Assistant Contextual Insights on the
APM Error details view.

Follow-up to: https://github.com/elastic/kibana/pull/182572

x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight_base.tsx

@@ -138,7 +138,12 @@ export function InsightBase({
         onToggle={onToggle}
       >
         <EuiSpacer size="m" />
-        <EuiPanel hasBorder={false} hasShadow={false} color="subdued">
+        <EuiPanel
+          hasBorder={false}
+          hasShadow={false}
+          color="subdued"
+          data-test-subj="obsAiAssistantInsightResponse"
+        >
           {children}
         </EuiPanel>
       </EuiAccordion>

x-pack/test/apm_api_integration/common/bootstrap_apm_synthtrace.ts

@@ -14,7 +14,7 @@ import url from 'url';
 import { kbnTestConfig } from '@kbn/test';
 import { InheritedFtrProviderContext } from './ftr_provider_context';
 
-export async function bootstrapApmSynthtrace(
+export async function getApmSynthtraceEsClient(
   context: InheritedFtrProviderContext,
   kibanaClient: ApmSynthtraceKibanaClient
 ) {

x-pack/test/apm_api_integration/common/config.ts

@@ -21,7 +21,7 @@ import { format, UrlObject } from 'url';
 import { MachineLearningAPIProvider } from '../../functional/services/ml/api';
 import { APMFtrConfigName } from '../configs';
 import { createApmApiClient } from './apm_api_supertest';
-import { bootstrapApmSynthtrace, getApmSynthtraceKibanaClient } from './bootstrap_apm_synthtrace';
+import { getApmSynthtraceEsClient, getApmSynthtraceKibanaClient } from './bootstrap_apm_synthtrace';
 import {
   FtrProviderContext,
   InheritedFtrProviderContext,
@@ -118,7 +118,7 @@ export function createTestConfig(
         apmFtrConfig: () => config,
         registry: RegistryProvider,
         apmSynthtraceEsClient: (context: InheritedFtrProviderContext) => {
-          return bootstrapApmSynthtrace(context, synthtraceKibanaClient);
+          return getApmSynthtraceEsClient(context, synthtraceKibanaClient);
         },
         logSynthtraceEsClient: (context: InheritedFtrProviderContext) =>
           new LogsSynthtraceEsClient({

x-pack/test/observability_ai_assistant_api_integration/common/config.ts

@@ -9,7 +9,8 @@ import { Config, FtrConfigProviderContext } from '@kbn/test';
 import supertest from 'supertest';
 import { format, UrlObject } from 'url';
 import { ObservabilityAIAssistantFtrConfigName } from '../configs';
-import { InheritedServices } from './ftr_provider_context';
+import { getApmSynthtraceEsClient } from './create_synthtrace_client';
+import { InheritedFtrProviderContext, InheritedServices } from './ftr_provider_context';
 import {
   createObservabilityAIAssistantApiClient,
   ObservabilityAIAssistantAPIClient,
@@ -21,12 +22,8 @@ export interface ObservabilityAIAssistantFtrConfig {
   kibanaConfig?: Record<string, any>;
 }
 
-async function getObservabilityAIAssistantAPIClient({ kibanaServer }: { kibanaServer: UrlObject }) {
-  const url = format({
-    ...kibanaServer,
-  });
-
-  return createObservabilityAIAssistantApiClient(supertest(url));
+async function getObservabilityAIAssistantAPIClient(kibanaServerUrl: string) {
+  return createObservabilityAIAssistantApiClient(supertest(kibanaServerUrl));
 }
 
 export type CreateTestConfig = ReturnType<typeof createTestConfig>;
@@ -59,20 +56,20 @@ export function createObservabilityAIAssistantAPIConfig({
   const services = config.get('services') as InheritedServices;
   const servers = config.get('servers');
   const kibanaServer = servers.kibana as UrlObject;
+  const kibanaServerUrl = format(kibanaServer);
+  const apmSynthtraceKibanaClient = services.apmSynthtraceKibanaClient();
 
   const createTest: Omit<CreateTest, 'testFiles'> = {
     ...config.getAll(),
     servers,
     services: {
       ...services,
+      apmSynthtraceEsClient: (context: InheritedFtrProviderContext) =>
+        getApmSynthtraceEsClient(context, apmSynthtraceKibanaClient),
       observabilityAIAssistantAPIClient: async () => {
         return {
-          readUser: await getObservabilityAIAssistantAPIClient({
-            kibanaServer,
-          }),
-          writeUser: await getObservabilityAIAssistantAPIClient({
-            kibanaServer,
-          }),
+          readUser: await getObservabilityAIAssistantAPIClient(kibanaServerUrl),
+          writeUser: await getObservabilityAIAssistantAPIClient(kibanaServerUrl),
         };
       },
     },

x-pack/test/observability_ai_assistant_api_integration/common/create_synthtrace_client.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  ApmSynthtraceEsClient,
+  ApmSynthtraceKibanaClient,
+  createLogger,
+  LogLevel,
+} from '@kbn/apm-synthtrace';
+import { InheritedFtrProviderContext } from './ftr_provider_context';
+
+export async function getApmSynthtraceEsClient(
+  context: InheritedFtrProviderContext,
+  kibanaClient: ApmSynthtraceKibanaClient
+) {
+  const es = context.getService('es');
+
+  const kibanaVersion = await kibanaClient.fetchLatestApmPackageVersion();
+  await kibanaClient.installApmPackage(kibanaVersion);
+
+  const esClient = new ApmSynthtraceEsClient({
+    client: es,
+    logger: createLogger(LogLevel.info),
+    version: kibanaVersion,
+    refreshAfterIndex: true,
+  });
+
+  return esClient;
+}

x-pack/test/observability_ai_assistant_functional/common/config.ts

@@ -10,6 +10,7 @@ import { merge } from 'lodash';
 import supertest from 'supertest';
 import { format, UrlObject } from 'url';
 import type { EBTHelpersContract } from '@kbn/analytics-ftr-helpers-plugin/common/types';
+import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
 import {
   KibanaEBTServerProvider,
   KibanaEBTUIProvider,
@@ -39,6 +40,9 @@ export interface TestConfig extends CreateTestAPI {
       >;
       kibana_ebt_server: (context: InheritedFtrProviderContext) => EBTHelpersContract;
       kibana_ebt_ui: (context: InheritedFtrProviderContext) => EBTHelpersContract;
+      apmSynthtraceEsClient: (
+        context: InheritedFtrProviderContext
+      ) => Promise<ApmSynthtraceEsClient>;
     };
 }
 

x-pack/test/observability_ai_assistant_functional/common/ui/index.ts

@@ -44,6 +44,10 @@ const pages = {
     apiKeyInput: 'secrets.apiKey-input',
     saveButton: 'create-connector-flyout-save-btn',
   },
+  contextualInsights: {
+    button: 'obsAiAssistantInsightButton',
+    text: 'obsAiAssistantInsightResponse',
+  },
 };
 
 export async function ObservabilityAIAssistantUIProvider({
@@ -55,40 +59,36 @@ export async function ObservabilityAIAssistantUIProvider({
   const security = getService('security');
   const pageObjects = getPageObjects(['common']);
 
-  const roleName = 'observability-ai-assistant-functional-test-role';
+  const roleDefinition: Role = {
+    name: 'observability-ai-assistant-functional-test-role',
+    elasticsearch: {
+      cluster: [],
+      indices: [],
+      run_as: [],
+    },
+    kibana: [
+      {
+        spaces: ['*'],
+        base: [],
+        feature: {
+          actions: ['all'],
+          [APM_SERVER_FEATURE_ID]: ['all'],
+          [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID]: ['all'],
+        },
+      },
+    ],
+  };
 
   return {
     pages,
     auth: {
       login: async () => {
         await browser.navigateTo(deployment.getHostPort());
-
-        const roleDefinition: Role = {
-          name: roleName,
-          elasticsearch: {
-            cluster: [],
-            indices: [],
-            run_as: [],
-          },
-          kibana: [
-            {
-              spaces: ['*'],
-              base: [],
-              feature: {
-                actions: ['all'],
-                [APM_SERVER_FEATURE_ID]: ['all'],
-                [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID]: ['all'],
-              },
-            },
-          ],
-        };
-
-        await security.role.create(roleName, roleDefinition);
-
-        await security.testUser.setRoles([roleName, 'apm_user']); // performs a page reload
+        await security.role.create(roleDefinition.name, roleDefinition);
+        await security.testUser.setRoles([roleDefinition.name, 'apm_user', 'viewer']); // performs a page reload
       },
       logout: async () => {
-        await security.role.delete(roleName);
+        await security.role.delete(roleDefinition.name);
         await security.testUser.restoreDefaults();
       },
     },

x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts

@@ -0,0 +1,160 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { apm, timerange } from '@kbn/apm-synthtrace-client';
+import expect from '@kbn/expect';
+import moment from 'moment';
+import OpenAI from 'openai';
+import {
+  createLlmProxy,
+  LlmProxy,
+} from '../../../observability_ai_assistant_api_integration/common/create_llm_proxy';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ApiTest({ getService, getPageObjects }: FtrProviderContext) {
+  const ui = getService('observabilityAIAssistantUI');
+  const testSubjects = getService('testSubjects');
+  const supertest = getService('supertest');
+  const retry = getService('retry');
+  const log = getService('log');
+  const browser = getService('browser');
+  const deployment = getService('deployment');
+  const apmSynthtraceEsClient = getService('apmSynthtraceEsClient');
+  const { common } = getPageObjects(['header', 'common']);
+
+  async function createSynthtraceErrors() {
+    const start = moment().subtract(5, 'minutes').valueOf();
+    const end = moment().valueOf();
+    const serviceName = 'opbeans-go';
+
+    const serviceInstance = apm
+      .service({ name: serviceName, environment: 'production', agentName: 'go' })
+      .instance('instance-a');
+
+    const interval = '1m';
+    const documents = [
+      timerange(start, end)
+        .interval(interval)
+        .rate(50)
+        .generator((timestamp) =>
+          serviceInstance
+            .transaction({ transactionName: 'GET /banana' })
+            .errors(
+              serviceInstance
+                .error({ message: 'Some exception', type: 'exception' })
+                .timestamp(timestamp)
+            )
+            .duration(10)
+            .timestamp(timestamp)
+            .failure()
+        ),
+    ];
+
+    await apmSynthtraceEsClient.index(documents);
+  }
+
+  async function createConnector(proxy: LlmProxy) {
+    await supertest
+      .post('/api/actions/connector')
+      .set('kbn-xsrf', 'foo')
+      .send({
+        name: 'foo',
+        config: {
+          apiProvider: 'OpenAI',
+          apiUrl: `http://localhost:${proxy.getPort()}`,
+          defaultModel: 'gpt-4',
+        },
+        secrets: { apiKey: 'myApiKey' },
+        connector_type_id: '.gen-ai',
+      })
+      .expect(200);
+  }
+
+  async function deleteConnectors() {
+    const connectors = await supertest.get('/api/actions/connectors').expect(200);
+    const promises = connectors.body.map((connector: { id: string }) => {
+      return supertest
+        .delete(`/api/actions/connector/${connector.id}`)
+        .set('kbn-xsrf', 'foo')
+        .expect(204);
+    });
+
+    return Promise.all(promises);
+  }
+
+  async function navigateToError() {
+    await common.navigateToApp('apm');
+    await browser.get(`${deployment.getHostPort()}/app/apm/services/opbeans-go/errors/`);
+    await testSubjects.click('errorGroupId');
+  }
+
+  describe('Contextual insights for APM errors', () => {
+    before(async () => {
+      await Promise.all([
+        deleteConnectors(), // cleanup previous connectors
+        apmSynthtraceEsClient.clean(), // cleanup previous synthtrace data
+      ]);
+
+      await Promise.all([
+        createSynthtraceErrors(), // create synthtrace
+        ui.auth.login(), // login
+      ]);
+    });
+
+    after(async () => {
+      await Promise.all([
+        deleteConnectors(), // cleanup previous connectors
+        apmSynthtraceEsClient.clean(), // cleanup synthtrace data
+        ui.auth.logout(), // logout
+      ]);
+    });
+
+    describe('when there are no connectors', () => {
+      it('should not show the contextual insight component', async () => {
+        await navigateToError();
+        await testSubjects.missingOrFail(ui.pages.contextualInsights.button);
+      });
+    });
+
+    describe('when there are connectors', () => {
+      let proxy: LlmProxy;
+
+      before(async () => {
+        proxy = await createLlmProxy(log);
+        await createConnector(proxy);
+      });
+
+      after(async () => {
+        proxy.close();
+      });
+
+      it('should show the contextual insight component on the APM error details page', async () => {
+        await navigateToError();
+
+        proxy
+          .intercept(
+            'conversation',
+            (body) => !isFunctionTitleRequest(body),
+            'This error is nothing to worry about. Have a nice day!'
+          )
+          .complete();
+
+        await testSubjects.click(ui.pages.contextualInsights.button);
+
+        await retry.try(async () => {
+          const llmResponse = await testSubjects.getVisibleText(ui.pages.contextualInsights.text);
+          expect(llmResponse).to.contain('This error is nothing to worry about. Have a nice day!');
+        });
+      });
+    });
+  });
+}
+
+function isFunctionTitleRequest(body: string) {
+  const parsedBody = JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming;
+  return parsedBody.functions?.find((fn) => fn.name === 'title_conversation') !== undefined;
+}