Set up feedback telemetry gathering (#172485)

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/observability_ai_assistant/public/analytics/schema.ts

@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { RootSchema } from '@kbn/analytics-client';
+import { Message } from '../../common';
+import type { Feedback } from '../components/feedback_buttons';
+
+export const MESSAGE_FEEDBACK = 'observability_ai_assistant_chat_message_feedback' as const;
+export const INSIGHT_FEEDBACK = 'observability_ai_assistant_chat_insight_feedback' as const;
+
+export interface MessageFeedback extends Message {
+  feedback: Feedback;
+}
+
+export interface TelemetryEvent {
+  eventType: typeof MESSAGE_FEEDBACK | typeof INSIGHT_FEEDBACK;
+  schema: RootSchema<MessageFeedback>;
+}
+
+export const MESSAGE_FEEDBACK_SCHEMA: TelemetryEvent = {
+  eventType: MESSAGE_FEEDBACK,
+  schema: {
+    '@timestamp': {
+      type: 'text',
+      _meta: {
+        description: 'The timestamp of the message.',
+      },
+    },
+    feedback: {
+      type: 'text',
+      _meta: {
+        description: 'Whether the user has deemed this response useful or not',
+      },
+    },
+    message: {
+      properties: {
+        content: {
+          type: 'text',
+          _meta: {
+            description: 'The response generated by the LLM.',
+            optional: true,
+          },
+        },
+        name: {
+          type: 'text',
+          _meta: {
+            description: 'The name of the function that was executed.',
+            optional: true,
+          },
+        },
+        role: {
+          type: 'text',
+          _meta: {
+            description: 'The actor that generated the response.',
+          },
+        },
+        data: {
+          type: 'text',
+          _meta: {
+            description: '',
+            optional: true,
+          },
+        },
+        function_call: {
+          properties: {
+            name: {
+              type: 'text',
+              _meta: {
+                description: 'The name of the function that was executed.',
+                optional: false,
+              },
+            },
+            arguments: {
+              type: 'text',
+              _meta: {
+                description: 'The arguments that were used when executing the function.',
+                optional: true,
+              },
+            },
+            trigger: {
+              type: 'text',
+              _meta: {
+                description: 'The actor which triggered the execution of this function.',
+              },
+            },
+          },
+        },
+      },
+    },
+  },
+};

x-pack/plugins/observability_ai_assistant/public/application.tsx

@@ -4,14 +4,14 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import React, { useMemo } from 'react';
+import type { History } from 'history';
+import type { Observable } from 'rxjs';
 import { EuiErrorBoundary } from '@elastic/eui';
 import type { CoreStart, CoreTheme } from '@kbn/core/public';
 import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
 import { RedirectAppLinks } from '@kbn/shared-ux-link-redirect-app';
 import { RouteRenderer, RouterProvider } from '@kbn/typed-react-router-config';
-import type { History } from 'history';
-import React, { useMemo } from 'react';
-import type { Observable } from 'rxjs';
 import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme';
 import { ObservabilityAIAssistantProvider } from './context/observability_ai_assistant_provider';
 import { observabilityAIAssistantRouter } from './routes/config';
@@ -21,17 +21,17 @@ import type {
 } from './types';
 
 export function Application({
-  theme$,
-  history,
   coreStart,
+  history,
   pluginsStart,
   service,
+  theme$,
 }: {
-  theme$: Observable<CoreTheme>;
-  history: History;
   coreStart: CoreStart;
+  history: History;
   pluginsStart: ObservabilityAIAssistantPluginStartDependencies;
   service: ObservabilityAIAssistantService;
+  theme$: Observable<CoreTheme>;
 }) {
   const theme = useMemo(() => {
     return { theme$ };

x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx

@@ -35,6 +35,8 @@ import { IncorrectLicensePanel } from './incorrect_license_panel';
 import { InitialSetupPanel } from './initial_setup_panel';
 import { ChatActionClickType } from './types';
 import { EMPTY_CONVERSATION_TITLE } from '../../i18n';
+import { Feedback } from '../feedback_buttons';
+import { MESSAGE_FEEDBACK } from '../../analytics/schema';
 
 const timelineClassName = css`
   overflow-y: auto;
@@ -113,6 +115,11 @@ export function ChatBody({
   const isAtBottom = (parent: HTMLElement) =>
     parent.scrollTop + parent.clientHeight >= parent.scrollHeight;
 
+  const handleFeedback = (message: Message, feedback: Feedback) => {
+    const feedbackEvent = { ...message, feedback };
+    chatService.analytics.reportEvent(MESSAGE_FEEDBACK, feedbackEvent);
+  };
+
   useEffect(() => {
     const parent = timelineContainerRef.current?.parentElement;
     if (!parent) {
@@ -208,7 +215,7 @@ export function ChatBody({
                   const indexOf = messages.indexOf(editedMessage);
                   next(messages.slice(0, indexOf).concat(newMessage));
                 }}
-                onFeedback={(message, feedback) => {}}
+                onFeedback={handleFeedback}
                 onRegenerate={(message) => {
                   const indexOf = messages.indexOf(message);
                   next(messages.slice(0, indexOf));

x-pack/plugins/observability_ai_assistant/public/components/insight/insight.tsx

@@ -5,9 +5,10 @@
  * 2.0.
  */
 import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
-import { last } from 'lodash';
+import { last, noop } from 'lodash';
 import React, { useEffect, useRef, useState } from 'react';
 import { MessageRole, type Message } from '../../../common/types';
+import { INSIGHT_FEEDBACK } from '../../analytics/schema';
 import { ObservabilityAIAssistantChatServiceProvider } from '../../context/observability_ai_assistant_chat_service_provider';
 import { useAbortableAsync } from '../../hooks/use_abortable_async';
 import { ChatState, useChat } from '../../hooks/use_chat';
@@ -21,6 +22,7 @@ import { StartChatButton } from '../buttons/start_chat_button';
 import { StopGeneratingButton } from '../buttons/stop_generating_button';
 import { ChatFlyout } from '../chat/chat_flyout';
 import { ConnectorSelectorBase } from '../connector_selector/connector_selector_base';
+import { FeedbackButtons } from '../feedback_buttons';
 import { MessagePanel } from '../message_panel/message_panel';
 import { MessageText } from '../message_panel/message_text';
 import { MissingCredentialsCallout } from '../missing_credentials_callout';
@@ -75,6 +77,16 @@ function ChatContent({
             />
           ) : (
             <EuiFlexGroup direction="row">
+              <FeedbackButtons
+                onClickFeedback={(feedback) =>
+                  lastAssistantResponse
+                    ? chatService.analytics.reportEvent(INSIGHT_FEEDBACK, {
+                        feedback,
+                        ...lastAssistantResponse,
+                      })
+                    : noop
+                }
+              />
               <EuiFlexItem grow={false}>
                 <RegenerateResponseButton
                   onClick={() => {

x-pack/plugins/observability_ai_assistant/public/hooks/use_chat.test.ts

@@ -6,7 +6,7 @@
  */
 import type { DeeplyMockedKeys } from '@kbn/utility-types-jest';
 import { type RenderHookResult, renderHook, act } from '@testing-library/react-hooks';
-import { Subject } from 'rxjs';
+import { Observable, Subject } from 'rxjs';
 import { MessageRole } from '../../common';
 import type { ObservabilityAIAssistantChatService, PendingMessage } from '../types';
 import { type UseChatResult, useChat, type UseChatProps, ChatState } from './use_chat';
@@ -15,6 +15,11 @@ import * as useKibanaModule from './use_kibana';
 type MockedChatService = DeeplyMockedKeys<ObservabilityAIAssistantChatService>;
 
 const mockChatService: MockedChatService = {
+  analytics: {
+    optIn: jest.fn(),
+    reportEvent: jest.fn(),
+    telemetryCounter$: new Observable() as any,
+  },
   chat: jest.fn(),
   executeFunction: jest.fn(),
   getContexts: jest.fn().mockReturnValue([{ name: 'core', description: '' }]),

x-pack/plugins/observability_ai_assistant/public/plugin.tsx

@@ -4,6 +4,8 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import React from 'react';
+import ReactDOM from 'react-dom';
 import {
   AppNavLinkStatus,
   DEFAULT_APP_CATEGORIES,
@@ -15,8 +17,6 @@ import {
 } from '@kbn/core/public';
 import { i18n } from '@kbn/i18n';
 import type { Logger } from '@kbn/logging';
-import React from 'react';
-import ReactDOM from 'react-dom';
 import { createService } from './service/create_service';
 import type {
   ConfigSchema,
@@ -26,6 +26,7 @@ import type {
   ObservabilityAIAssistantPluginStartDependencies,
   ObservabilityAIAssistantService,
 } from './types';
+import { MessageFeedback, MESSAGE_FEEDBACK_SCHEMA } from './analytics/schema';
 
 export class ObservabilityAIAssistantPlugin
   implements
@@ -64,7 +65,6 @@ export class ObservabilityAIAssistantPlugin
           path: '/conversations/new',
         },
       ],
-
       mount: async (appMountParameters: AppMountParameters<unknown>) => {
         // Load application bundle and Get start services
         const [{ Application }, [coreStart, pluginsStart]] = await Promise.all([
@@ -87,6 +87,9 @@ export class ObservabilityAIAssistantPlugin
         };
       },
     });
+
+    coreSetup.analytics.registerEventType<MessageFeedback>(MESSAGE_FEEDBACK_SCHEMA);
+
     return {};
   }
 
@@ -95,11 +98,12 @@ export class ObservabilityAIAssistantPlugin
     pluginsStart: ObservabilityAIAssistantPluginStartDependencies
   ): ObservabilityAIAssistantPluginStart {
     const service = (this.service = createService({
+      analytics: coreStart.analytics,
       coreStart,
-      securityStart: pluginsStart.security,
-      licenseStart: pluginsStart.licensing,
-      shareStart: pluginsStart.share,
       enabled: coreStart.application.capabilities.observabilityAIAssistant.show === true,
+      licenseStart: pluginsStart.licensing,
+      securityStart: pluginsStart.security,
+      shareStart: pluginsStart.share,
     }));
 
     service.register(async ({ signal, registerContext, registerFunction }) => {

x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 import type { HttpFetchOptions } from '@kbn/core/public';
-import { lastValueFrom } from 'rxjs';
+import { lastValueFrom, Observable } from 'rxjs';
 import { ReadableStream } from 'stream/web';
 import type { ObservabilityAIAssistantChatService } from '../types';
 import { createChatService } from './create_chat_service';
@@ -40,6 +40,11 @@ describe('createChatService', () => {
 
     beforeEach(async () => {
       service = await createChatService({
+        analytics: {
+          optIn: () => {},
+          reportEvent: () => {},
+          telemetryCounter$: new Observable(),
+        },
         client: clientSpy,
         registrations: [],
         signal: new AbortController().signal,

x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts

@@ -7,7 +7,7 @@
 /* eslint-disable max-classes-per-file*/
 import { Validator, type Schema, type OutputUnit } from '@cfworker/json-schema';
 
-import { HttpResponse } from '@kbn/core/public';
+import { AnalyticsServiceStart, HttpResponse } from '@kbn/core/public';
 import { AbortError } from '@kbn/kibana-utils-plugin/common';
 import { IncomingMessage } from 'http';
 import { cloneDeep, pick } from 'lodash';
@@ -58,10 +58,12 @@ export class FunctionArgsValidationError extends Error {
 }
 
 export async function createChatService({
+  analytics,
   signal: setupAbortSignal,
   registrations,
   client,
 }: {
+  analytics: AnalyticsServiceStart;
   signal: AbortSignal;
   registrations: ChatRegistrationFunction[];
   client: ObservabilityAIAssistantAPIClient;
@@ -114,6 +116,7 @@ export async function createChatService({
   }
 
   return {
+    analytics,
     executeFunction: async ({ name, args, signal, messages, connectorId }) => {
       const fn = functionRegistry.get(name);
 

x-pack/plugins/observability_ai_assistant/public/service/create_mock_chat_service.ts

@@ -5,13 +5,20 @@
  * 2.0.
  */
 
+import { TelemetryCounter } from '@kbn/analytics-client';
 import type { DeeplyMockedKeys } from '@kbn/utility-types-jest';
+import { Observable } from 'rxjs';
 import type { ObservabilityAIAssistantChatService } from '../types';
 
 type MockedChatService = DeeplyMockedKeys<ObservabilityAIAssistantChatService>;
 
 export const createMockChatService = (): MockedChatService => {
   const mockChatService: MockedChatService = {
+    analytics: {
+      optIn: jest.fn(),
+      reportEvent: jest.fn(),
+      telemetryCounter$: new Observable<TelemetryCounter>() as any,
+    },
     chat: jest.fn(),
     executeFunction: jest.fn(),
     getContexts: jest.fn().mockReturnValue([{ name: 'core', description: '' }]),

x-pack/plugins/observability_ai_assistant/public/service/create_service.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { CoreStart } from '@kbn/core/public';
+import type { AnalyticsServiceStart, CoreStart } from '@kbn/core/public';
 import type { LicensingPluginStart } from '@kbn/licensing-plugin/public';
 import type { SecurityPluginStart } from '@kbn/security-plugin/public';
 import type { SharePluginStart } from '@kbn/share-plugin/public';
@@ -13,12 +13,14 @@ import { createCallObservabilityAIAssistantAPI } from '../api';
 import type { ChatRegistrationFunction, ObservabilityAIAssistantService } from '../types';
 
 export function createService({
+  analytics,
   coreStart,
   enabled,
   licenseStart,
   securityStart,
   shareStart,
 }: {
+  analytics: AnalyticsServiceStart;
   coreStart: CoreStart;
   enabled: boolean;
   licenseStart: LicensingPluginStart;
@@ -38,7 +40,7 @@ export function createService({
     },
     start: async ({ signal }) => {
       const mod = await import('./create_chat_service');
-      return await mod.createChatService({ client, signal, registrations });
+      return await mod.createChatService({ analytics, client, signal, registrations });
     },
 
     callApi: client,

x-pack/plugins/observability_ai_assistant/public/types.ts

@@ -4,6 +4,8 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
+import type { AnalyticsServiceStart } from '@kbn/core/public';
 import type { FeaturesPluginStart, FeaturesPluginSetup } from '@kbn/features-plugin/public';
 import type {
   ObservabilitySharedPluginSetup,
@@ -52,6 +54,7 @@ export type CreateChatCompletionResponseChunk = Omit<CreateChatCompletionRespons
 };
 
 export interface ObservabilityAIAssistantChatService {
+  analytics: AnalyticsServiceStart;
   chat: (options: {
     messages: Message[];
     connectorId: string;

x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.test.tsx

@@ -137,7 +137,7 @@ describe('getTimelineItemsFromConversation', () => {
         actions: {
           canCopy: true,
           canEdit: true,
-          canGiveFeedback: false,
+          canGiveFeedback: true,
           canRegenerate: true,
         },
         display: {
@@ -499,7 +499,7 @@ describe('getTimelineItemsFromConversation', () => {
           canCopy: true,
           canRegenerate: true,
           canEdit: false,
-          canGiveFeedback: false,
+          canGiveFeedback: true,
         },
         display: {
           collapsed: false,

x-pack/plugins/observability_ai_assistant/public/utils/get_timeline_items_from_conversation.tsx

@@ -219,7 +219,7 @@ export function getTimelineItemsfromConversation({
         case MessageRole.Assistant:
           actions.canRegenerate = hasConnector;
           actions.canCopy = true;
-          actions.canGiveFeedback = false;
+          actions.canGiveFeedback = true;
           display.hide = false;
 
           // is a function suggestion by the assistant

x-pack/plugins/observability_ai_assistant/public/utils/storybook_decorator.tsx

@@ -22,6 +22,11 @@ import { buildFunctionElasticsearch, buildFunctionServiceSummary } from './build
 import { ObservabilityAIAssistantChatServiceProvider } from '../context/observability_ai_assistant_chat_service_provider';
 
 const chatService: ObservabilityAIAssistantChatService = {
+  analytics: {
+    optIn: () => {},
+    reportEvent: () => {},
+    telemetryCounter$: new Observable(),
+  },
   chat: (options: { messages: Message[]; connectorId: string }) => new Observable<PendingMessage>(),
   getContexts: () => [],
   getFunctions: () => [buildFunctionElasticsearch(), buildFunctionServiceSummary()],
@@ -32,6 +37,7 @@ const chatService: ObservabilityAIAssistantChatService = {
     signal: AbortSignal;
   }): Promise<{ content?: Serializable; data?: Serializable }> => ({}),
   renderFunction: (name: string, args: string | undefined, response: {}) => (
+    // eslint-disable-next-line @kbn/i18n/strings_should_be_translated_with_i18n
     <div>Hello! {name}</div>
   ),
   hasFunction: () => true,

x-pack/plugins/observability_ai_assistant/tsconfig.json

@@ -47,7 +47,8 @@
     "@kbn/rule-registry-plugin",
     "@kbn/licensing-plugin",
     "@kbn/share-plugin",
-    "@kbn/utility-types-jest"
+    "@kbn/utility-types-jest",
+    "@kbn/analytics-client"
   ],
   "exclude": ["target/**/*"]
 }