[Obs AI Assistant] Remove prompts from recall ranking Telemetry event (#214573)

Closes https://github.com/elastic/obs-ai-assistant-team/issues/224

## Summary

Relates to https://github.com/elastic/obs-ai-assistant-team/issues/186

Removes the prompts from the recall ranking telemetry event.

### Checklist

- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
This commit is contained in:
Viduni Wickramarachchi 2025-03-17 16:10:44 -04:00 committed by GitHub
parent 508c89fc15
commit 43079c7603
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 17 additions and 21 deletions

View file

@ -38,6 +38,7 @@ import {
import type { AuthenticatedUser } from '@kbn/security-plugin/common';
import { findLastIndex } from 'lodash';
import React, { useCallback, useEffect, useRef, useState } from 'react';
import { ChatFeedback } from '@kbn/observability-ai-assistant-plugin/public/analytics/schemas/chat_feedback';
import type { UseKnowledgeBaseResult } from '../hooks/use_knowledge_base';
import { ASSISTANT_SETUP_TITLE, EMPTY_CONVERSATION_TITLE, UPGRADE_LICENSE_TITLE } from '../i18n';
import { useAIAssistantChatService } from '../hooks/use_ai_assistant_chat_service';
@ -220,13 +221,24 @@ export function ChatBody({
if (conversation.value?.conversation && 'user' in conversation.value) {
const {
messages: _removedMessages, // Exclude messages
conversation: { title: _removedTitle, ...conversationRest }, // Exclude title
...rest
systemMessage: _removedSystemMessage, // Exclude systemMessage
conversation: { title: _removedTitle, id, last_updated: lastUpdated }, // Exclude title
user,
labels,
numeric_labels: numericLabels,
namespace,
public: isPublic,
'@timestamp': timestamp,
} = conversation.value;
const conversationWithoutMessagesAndTitle = {
...rest,
conversation: conversationRest,
const conversationWithoutMessagesAndTitle: ChatFeedback['conversation'] = {
'@timestamp': timestamp,
user,
labels,
numeric_labels: numericLabels,
namespace,
public: isPublic,
conversation: { id, last_updated: lastUpdated },
};
chatService.sendAnalyticsEvent({

View file

@ -8,33 +8,19 @@
import { RootSchema, EventTypeOpts } from '@kbn/core/server';
interface ScoredDocument {
content: string;
elserScore: number;
llmScore: number;
}
export interface RecallRanking {
prompt: string;
scoredDocuments: ScoredDocument[];
}
const schema: RootSchema<RecallRanking> = {
prompt: {
type: 'text',
_meta: {
description: 'The user prompt that was used for the ELSER text_expansion',
},
},
scoredDocuments: {
type: 'array',
items: {
properties: {
content: {
type: 'text',
_meta: {
description: 'The raw content of the recalled document',
},
},
elserScore: {
type: 'float',
_meta: {

View file

@ -68,11 +68,9 @@ export async function recallAndScore({
});
analytics.reportEvent<RecallRanking>(recallRankingEventType, {
prompt: queries.map((query) => query.text).join('\n\n'),
scoredDocuments: suggestions.map((suggestion) => {
const llmScore = scores.find((score) => score.id === suggestion.id);
return {
content: suggestion.text,
elserScore: suggestion.score ?? -1,
llmScore: llmScore ? llmScore.score : -1,
};