mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 01:13:23 -04:00
[Obs AI Assistant] Remove prompts from recall ranking Telemetry event (#214573)
Closes https://github.com/elastic/obs-ai-assistant-team/issues/224 ## Summary Relates to https://github.com/elastic/obs-ai-assistant-team/issues/186 Removes the prompts from the recall ranking telemetry event. ### Checklist - [x] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
This commit is contained in:
parent
508c89fc15
commit
43079c7603
3 changed files with 17 additions and 21 deletions
|
@ -38,6 +38,7 @@ import {
|
|||
import type { AuthenticatedUser } from '@kbn/security-plugin/common';
|
||||
import { findLastIndex } from 'lodash';
|
||||
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { ChatFeedback } from '@kbn/observability-ai-assistant-plugin/public/analytics/schemas/chat_feedback';
|
||||
import type { UseKnowledgeBaseResult } from '../hooks/use_knowledge_base';
|
||||
import { ASSISTANT_SETUP_TITLE, EMPTY_CONVERSATION_TITLE, UPGRADE_LICENSE_TITLE } from '../i18n';
|
||||
import { useAIAssistantChatService } from '../hooks/use_ai_assistant_chat_service';
|
||||
|
@ -220,13 +221,24 @@ export function ChatBody({
|
|||
if (conversation.value?.conversation && 'user' in conversation.value) {
|
||||
const {
|
||||
messages: _removedMessages, // Exclude messages
|
||||
conversation: { title: _removedTitle, ...conversationRest }, // Exclude title
|
||||
...rest
|
||||
systemMessage: _removedSystemMessage, // Exclude systemMessage
|
||||
conversation: { title: _removedTitle, id, last_updated: lastUpdated }, // Exclude title
|
||||
user,
|
||||
labels,
|
||||
numeric_labels: numericLabels,
|
||||
namespace,
|
||||
public: isPublic,
|
||||
'@timestamp': timestamp,
|
||||
} = conversation.value;
|
||||
|
||||
const conversationWithoutMessagesAndTitle = {
|
||||
...rest,
|
||||
conversation: conversationRest,
|
||||
const conversationWithoutMessagesAndTitle: ChatFeedback['conversation'] = {
|
||||
'@timestamp': timestamp,
|
||||
user,
|
||||
labels,
|
||||
numeric_labels: numericLabels,
|
||||
namespace,
|
||||
public: isPublic,
|
||||
conversation: { id, last_updated: lastUpdated },
|
||||
};
|
||||
|
||||
chatService.sendAnalyticsEvent({
|
||||
|
|
|
@ -8,33 +8,19 @@
|
|||
import { RootSchema, EventTypeOpts } from '@kbn/core/server';
|
||||
|
||||
interface ScoredDocument {
|
||||
content: string;
|
||||
elserScore: number;
|
||||
llmScore: number;
|
||||
}
|
||||
|
||||
export interface RecallRanking {
|
||||
prompt: string;
|
||||
scoredDocuments: ScoredDocument[];
|
||||
}
|
||||
|
||||
const schema: RootSchema<RecallRanking> = {
|
||||
prompt: {
|
||||
type: 'text',
|
||||
_meta: {
|
||||
description: 'The user prompt that was used for the ELSER text_expansion',
|
||||
},
|
||||
},
|
||||
scoredDocuments: {
|
||||
type: 'array',
|
||||
items: {
|
||||
properties: {
|
||||
content: {
|
||||
type: 'text',
|
||||
_meta: {
|
||||
description: 'The raw content of the recalled document',
|
||||
},
|
||||
},
|
||||
elserScore: {
|
||||
type: 'float',
|
||||
_meta: {
|
||||
|
|
|
@ -68,11 +68,9 @@ export async function recallAndScore({
|
|||
});
|
||||
|
||||
analytics.reportEvent<RecallRanking>(recallRankingEventType, {
|
||||
prompt: queries.map((query) => query.text).join('\n\n'),
|
||||
scoredDocuments: suggestions.map((suggestion) => {
|
||||
const llmScore = scores.find((score) => score.id === suggestion.id);
|
||||
return {
|
||||
content: suggestion.text,
|
||||
elserScore: suggestion.score ?? -1,
|
||||
llmScore: llmScore ? llmScore.score : -1,
|
||||
};
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue