github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Obs AI Assistant] Avoid sending user prompts and LLM responses to Telemetry (#208118)

Browse source 
Closes https://github.com/elastic/obs-ai-assistant-team/issues/186

### Problem
Prompts entered by users and responses from LLMs must not be sent to
telemetry

### Solution
Remove `messages` from the payloads sent to Telemetry

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
This commit is contained in:
Viduni Wickramarachchi 2025-01-28 14:00:18 -05:00 • committed by GitHub
parent 5b6261782e
commit f10d8f7c56
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 51 additions and 94 deletions
Download patch file Download diff file Expand all files Collapse all files

17
x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_body.tsx
View file

@ -183,13 +183,24 @@ export function ChatBody({
const [promptEditorHeight, setPromptEditorHeight] = useState<number>(0);
const handleFeedback = (message: Message, feedback: Feedback) => {
const handleFeedback = (feedback: Feedback) => {
if (conversation.value?.conversation && 'user' in conversation.value) {
const {
messages: _removedMessages, // Exclude messages
conversation: { title: _removedTitle, ...conversationRest }, // Exclude title
...rest
} = conversation.value;
const conversationWithoutMessagesAndTitle = {
...rest,
conversation: conversationRest,
};
chatService.sendAnalyticsEvent({
type: ObservabilityAIAssistantTelemetryEventType.ChatFeedback,
payload: {
messageWithFeedback: { message, feedback },
conversation: conversation.value,
feedback,
conversation: conversationWithoutMessagesAndTitle,
},
});
}

4
x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_consolidated_items.tsx
View file

@ -128,9 +128,7 @@ export function ChatConsolidatedItems({
});
}}
onEditSubmit={(message) => onEditSubmit(item.message, message)}
onFeedbackClick={(feedback) => {
onFeedback(item.message, feedback);
}}
onFeedbackClick={(feedback) => onFeedback(feedback)}
onRegenerateClick={() => {
onRegenerate(item.message);
}}

6
x-pack/platform/packages/shared/kbn-ai-assistant/src/chat/chat_timeline.tsx
View file

@ -53,7 +53,7 @@ export interface ChatTimelineProps {
chatState: ChatState;
currentUser?: Pick<AuthenticatedUser, 'full_name' | 'username'>;
onEdit: (message: Message, messageAfterEdit: Message) => void;
onFeedback: (message: Message, feedback: Feedback) => void;
onFeedback: (feedback: Feedback) => void;
onRegenerate: (message: Message) => void;
onSendTelemetry: (eventWithPayload: TelemetryEventTypeWithPayload) => void;
onStopGenerating: () => void;
@ -137,9 +137,7 @@ export function ChatTimeline({
onActionClick={(payload) => {
onActionClick({ message: item.message, payload });
}}
onFeedbackClick={(feedback) => {
onFeedback(item.message, feedback);
}}
onFeedbackClick={(feedback) => onFeedback(feedback)}
onRegenerateClick={() => {
onRegenerate(item.message);
}}

2
x-pack/platform/packages/shared/kbn-ai-assistant/src/prompt_editor/prompt_editor.tsx
View file

@ -125,7 +125,7 @@ export function PromptEditor({
setMode('prompt');
onSendTelemetry({
type: ObservabilityAIAssistantTelemetryEventType.UserSentPromptInChat,
payload: { ...message, scopes },
payload: { scopes },
});
} catch (_) {
setInnerMessage(oldMessage);

6
x-pack/platform/plugins/shared/observability_ai_assistant/public/analytics/index.ts
View file

@ -6,12 +6,10 @@
*/
import type { AnalyticsServiceSetup, AnalyticsServiceStart } from '@kbn/core-analytics-browser';
import { AssistantScope } from '@kbn/ai-assistant-common';
import type { Message } from '../../common';
import { chatFeedbackEventSchema, ChatFeedback } from './schemas/chat_feedback';
import { insightFeedbackEventSchema, InsightFeedback } from './schemas/insight_feedback';
import { insightResponseEventSchema, InsightResponse } from './schemas/insight_response';
import { userSentPromptEventSchema } from './schemas/user_sent_prompt';
import { userSentPromptEventSchema, UserSentPrompt } from './schemas/user_sent_prompt';
import { ObservabilityAIAssistantTelemetryEventType } from './telemetry_event_type';
const schemas = [
@ -26,7 +24,7 @@ export type TelemetryEventTypeWithPayload =
| { type: ObservabilityAIAssistantTelemetryEventType.InsightFeedback; payload: InsightFeedback }
| {
type: ObservabilityAIAssistantTelemetryEventType.UserSentPromptInChat;
payload: Message & { scopes: AssistantScope[] };
payload: UserSentPrompt;
}
| {
type: ObservabilityAIAssistantTelemetryEventType.InsightResponse;

40
x-pack/platform/plugins/shared/observability_ai_assistant/public/analytics/schemas/chat_feedback.ts
View file

@ -6,33 +6,24 @@
*/
import type { EventTypeOpts } from '@kbn/core/public';
import type { Message, Conversation } from '../../../common';
import type { Conversation } from '../../../common';
import type { Feedback } from '../../components/buttons/feedback_buttons';
import { ObservabilityAIAssistantTelemetryEventType } from '../telemetry_event_type';
import { messageSchema } from './common';
export interface ChatFeedback {
messageWithFeedback: {
message: Message;
feedback: Feedback;
feedback: Feedback;
conversation: Omit<Omit<Conversation, 'messages'>, 'conversation'> & {
conversation: Omit<Conversation['conversation'], 'title'>;
};
conversation: Conversation;
}
export const chatFeedbackEventSchema: EventTypeOpts<ChatFeedback> = {
eventType: ObservabilityAIAssistantTelemetryEventType.ChatFeedback,
schema: {
messageWithFeedback: {
properties: {
message: {
properties: messageSchema,
},
feedback: {
type: 'text',
_meta: {
description: 'Whether the user has deemed this response useful or not',
},
},
feedback: {
type: 'text',
_meta: {
description: 'Whether the user has deemed this response useful or not',
},
},
conversation: {
@ -68,12 +59,6 @@ export const chatFeedbackEventSchema: EventTypeOpts<ChatFeedback> = {
description: 'The id of the conversation.',
},
},
title: {
type: 'text',
_meta: {
description: 'The title of the conversation.',
},
},
last_updated: {
type: 'text',
_meta: {
@ -104,15 +89,6 @@ export const chatFeedbackEventSchema: EventTypeOpts<ChatFeedback> = {
},
},
},
messages: {
type: 'array',
items: {
properties: messageSchema,
},
_meta: {
description: 'The messages in the conversation.',
},
},
labels: {
type: 'pass_through',
_meta: {

6
x-pack/platform/plugins/shared/observability_ai_assistant/public/analytics/schemas/insight_feedback.ts
View file

@ -6,14 +6,11 @@
*/
import type { EventTypeOpts } from '@kbn/core/public';
import type { Message } from '../../../common';
import type { Feedback } from '../../components/buttons/feedback_buttons';
import { ObservabilityAIAssistantTelemetryEventType } from '../telemetry_event_type';
import { messageSchema } from './common';
export interface InsightFeedback {
feedback: Feedback;
message: Message;
}
export const insightFeedbackEventSchema: EventTypeOpts<InsightFeedback> = {
@ -25,8 +22,5 @@ export const insightFeedbackEventSchema: EventTypeOpts<InsightFeedback> = {
description: 'Whether the user has deemed this response useful or not',
},
},
message: {
properties: messageSchema,
},
},
};

19
x-pack/platform/plugins/shared/observability_ai_assistant/public/analytics/schemas/user_sent_prompt.ts
View file

@ -6,11 +6,22 @@
*/
import type { EventTypeOpts } from '@kbn/core/public';
import type { Message } from '../../../common';
import { AssistantScope } from '@kbn/ai-assistant-common';
import { ObservabilityAIAssistantTelemetryEventType } from '../telemetry_event_type';
import { messageSchema } from './common';
export const userSentPromptEventSchema: EventTypeOpts<Message> = {
export interface UserSentPrompt {
scopes: AssistantScope[];
}
export const userSentPromptEventSchema: EventTypeOpts<UserSentPrompt> = {
eventType: ObservabilityAIAssistantTelemetryEventType.UserSentPromptInChat,
schema: messageSchema,
schema: {
scopes: {
type: 'array',
items: {
type: 'text',
_meta: { description: 'Scope of the AI Assistant' },
},
},
},
};

1
x-pack/platform/plugins/shared/observability_ai_assistant/public/components/insight/insight.tsx
View file

@ -120,7 +120,6 @@ function ChatContent({
type: ObservabilityAIAssistantTelemetryEventType.InsightFeedback,
payload: {
feedback,
message: lastAssistantResponse,
},
});
}

44
x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts
View file

@ -417,30 +417,16 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
expect(events.length).to.eql(1);
const { messageWithFeedback, conversation } = events[0]
const { feedback, conversation } = events[0]
.properties as unknown as ChatFeedback;
expect(messageWithFeedback.feedback).to.eql('positive');
expect(messageWithFeedback.message.message).to.eql({
content: 'My response',
function_call: {
arguments: '',
name: '',
trigger: 'assistant',
},
role: 'assistant',
});
expect(conversation.conversation.title).to.eql('My title');
expect(feedback).to.eql('positive');
expect(conversation.namespace).to.eql('default');
expect(conversation.public).to.eql(false);
expect(conversation.user?.name).to.eql('editor');
const { messages } = conversation;
expect(messages.length).to.eql(9);
expect(messages[0].message.role).to.eql('system');
expect(conversation.conversation).to.not.have.property('title');
expect(conversation).to.not.have.property('messages');
});
});
});
@ -492,30 +478,16 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
expect(events.length).to.eql(2);
const { messageWithFeedback, conversation } = events[1]
const { feedback, conversation } = events[1]
.properties as unknown as ChatFeedback;
expect(messageWithFeedback.feedback).to.eql('positive');
expect(messageWithFeedback.message.message).to.eql({
content:
'Service Level Indicators (SLIs) are quantifiable defined metrics that measure the performance and availability of a service or distributed system.',
function_call: {
arguments: '',
name: '',
trigger: 'assistant',
},
role: 'assistant',
});
expect(conversation.conversation.title).to.eql('My old conversation');
expect(feedback).to.eql('positive');
expect(conversation.namespace).to.eql('default');
expect(conversation.public).to.eql(false);
expect(conversation.user?.name).to.eql('editor');
const { messages } = conversation;
// Verify that data changed after user interaction before being sent to telemetry
expect(messages.length).to.eql(9);
expect(conversation.conversation).to.not.have.property('title');
expect(conversation).to.not.have.property('messages');
});
});
});