mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[8.14] [Obs AI Assistant] Add tests for if chat telemetry is sent and correctly shaped (#181086) (#182341)
# Backport This will backport the following commits from `main` to `8.14`: - [[Obs AI Assistant] Add tests for if chat telemetry is sent and correctly shaped (#181086)](https://github.com/elastic/kibana/pull/181086) <!--- Backport version: 9.4.3 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Milton Hultgren","email":"milton.hultgren@elastic.co"},"sourceCommit":{"committedDate":"2024-05-02T10:00:56Z","message":"[Obs AI Assistant] Add tests for if chat telemetry is sent and correctly shaped (#181086)\n\nCo-authored-by: Alejandro Fernández Haro <alejandro.haro@elastic.co>\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"a697da262f8820839d96702b5e6cf6ca02757e0e","branchLabelMapping":{"^v8.15.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:Obs AI Assistant","ci:project-deploy-observability","Team:obs-knowledge","v8.14.0","v8.15.0"],"title":"[Obs AI Assistant] Add tests for if chat telemetry is sent and correctly shaped","number":181086,"url":"https://github.com/elastic/kibana/pull/181086","mergeCommit":{"message":"[Obs AI Assistant] Add tests for if chat telemetry is sent and correctly shaped (#181086)\n\nCo-authored-by: Alejandro Fernández Haro <alejandro.haro@elastic.co>\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"a697da262f8820839d96702b5e6cf6ca02757e0e"}},"sourceBranch":"main","suggestedTargetBranches":["8.14"],"targetPullRequestStates":[{"branch":"8.14","label":"v8.14.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.15.0","branchLabelMappingKey":"^v8.15.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/181086","number":181086,"mergeCommit":{"message":"[Obs AI Assistant] Add tests for if chat telemetry is sent and correctly shaped (#181086)\n\nCo-authored-by: Alejandro Fernández Haro <alejandro.haro@elastic.co>\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"a697da262f8820839d96702b5e6cf6ca02757e0e"}}]}] BACKPORT--> Co-authored-by: Milton Hultgren <milton.hultgren@elastic.co>
This commit is contained in:
Kibana Machine
GitHub
parent
059a72b91c
commit
2f78ecc523
8 changed files with 237 additions and 5 deletions
|
|
@ -18,6 +18,21 @@ const events = await getService('kibana_ebt_ui').getEvents(2, { eventTypes: ['my
|
|||
expect(events).to...
|
||||
```
|
||||
|
||||
Refer to [`EBTHelpersContract`](./plugins/analytics_ftr_helpers/common/types.ts#:~:text=EBTHelpersContract) for more details about the existing APIs and all the options they accept.
|
||||
|
||||
## Usage outside of this test suite
|
||||
|
||||
First, you'll need to add the services to your config, such as:
|
||||
```
|
||||
kibana_ebt_server: KibanaEBTServerProvider,
|
||||
kibana_ebt_ui: KibanaEBTUIProvider,
|
||||
```
|
||||
|
||||
The providers are found inside `./services/kibana_ebt.ts`
|
||||
|
||||
You'll also need to inject a plugin to Kibana that loads the helper code, by adding a CLI flag like:
|
||||
`--plugin-path=PATH_TO/test/analytics/plugins/analytics_ftr_helpers`
|
||||
|
||||
If you are reusing these helpers in another suite, please remember to make sure to optIn via `await getService('kibana_ebt_ui').setOptIn(true);`
|
||||
|
||||
Refer to [`EBTHelpersContract`](./plugins/analytics_ftr_helpers/common/types.ts#:~:text=EBTHelpersContract) for more details about the existing APIs and all the options they accept.
|
||||
> Note: This opt in is only stored in memory, so a page refresh will reset it.
|
||||
|
|
|
|||
|
|
@ -67,7 +67,10 @@ export function KibanaEBTUIProvider({
|
|||
|
||||
return {
|
||||
setOptIn: async (optIn) => {
|
||||
await common.navigateToApp('home');
|
||||
if (!(await common.isChromeVisible())) {
|
||||
// The browser has not loaded Kibana yet. Let's open `home`.
|
||||
await common.navigateToApp('home');
|
||||
}
|
||||
await setOptIn(optIn);
|
||||
},
|
||||
getEvents: async (numberOfEvents, options) => {
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ export function useConversation({
|
|||
service,
|
||||
connectorId,
|
||||
onConversationUpdate: (event) => {
|
||||
conversation.refresh();
|
||||
setDisplayedConversationId(event.conversation.id);
|
||||
onConversationUpdate?.({ conversation: event.conversation });
|
||||
},
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
*/
|
||||
|
||||
import { mapValues } from 'lodash';
|
||||
import path from 'path';
|
||||
import { createTestConfig, CreateTestConfig } from '../common/config';
|
||||
|
||||
export const observabilityAIAssistantDebugLogger = {
|
||||
|
|
@ -25,6 +26,10 @@ export const observabilityAIAssistantFtrConfigs = {
|
|||
license: 'trial' as const,
|
||||
kibanaConfig: {
|
||||
'logging.loggers': [observabilityAIAssistantDebugLogger],
|
||||
'plugin-path': path.resolve(
|
||||
__dirname,
|
||||
'../../../../test/analytics/plugins/analytics_ftr_helpers'
|
||||
),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -9,6 +9,11 @@ import { FtrConfigProviderContext } from '@kbn/test';
|
|||
import { merge } from 'lodash';
|
||||
import supertest from 'supertest';
|
||||
import { format, UrlObject } from 'url';
|
||||
import type { EBTHelpersContract } from '@kbn/analytics-ftr-helpers-plugin/common/types';
|
||||
import {
|
||||
KibanaEBTServerProvider,
|
||||
KibanaEBTUIProvider,
|
||||
} from '../../../../test/analytics/services/kibana_ebt';
|
||||
import {
|
||||
ObservabilityAIAssistantFtrConfig,
|
||||
CreateTest as CreateTestAPI,
|
||||
|
|
@ -32,6 +37,8 @@ export interface TestConfig extends CreateTestAPI {
|
|||
testUser: ObservabilityAIAssistantAPIClient;
|
||||
}
|
||||
>;
|
||||
kibana_ebt_server: (context: InheritedFtrProviderContext) => EBTHelpersContract;
|
||||
kibana_ebt_ui: (context: InheritedFtrProviderContext) => EBTHelpersContract;
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -76,6 +83,8 @@ export function createTestConfig(
|
|||
),
|
||||
};
|
||||
},
|
||||
kibana_ebt_server: KibanaEBTServerProvider,
|
||||
kibana_ebt_ui: KibanaEBTUIProvider,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ const pages = {
|
|||
chatInput: 'observabilityAiAssistantChatPromptEditorTextArea',
|
||||
retryButton: 'observabilityAiAssistantWelcomeMessageSetUpKnowledgeBaseButton',
|
||||
conversationLink: 'observabilityAiAssistantConversationsLink',
|
||||
positiveFeedbackButton: 'observabilityAiAssistantFeedbackButtonsPositiveButton',
|
||||
},
|
||||
createConnectorFlyout: {
|
||||
flyout: 'create-connector-flyout',
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@
|
|||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common';
|
||||
import { ChatFeedback } from '@kbn/observability-ai-assistant-plugin/public/analytics/schemas/chat_feedback';
|
||||
import { pick } from 'lodash';
|
||||
import type OpenAI from 'openai';
|
||||
import {
|
||||
|
|
@ -23,6 +25,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
|
|||
const supertest = getService('supertest');
|
||||
const retry = getService('retry');
|
||||
const log = getService('log');
|
||||
const telemetry = getService('kibana_ebt_ui');
|
||||
|
||||
const driver = getService('__webdriver__');
|
||||
|
||||
|
|
@ -62,12 +65,89 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
|
|||
}
|
||||
}
|
||||
|
||||
async function createOldConversation() {
|
||||
await observabilityAIAssistantAPIClient.testUser({
|
||||
endpoint: 'POST /internal/observability_ai_assistant/conversation',
|
||||
params: {
|
||||
body: {
|
||||
conversation: {
|
||||
messages: [
|
||||
{
|
||||
'@timestamp': '2024-04-18T14:28:50.118Z',
|
||||
message: {
|
||||
role: MessageRole.System,
|
||||
content:
|
||||
'You are a helpful assistant for Elastic Observability. Your goal is to help the Elastic Observability users to quickly assess what is happening in their observed systems. You can help them visualise and analyze data, investigate their systems, perform root cause analysis or identify optimisation opportunities.\n\nIt\'s very important to not assume what the user is meaning. Ask them for clarification if needed.\n\nIf you are unsure about which function should be used and with what arguments, ask the user for clarification or confirmation.\n\nIn KQL ("kqlFilter")) escaping happens with double quotes, not single quotes. Some characters that need escaping are: \':()\\ /". Always put a field value in double quotes. Best: service.name:"opbeans-go". Wrong: service.name:opbeans-go. This is very important!\n\nYou can use Github-flavored Markdown in your responses. If a function returns an array, consider using a Markdown table to format the response.\n\nNote that ES|QL (the Elasticsearch Query Language which is a new piped language) is the preferred query language.\n\nYou MUST use the "query" function when the user wants to:\n- visualize data\n- run any arbitrary query\n- breakdown or filter ES|QL queries that are displayed on the current page\n- convert queries from another language to ES|QL\n- asks general questions about ES|QL\n\nDO NOT UNDER ANY CIRCUMSTANCES generate ES|QL queries or explain anything about the ES|QL query language yourself.\nDO NOT UNDER ANY CIRCUMSTANCES try to correct an ES|QL query yourself - always use the "query" function for this.\n\nDO NOT UNDER ANY CIRCUMSTANCES USE ES|QL syntax (`service.name == "foo"`) with "kqlFilter" (`service.name:"foo"`).\n\nEven if the "context" function was used before that, follow it up with the "query" function. If a query fails, do not attempt to correct it yourself. Again you should call the "query" function,\neven if it has been called before.\n\nWhen the "visualize_query" function has been called, a visualization has been displayed to the user. DO NOT UNDER ANY CIRCUMSTANCES follow up a "visualize_query" function call with your own visualization attempt.\nIf the "execute_query" function has been called, summarize these results for the user. The user does not see a visualization in this case.\n\nYou MUST use the get_dataset_info function function before calling the "query" or "changes" function.\n\nIf a function requires an index, you MUST use the results from the dataset info functions.\n\n\n\nThe user is able to change the language which they want you to reply in on the settings page of the AI Assistant for Observability, which can be found in the Stack Management app under the option AI Assistants.\nIf the user asks how to change the language, reply in the same language the user asked in.You do not have a working memory. If the user expects you to remember the previous conversations, tell them they can set up the knowledge base.\n\nYou MUST respond in the users preferred language which is: English.',
|
||||
},
|
||||
},
|
||||
{
|
||||
'@timestamp': '2024-04-18T14:29:01.615Z',
|
||||
message: {
|
||||
content: 'What are SLOs?',
|
||||
role: MessageRole.User,
|
||||
},
|
||||
},
|
||||
{
|
||||
'@timestamp': '2024-04-18T14:29:01.876Z',
|
||||
message: {
|
||||
role: MessageRole.Assistant,
|
||||
content: '',
|
||||
function_call: {
|
||||
name: 'context',
|
||||
arguments: '{"queries":[],"categories":[]}',
|
||||
trigger: MessageRole.Assistant,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
'@timestamp': '2024-04-18T14:29:01.876Z',
|
||||
message: {
|
||||
content:
|
||||
'{"screen_description":"The user is looking at http://localhost:5601/ftw/app/observabilityAIAssistant/conversations/new. The current time range is 2024-04-18T14:13:49.815Z - 2024-04-18T14:28:49.815Z.","learnings":[]}',
|
||||
name: 'context',
|
||||
role: MessageRole.User,
|
||||
},
|
||||
},
|
||||
{
|
||||
'@timestamp': '2024-04-18T14:29:22.945Z',
|
||||
message: {
|
||||
content:
|
||||
"SLOs, or Service Level Objectives, are a key part of the Site Reliability Engineering (SRE) methodology. They are a target value or range of values for a service level that is measured by an SLI (Service Level Indicator). \n\nAn SLO is a goal for how often and how much you want your service to meet a particular SLI. For example, you might have an SLO that your service should be up and running 99.9% of the time. \n\nSLOs are important because they set clear expectations for your team and your users about the level of service you aim to provide. They also help you make decisions about where to focus your efforts: if you're meeting your SLOs, you can focus on building new features; if you're not meeting your SLOs, you need to focus on improving reliability. \n\nIn Elastic Observability, you can define and monitor your SLOs to ensure your services are meeting their targets.",
|
||||
function_call: {
|
||||
name: '',
|
||||
arguments: '',
|
||||
trigger: MessageRole.Assistant,
|
||||
},
|
||||
role: MessageRole.Assistant,
|
||||
},
|
||||
},
|
||||
],
|
||||
conversation: {
|
||||
title: 'My old conversation',
|
||||
token_count: {
|
||||
completion: 1,
|
||||
prompt: 1,
|
||||
total: 2,
|
||||
},
|
||||
},
|
||||
'@timestamp': '2024-04-18T14:29:22.948',
|
||||
public: false,
|
||||
numeric_labels: {},
|
||||
labels: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
describe('Conversations', () => {
|
||||
let proxy: LlmProxy;
|
||||
before(async () => {
|
||||
await deleteConnectors();
|
||||
await deleteConversations();
|
||||
|
||||
await createOldConversation();
|
||||
|
||||
proxy = await createLlmProxy(log);
|
||||
|
||||
await ui.auth.login();
|
||||
|
|
@ -188,7 +268,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
|
|||
endpoint: 'POST /internal/observability_ai_assistant/conversations',
|
||||
});
|
||||
|
||||
expect(response.body.conversations.length).to.eql(1);
|
||||
expect(response.body.conversations.length).to.eql(2);
|
||||
|
||||
expect(response.body.conversations[0].conversation.title).to.be('My title');
|
||||
|
||||
|
|
@ -227,7 +307,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
|
|||
|
||||
it('updates the list of conversations', async () => {
|
||||
const links = await testSubjects.findAll(ui.pages.conversations.conversationLink);
|
||||
expect(links.length).to.eql(1);
|
||||
expect(links.length).to.eql(2);
|
||||
|
||||
const title = await links[0].getVisibleText();
|
||||
expect(title).to.eql('My title');
|
||||
|
|
@ -255,7 +335,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
|
|||
endpoint: 'POST /internal/observability_ai_assistant/conversations',
|
||||
});
|
||||
|
||||
expect(response.body.conversations.length).to.eql(1);
|
||||
expect(response.body.conversations.length).to.eql(2);
|
||||
});
|
||||
|
||||
it('appends to the existing one', async () => {
|
||||
|
|
@ -287,6 +367,123 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
|
|||
expect(response.body.conversations[0].messages.length).to.eql(9);
|
||||
});
|
||||
});
|
||||
|
||||
describe('and choosing to send feedback', () => {
|
||||
before(async () => {
|
||||
await telemetry.setOptIn(true);
|
||||
await testSubjects.click(ui.pages.conversations.positiveFeedbackButton);
|
||||
});
|
||||
|
||||
it('emits a telemetry event that captures the conversation', async () => {
|
||||
const events = await telemetry.getEvents(1, {
|
||||
eventTypes: ['observability_ai_assistant_chat_feedback'],
|
||||
});
|
||||
|
||||
expect(events.length).to.eql(1);
|
||||
|
||||
const { messageWithFeedback, conversation } = events[0]
|
||||
.properties as unknown as ChatFeedback;
|
||||
|
||||
expect(messageWithFeedback.feedback).to.eql('positive');
|
||||
expect(messageWithFeedback.message.message).to.eql({
|
||||
content: 'My response',
|
||||
function_call: {
|
||||
arguments: '',
|
||||
name: '',
|
||||
trigger: 'assistant',
|
||||
},
|
||||
role: 'assistant',
|
||||
});
|
||||
|
||||
expect(conversation.conversation.title).to.eql('My title');
|
||||
expect(conversation.namespace).to.eql('default');
|
||||
expect(conversation.public).to.eql(false);
|
||||
expect(conversation.user?.name).to.eql('test_user');
|
||||
|
||||
const { messages } = conversation;
|
||||
|
||||
expect(messages.length).to.eql(9);
|
||||
|
||||
expect(messages[0].message.role).to.eql('system');
|
||||
// Verify that system message extension that happen on the server are captured in the telemetry
|
||||
expect(messages[0].message.content).to.contain(
|
||||
'You MUST respond in the users preferred language which is: English.'
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('and opening an old conversation', () => {
|
||||
before(async () => {
|
||||
const conversations = await testSubjects.findAll(
|
||||
ui.pages.conversations.conversationLink
|
||||
);
|
||||
await conversations[1].click();
|
||||
});
|
||||
|
||||
describe('and sending another prompt', () => {
|
||||
before(async () => {
|
||||
const conversationInterceptor = proxy.intercept('conversation', () => true);
|
||||
|
||||
await testSubjects.setValue(
|
||||
ui.pages.conversations.chatInput,
|
||||
'And what are SLIs?'
|
||||
);
|
||||
await testSubjects.pressEnter(ui.pages.conversations.chatInput);
|
||||
|
||||
const conversationSimulator = await conversationInterceptor.waitForIntercept();
|
||||
|
||||
await conversationSimulator.next(
|
||||
'Service Level Indicators (SLIs) are quantifiable defined metrics that measure the performance and availability of a service or distributed system.'
|
||||
);
|
||||
await conversationSimulator.complete();
|
||||
|
||||
await header.waitUntilLoadingHasFinished();
|
||||
});
|
||||
|
||||
describe('and choosing to send feedback', () => {
|
||||
before(async () => {
|
||||
await telemetry.setOptIn(true);
|
||||
const feedbackButtons = await testSubjects.findAll(
|
||||
ui.pages.conversations.positiveFeedbackButton
|
||||
);
|
||||
await feedbackButtons[feedbackButtons.length - 1].click();
|
||||
});
|
||||
|
||||
it('emits a telemetry event that captures the conversation', async () => {
|
||||
const events = await telemetry.getEvents(2, {
|
||||
eventTypes: ['observability_ai_assistant_chat_feedback'],
|
||||
});
|
||||
|
||||
expect(events.length).to.eql(2);
|
||||
|
||||
const { messageWithFeedback, conversation } = events[1]
|
||||
.properties as unknown as ChatFeedback;
|
||||
|
||||
expect(messageWithFeedback.feedback).to.eql('positive');
|
||||
expect(messageWithFeedback.message.message).to.eql({
|
||||
content:
|
||||
'Service Level Indicators (SLIs) are quantifiable defined metrics that measure the performance and availability of a service or distributed system.',
|
||||
function_call: {
|
||||
arguments: '',
|
||||
name: '',
|
||||
trigger: 'assistant',
|
||||
},
|
||||
role: 'assistant',
|
||||
});
|
||||
|
||||
expect(conversation.conversation.title).to.eql('My old conversation');
|
||||
expect(conversation.namespace).to.eql('default');
|
||||
expect(conversation.public).to.eql(false);
|
||||
expect(conversation.user?.name).to.eql('test_user');
|
||||
|
||||
const { messages } = conversation;
|
||||
|
||||
// Verify that data changed after user interaction before being sent to telemetry
|
||||
expect(messages.length).to.eql(9);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -173,5 +173,6 @@
|
|||
"@kbn/apm-data-view",
|
||||
"@kbn/core-saved-objects-api-server",
|
||||
"@kbn/esql-utils",
|
||||
"@kbn/analytics-ftr-helpers-plugin",
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue