github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

tests forwarding systemMessage to the LLM (#212027)

Browse source 
Closes #211910

## Summary  
Currently, we validate that `inferenceClient.chatComplete` is called
twice (once for the title and once for the conversation) and that the
expected system message is included in each call. However, we do not
explicitly verify that the system message is actually passed to the LLM.

To improve reliability, we should introduce a test that directly
inspects the request sent to the LLM via `LLMProxy`.

### Solution - Add a test that explicitly inspects the request sent to
the LLM via `LLMProxy`. :
Forward the system message to the LLM'
Forward User Instructions via System Message to the LLM
sends the system message as the first message in the request to the LLM
This commit is contained in:
Arturo Lidueña 2025-02-24 20:59:56 +01:00 • committed by GitHub
parent 57f83bc201
commit 837c76105e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 160 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

48
x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/chat/chat.spec.ts
View file

@ -16,6 +16,8 @@ import {
import { SupertestWithRoleScope } from '../../../../services/role_scoped_supertest';
import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
const SYSTEM_MESSAGE = `this is a system message`;
export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
const log = getService('log');
const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi');
@ -57,7 +59,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
params: {
body: {
name: 'my_api_call',
systemMessage: 'You are a helpful assistant',
systemMessage: SYSTEM_MESSAGE,
messages,
connectorId: 'does not exist',
functions: [],
@ -68,6 +70,46 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
expect(status).to.be(404);
});
it('returns a 200 if the connector exists', async () => {
void proxy.interceptConversation('Hello from LLM Proxy');
const { status } = await observabilityAIAssistantAPIClient.editor({
endpoint: 'POST /internal/observability_ai_assistant/chat',
params: {
body: {
name: 'my_api_call',
systemMessage: '',
messages,
connectorId,
functions: [],
scopes: ['all'],
},
},
});
await proxy.waitForAllInterceptorsSettled();
expect(status).to.be(200);
});
it('should forward the system message to the LLM', async () => {
const simulatorPromise = proxy.interceptConversation('Hello from LLM Proxy');
await observabilityAIAssistantAPIClient.editor({
endpoint: 'POST /internal/observability_ai_assistant/chat',
params: {
body: {
name: 'my_api_call',
systemMessage: SYSTEM_MESSAGE,
messages,
connectorId,
functions: [],
scopes: ['all'],
},
},
});
await proxy.waitForAllInterceptorsSettled();
const simulator = await simulatorPromise;
const requestData = simulator.requestBody; // This is the request sent to the LLM
expect(requestData.messages[0].content).to.eql(SYSTEM_MESSAGE);
});
it('returns a streaming response from the server', async () => {
const NUM_RESPONSES = 5;
const roleScopedSupertest = getService('roleScopedSupertest');
@ -96,7 +138,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
.on('error', reject)
.send({
name: 'my_api_call',
systemMessage: 'You are a helpful assistant',
systemMessage: SYSTEM_MESSAGE,
messages,
connectorId,
functions: [],
@ -134,7 +176,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
params: {
body: {
name: 'my_api_call',
systemMessage: 'You are a helpful assistant',
systemMessage: SYSTEM_MESSAGE,
messages,
connectorId,
functions: [],

38
x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/complete.spec.ts
View file

@ -213,6 +213,44 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
});
});
describe('LLM invocation with system message', () => {
let systemMessage: string;
before(async () => {
const { status, body } = await observabilityAIAssistantAPIClient.editor({
endpoint: 'GET /internal/observability_ai_assistant/functions',
params: {
query: {
scopes: ['all'],
},
},
});
expect(status).to.be(200);
systemMessage = body.systemMessage;
});
it('forwards the system message as the first message in the request to the LLM with message role "system"', async () => {
const simulatorPromise = proxy.interceptConversation('Hello from LLM Proxy');
await observabilityAIAssistantAPIClient.editor({
endpoint: 'POST /internal/observability_ai_assistant/chat/complete',
params: {
body: {
messages,
connectorId,
persist: false,
screenContexts: [],
scopes: ['all'],
},
},
});
await proxy.waitForAllInterceptorsSettled();
const simulator = await simulatorPromise;
const requestData = simulator.requestBody;
expect(requestData.messages[0].role).to.eql('system');
expect(requestData.messages[0].content).to.eql(systemMessage);
});
});
describe('when creating a new conversation', () => {
let events: StreamingChatResponseEvent[];

77
x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_user_instructions.spec.ts
View file

@ -401,6 +401,83 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
});
});
describe('Forwarding User Instructions via System Message to the LLM', () => {
// Fails on MKI because the LLM Proxy does not yet work there: https://github.com/elastic/obs-ai-assistant-team/issues/199
this.tags(['failsOnMKI']);
let proxy: LlmProxy;
let connectorId: string;
const userInstructionText = 'This is a private instruction';
let systemMessage: string;
before(async () => {
proxy = await createLlmProxy(log);
connectorId = await observabilityAIAssistantAPIClient.createProxyActionConnector({
port: proxy.getPort(),
});
const res = await observabilityAIAssistantAPIClient.editor({
endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions',
params: {
body: {
id: 'private-instruction-id',
text: userInstructionText,
public: false,
},
},
});
expect(res.status).to.be(200);
const { status, body } = await observabilityAIAssistantAPIClient.editor({
endpoint: 'GET /internal/observability_ai_assistant/functions',
params: {
query: {
scopes: ['all'],
},
},
});
expect(status).to.be(200);
systemMessage = body.systemMessage;
});
after(async () => {
proxy.close();
await observabilityAIAssistantAPIClient.deleteActionConnector({
actionId: connectorId,
});
});
it('includes private KB instructions in the system message sent to the LLM', async () => {
const simulatorPromise = proxy.interceptConversation('Hello from LLM Proxy');
const messages: Message[] = [
{
'@timestamp': new Date().toISOString(),
message: {
role: MessageRole.User,
content: 'Today we will be testing user instructions!',
},
},
];
await observabilityAIAssistantAPIClient.editor({
endpoint: 'POST /internal/observability_ai_assistant/chat/complete',
params: {
body: {
messages,
connectorId,
persist: false,
screenContexts: [],
scopes: ['all'],
},
},
});
await proxy.waitForAllInterceptorsSettled();
const simulator = await simulatorPromise;
const requestData = simulator.requestBody;
expect(requestData.messages[0].content).to.contain(userInstructionText);
expect(requestData.messages[0].content).to.eql(systemMessage);
});
});
describe('security roles and access privileges', () => {
describe('should deny access for users without the ai_assistant privilege', () => {
it('PUT /internal/observability_ai_assistant/kb/user_instructions', async () => {