[8.10] [Observability AI Assistant] More specific error handling (#165068) (#165188)

# Backport

This will backport the following commits from `main` to `8.10`:
- [[Observability AI Assistant] More specific error handling
(#165068)](https://github.com/elastic/kibana/pull/165068)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Dario
Gieselaar","email":"dario.gieselaar@elastic.co"},"sourceCommit":{"committedDate":"2023-08-30T07:34:02Z","message":"[Observability
AI Assistant] More specific error handling (#165068)\n\nImplements more
specific error handling, in addition to a bug fix w/ the\r\nLens
function where it was not rendering outside of the Observability
AI\r\nAssistant plugin's context, and validation of the parameters
before\r\nexecuting the function.\r\n\r\nAdditionally:\r\n- improves
recall function\r\n- filter alerts and services by alert/health
status\r\n\r\n---------\r\n\r\nCo-authored-by: Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"c5c6574592ee2b88f48454b8ad6d23229bdafab2","branchLabelMapping":{"^v8.11.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:APM","release_note:skip","v8.10.0","v8.11.0"],"number":165068,"url":"https://github.com/elastic/kibana/pull/165068","mergeCommit":{"message":"[Observability
AI Assistant] More specific error handling (#165068)\n\nImplements more
specific error handling, in addition to a bug fix w/ the\r\nLens
function where it was not rendering outside of the Observability
AI\r\nAssistant plugin's context, and validation of the parameters
before\r\nexecuting the function.\r\n\r\nAdditionally:\r\n- improves
recall function\r\n- filter alerts and services by alert/health
status\r\n\r\n---------\r\n\r\nCo-authored-by: Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"c5c6574592ee2b88f48454b8ad6d23229bdafab2"}},"sourceBranch":"main","suggestedTargetBranches":["8.10"],"targetPullRequestStates":[{"branch":"8.10","label":"v8.10.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.11.0","labelRegex":"^v8.11.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/165068","number":165068,"mergeCommit":{"message":"[Observability
AI Assistant] More specific error handling (#165068)\n\nImplements more
specific error handling, in addition to a bug fix w/ the\r\nLens
function where it was not rendering outside of the Observability
AI\r\nAssistant plugin's context, and validation of the parameters
before\r\nexecuting the function.\r\n\r\nAdditionally:\r\n- improves
recall function\r\n- filter alerts and services by alert/health
status\r\n\r\n---------\r\n\r\nCo-authored-by: Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"c5c6574592ee2b88f48454b8ad6d23229bdafab2"}}]}]
BACKPORT-->
This commit is contained in:
Dario Gieselaar 2023-08-30 10:51:45 +02:00 committed by GitHub
parent 40b6e3170c
commit 764c93f5ba
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 184 additions and 46 deletions

View file

@ -90,6 +90,7 @@
"dependencies": {
"@appland/sql-parser": "^1.5.1",
"@babel/runtime": "^7.21.0",
"@cfworker/json-schema": "^1.12.7",
"@dnd-kit/core": "^3.1.1",
"@dnd-kit/sortable": "^4.0.0",
"@dnd-kit/utilities": "^2.0.0",

View file

@ -105,7 +105,7 @@ module.exports = {
transformIgnorePatterns: [
// ignore all node_modules except monaco-editor and react-monaco-editor which requires babel transforms to handle dynamic import()
// since ESM modules are not natively supported in Jest yet (https://github.com/facebook/jest/issues/4842)
'[/\\\\]node_modules(?![\\/\\\\](byte-size|monaco-editor|monaco-yaml|vscode-languageserver-types|react-monaco-editor|d3-interpolate|d3-color))[/\\\\].+\\.js$',
'[/\\\\]node_modules(?![\\/\\\\](byte-size|monaco-editor|monaco-yaml|vscode-languageserver-types|react-monaco-editor|d3-interpolate|d3-color|@cfworker))[/\\\\].+\\.js$',
'packages/kbn-pm/dist/index.js',
],

View file

@ -7,6 +7,7 @@
import { i18n } from '@kbn/i18n';
import type { RegisterFunctionDefinition } from '@kbn/observability-ai-assistant-plugin/common/types';
import { ServiceHealthStatus } from '../../common/service_health_status';
import { callApmApi } from '../services/rest/create_call_apm_api';
import { NON_EMPTY_STRING } from '../utils/non_empty_string_ref';
@ -28,6 +29,7 @@ export function registerGetApmServicesListFunction({
),
parameters: {
type: 'object',
additionalProperties: false,
properties: {
'service.environment': {
...NON_EMPTY_STRING,
@ -44,6 +46,21 @@ export function registerGetApmServicesListFunction({
description:
'The end of the time range, in Elasticsearch date math, like `now-24h`.',
},
healthStatus: {
type: 'array',
description: 'Filter service list by health status',
additionalProperties: false,
additionalItems: false,
items: {
type: 'string',
enum: [
ServiceHealthStatus.unknown,
ServiceHealthStatus.healthy,
ServiceHealthStatus.warning,
ServiceHealthStatus.critical,
],
},
},
},
required: ['start', 'end'],
} as const,

View file

@ -213,6 +213,14 @@ const getApmServicesListRoute = createApmServerRoute({
}),
t.partial({
'service.environment': environmentRt.props.environment,
healthStatus: t.array(
t.union([
t.literal(ServiceHealthStatus.unknown),
t.literal(ServiceHealthStatus.healthy),
t.literal(ServiceHealthStatus.warning),
t.literal(ServiceHealthStatus.critical),
])
),
}),
]),
}),
@ -223,6 +231,8 @@ const getApmServicesListRoute = createApmServerRoute({
const { params } = resources;
const { query } = params;
const { healthStatus } = query;
const [apmEventClient, apmAlertsClient, mlClient, randomSampler] =
await Promise.all([
getApmEventClient(resources),
@ -253,7 +263,7 @@ const getApmServicesListRoute = createApmServerRoute({
mlClient,
});
const mappedItems = serviceItems.items.map((item): ApmServicesListItem => {
let mappedItems = serviceItems.items.map((item): ApmServicesListItem => {
return {
'service.name': item.serviceName,
'agent.name': item.agentName,
@ -264,6 +274,12 @@ const getApmServicesListRoute = createApmServerRoute({
};
});
if (healthStatus && healthStatus.length) {
mappedItems = mappedItems.filter((item): boolean =>
healthStatus.includes(item.healthStatus)
);
}
return {
content: mappedItems,
};

View file

@ -58,11 +58,16 @@ export function registerAlertsFunction({
description:
'a KQL query to filter the data by. If no filter should be applied, leave it empty.',
},
includeRecovered: {
type: 'boolean',
description:
'Whether to include recovered/closed alerts. Defaults to false, which means only active alerts will be returned',
},
},
required: ['start', 'end'],
} as const,
},
({ arguments: { start, end, featureIds, filter } }, signal) => {
({ arguments: { start, end, featureIds, filter, includeRecovered } }, signal) => {
return service.callApi('POST /internal/observability_ai_assistant/functions/alerts', {
params: {
body: {
@ -71,6 +76,7 @@ export function registerAlertsFunction({
featureIds:
featureIds && featureIds.length > 0 ? featureIds : DEFAULT_FEATURE_IDS.concat(),
filter,
includeRecovered,
},
},
signal,

View file

@ -5,12 +5,13 @@
* 2.0.
*/
import { EuiLoadingSpinner } from '@elastic/eui';
import type { DataViewsServicePublic } from '@kbn/data-views-plugin/public/types';
import { FIELD_FORMAT_IDS } from '@kbn/field-formats-plugin/common';
import { LensAttributesBuilder, XYChart, XYDataLayer } from '@kbn/lens-embeddable-utils';
import type { LensPublicStart } from '@kbn/lens-plugin/public';
import React from 'react';
import useAsync from 'react-use/lib/useAsync';
import { FIELD_FORMAT_IDS } from '@kbn/field-formats-plugin/common';
import type { RegisterFunctionDefinition } from '../../common/types';
import { useKibana } from '../hooks/use_kibana';
import type {
ObservabilityAIAssistantPluginStartDependencies,
ObservabilityAIAssistantService,
@ -33,20 +34,16 @@ function Lens({
xyDataLayer,
start,
end,
lens,
dataViews,
}: {
indexPattern: string;
xyDataLayer: XYDataLayer;
start: string;
end: string;
lens: LensPublicStart;
dataViews: DataViewsServicePublic;
}) {
const {
services: {
plugins: {
start: { lens, dataViews },
},
},
} = useKibana();
const formulaAsync = useAsync(() => {
return lens.stateHelperApi();
}, [lens]);
@ -214,7 +211,16 @@ export function registerLensFunction({
},
});
return <Lens indexPattern={indexPattern} xyDataLayer={xyDataLayer} start={start} end={end} />;
return (
<Lens
indexPattern={indexPattern}
xyDataLayer={xyDataLayer}
start={start}
end={end}
lens={pluginsStart.lens}
dataViews={pluginsStart.dataViews}
/>
);
}
);
}

View file

@ -20,16 +20,24 @@ export function registerRecallFunction({
{
name: 'recall',
contexts: ['core'],
description: `Use this function to recall earlier learnings. Anything you will summarise can be retrieved again later via this function. The queries you use are very important, as they will decide the context that is included in the conversation. Make sure the query covers the following aspects:
- The user's intent
- Any data (like field names) mentioned in the user's request
- Anything you've inferred from the user's request
- The functions you think might be suitable for answering the user's request. If there are multiple functions that seem suitable, create multiple queries. Use the function name in the query.
description: `Use this function to recall earlier learnings. Anything you will summarise can be retrieved again later via this function. This is semantic/vector search so there's no need for an exact match.
For instance, when the user asks: "can you visualise the average request duration for opbeans-go over the last 7 days?", the queries could be:
- "visualise average request duration for APM service opbeans-go"
Make sure the query covers the following aspects:
- The user's prompt, verbatim
- Anything you've inferred from the user's request, but is not mentioned in the user's request
- The functions you think might be suitable for answering the user's request. If there are multiple functions that seem suitable, create multiple queries. Use the function name in the query.
Q: "can you visualise the average request duration for opbeans-go over the last 7 days?"
A: -"can you visualise the average request duration for opbeans-go over the last 7 days?"
- "APM service"
- "lens function usage"
- "get_apm_timeseries function usage"`,
- "get_apm_timeseries function usage"
Q: "what alerts are active?"
A: - "what alerts are active?"
- "alerts function usage"
`,
descriptionForUser: 'This function allows the assistant to recall previous learnings.',
parameters: {
type: 'object',

View file

@ -122,8 +122,10 @@ export function useTimeline({
},
error: reject,
complete: () => {
if (pendingMessageLocal?.error) {
notifications.toasts.addError(pendingMessageLocal?.error, {
const error = pendingMessageLocal?.error;
if (error) {
notifications.toasts.addError(error, {
title: i18n.translate('xpack.observabilityAiAssistant.failedToLoadResponse', {
defaultMessage: 'Failed to load response from the AI Assistant',
}),
@ -190,7 +192,7 @@ export function useTimeline({
name,
content: JSON.stringify({
message: error.toString(),
error: error.body,
error,
}),
},
})

View file

@ -156,6 +156,26 @@ describe('createChatService', () => {
});
});
it('propagates content errors', async () => {
respondWithChunks({
chunks: [
`data: {"error":{"message":"The server had an error while processing your request. Sorry about that!","type":"server_error","param":null,"code":null}}`,
],
});
const response$ = chat();
const value = await lastValueFrom(response$);
expect(value).toEqual({
aborted: false,
error: expect.any(Error),
message: {
role: 'assistant',
},
});
});
it('cancels a running http request when aborted', async () => {
clientSpy.mockImplementationOnce((endpoint: string, options: HttpFetchOptions) => {
options.signal?.addEventListener('abort', () => {

View file

@ -4,31 +4,33 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/* eslint-disable max-classes-per-file*/
import { Validator, type Schema, type OutputUnit } from '@cfworker/json-schema';
import { HttpResponse } from '@kbn/core/public';
import { AbortError } from '@kbn/kibana-utils-plugin/common';
import { IncomingMessage } from 'http';
import { cloneDeep, pick } from 'lodash';
import {
BehaviorSubject,
map,
filter as rxJsFilter,
scan,
catchError,
of,
concatMap,
shareReplay,
finalize,
delay,
filter as rxJsFilter,
finalize,
map,
of,
scan,
shareReplay,
tap,
} from 'rxjs';
import { HttpResponse } from '@kbn/core/public';
import { AbortError } from '@kbn/kibana-utils-plugin/common';
import {
type RegisterContextDefinition,
type RegisterFunctionDefinition,
Message,
MessageRole,
ContextRegistry,
FunctionRegistry,
Message,
MessageRole,
type RegisterContextDefinition,
type RegisterFunctionDefinition,
} from '../../common/types';
import { ObservabilityAIAssistantAPIClient } from '../api';
import type {
@ -45,6 +47,14 @@ class TokenLimitReachedError extends Error {
}
}
class ServerError extends Error {}
export class FunctionArgsValidationError extends Error {
constructor(public readonly errors: OutputUnit[]) {
super('Function arguments are invalid');
}
}
export async function createChatService({
signal: setupAbortSignal,
registrations,
@ -57,11 +67,14 @@ export async function createChatService({
const contextRegistry: ContextRegistry = new Map();
const functionRegistry: FunctionRegistry = new Map();
const validators = new Map<string, Validator>();
const registerContext: RegisterContextDefinition = (context) => {
contextRegistry.set(context.name, context);
};
const registerFunction: RegisterFunctionDefinition = (def, respond, render) => {
validators.set(def.name, new Validator(def.parameters as Schema, '2020-12', false));
functionRegistry.set(def.name, { options: def, respond, render });
};
@ -90,6 +103,14 @@ export async function createChatService({
registrations.map((fn) => fn({ signal: setupAbortSignal, registerContext, registerFunction }))
);
function validate(name: string, parameters: unknown) {
const validator = validators.get(name)!;
const result = validator.validate(parameters);
if (!result.valid) {
throw new FunctionArgsValidationError(result.errors);
}
}
return {
executeFunction: async (name, args, signal) => {
const fn = functionRegistry.get(name);
@ -100,7 +121,7 @@ export async function createChatService({
const parsedArguments = args ? JSON.parse(args) : {};
// validate
validate(name, parsedArguments);
return await fn.respond({ arguments: parsedArguments }, signal);
},
@ -118,8 +139,6 @@ export async function createChatService({
data: JSON.parse(response.data ?? '{}'),
};
// validate
return fn.render?.({ response: parsedResponse, arguments: parsedArguments });
},
getContexts,
@ -171,10 +190,23 @@ export async function createChatService({
.pipe(
map((line) => line.substring(6)),
rxJsFilter((line) => !!line && line !== '[DONE]'),
map((line) => JSON.parse(line) as CreateChatCompletionResponseChunk),
rxJsFilter((line) => line.object === 'chat.completion.chunk'),
tap((choice) => {
if (choice.choices[0].finish_reason === 'length') {
map(
(line) =>
JSON.parse(line) as
| CreateChatCompletionResponseChunk
| { error: { message: string } }
),
tap((line) => {
if ('error' in line) {
throw new ServerError(line.error.message);
}
}),
rxJsFilter(
(line): line is CreateChatCompletionResponseChunk =>
'object' in line && line.object === 'chat.completion.chunk'
),
tap((line) => {
if (line.choices[0].finish_reason === 'length') {
throw new TokenLimitReachedError();
}
}),
@ -217,6 +249,16 @@ export async function createChatService({
subject.complete();
});
})
.catch(async (err) => {
if ('response' in err) {
const body = await (err.response as HttpResponse['response'])?.json();
err.body = body;
if (body.message) {
err.message = body.message;
}
}
throw err;
})
.catch((err) => {
subject.next({
...subject.value,

View file

@ -9,7 +9,6 @@ import type { CoreStart } from '@kbn/core/public';
import { SecurityPluginStart } from '@kbn/security-plugin/public';
import { createCallObservabilityAIAssistantAPI } from '../api';
import type { ChatRegistrationFunction, ObservabilityAIAssistantService } from '../types';
import { createChatService } from './create_chat_service';
export function createService({
coreStart,
@ -32,7 +31,8 @@ export function createService({
registrations.push(fn);
},
start: async ({ signal }) => {
return await createChatService({ client, signal, registrations });
const mod = await import('./create_chat_service');
return await mod.createChatService({ client, signal, registrations });
},
callApi: client,

View file

@ -11,6 +11,10 @@ import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils';
import * as t from 'io-ts';
import { omit } from 'lodash';
import { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common';
import {
ALERT_STATUS,
ALERT_STATUS_ACTIVE,
} from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names';
import type { KnowledgeBaseEntry } from '../../../common/types';
import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route';
@ -79,6 +83,7 @@ const functionAlertsRoute = createObservabilityAIAssistantServerRoute({
}),
t.partial({
filter: t.string,
includeRecovered: toBooleanRt,
}),
]),
}),
@ -95,6 +100,7 @@ const functionAlertsRoute = createObservabilityAIAssistantServerRoute({
start: startAsDatemath,
end: endAsDatemath,
filter,
includeRecovered,
} = resources.params.body;
const racContext = await resources.context.rac;
@ -120,6 +126,15 @@ const functionAlertsRoute = createObservabilityAIAssistantServerRoute({
},
},
...kqlQuery,
...(!includeRecovered
? [
{
term: {
[ALERT_STATUS]: ALERT_STATUS_ACTIVE,
},
},
]
: []),
],
},
},

View file

@ -1306,6 +1306,11 @@
resolved "https://registry.yarnpkg.com/@cbor-extract/cbor-extract-win32-x64/-/cbor-extract-win32-x64-2.0.0.tgz#4d4ad91527a8313c3db1e2167a8821dfae9d6211"
integrity sha512-XqVuJEnE0jpl/RkuSp04FF2UE73gY52Y4nZaIE6j9GAeSH2cHYU5CCd4TaVMDi2M18ZpZv7XhL/k+nneQzyJpQ==
"@cfworker/json-schema@^1.12.7":
version "1.12.7"
resolved "https://registry.yarnpkg.com/@cfworker/json-schema/-/json-schema-1.12.7.tgz#064d082a11881f684300bc7e6d3021e9d98f9a59"
integrity sha512-KEJUW22arGRQVoS6Ti8SvgXnme6NNMMcGBugdir1hf32ofWUXC8guwrFbepO2+YtqxNBUo5oO0pLYM5d4pyjOg==
"@cnakazawa/watch@^1.0.3":
version "1.0.3"
resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.3.tgz#099139eaec7ebf07a27c1786a3ff64f39464d2ef"