mirror of
https://github.com/elastic/kibana.git
synced 2025-06-27 18:51:07 -04:00
[Obs AI Assistant] Include an AdHoc instruction about the slack connector to avoid executing a loop (#199531)
Closes https://github.com/elastic/kibana/issues/185028 ## Summary ### Problem The ticket mentions that there was an issue with displaying results. However both `display results` and `visualize query` are working as expected based on my investigation. More details including a video are attached [here](https://github.com/elastic/kibana/issues/185028#issuecomment-2464879452). The function calling loop seems to occur when the AI Assistant is trying to send the output to Slack via the Kibana Slack connector. In order to do this, the LLM invokes the function `execute_connector`. For the Slack connector, `id` and `params` properties are required. However, the LLM only populated `id` and not `params` which causes an error when validated against the schema for the Slack connector. - Sometimes, it's able to retry a few times and successfully send the output to Slack. - Sometimes, it goes into a loop trying to find `params` and failing repeatedly. Attaching another similar issue for more context - https://github.com/elastic/kibana/issues/195564 _With the solution below, my intention is to send the output to Slack in one go, without retrying the function `execute_connector`._ ### Solution Based on the solutions I experimented with, seems like we need to force the LLM to understand what's needed for the Slack connector. I tried a few options here and the combination of updates that worked are as follows: - Appending an AdHoc instruction about the Slack connector properties. (This gives the LLM some additional information about the required properties) - Updating the `properties` attached to the connector (when passing the connector list to the LLM), to reflect both `id` and `params` with `message`. With the above change, the AI Assistant has managed to consistently send the output to Slack _**without any retries**_ because of missing `params`. ### Screenshots: (all alert triggers successfully sent the output to Slack without having to retry the function) <img width="1452" alt="success-attempts-to-slack-connector" src="https://github.com/user-attachments/assets/715a5957-2c04-4a55-868f-34abe564f6d4">
This commit is contained in:
parent
5d0b62ce9e
commit
ed4c0df9ee
2 changed files with 34 additions and 3 deletions
|
@ -8,13 +8,15 @@
|
||||||
import { FunctionRegistrationParameters } from '.';
|
import { FunctionRegistrationParameters } from '.';
|
||||||
import { FunctionVisibility } from '../../common';
|
import { FunctionVisibility } from '../../common';
|
||||||
|
|
||||||
|
export const EXECUTE_CONNECTOR_FUNCTION_NAME = 'execute_connector';
|
||||||
|
|
||||||
export function registerExecuteConnectorFunction({
|
export function registerExecuteConnectorFunction({
|
||||||
functions,
|
functions,
|
||||||
resources,
|
resources,
|
||||||
}: FunctionRegistrationParameters) {
|
}: FunctionRegistrationParameters) {
|
||||||
functions.registerFunction(
|
functions.registerFunction(
|
||||||
{
|
{
|
||||||
name: 'execute_connector',
|
name: EXECUTE_CONNECTOR_FUNCTION_NAME,
|
||||||
description: 'Use this function when user explicitly asks to call a kibana connector.',
|
description: 'Use this function when user explicitly asks to call a kibana connector.',
|
||||||
visibility: FunctionVisibility.AssistantOnly,
|
visibility: FunctionVisibility.AssistantOnly,
|
||||||
parameters: {
|
parameters: {
|
||||||
|
|
|
@ -23,7 +23,6 @@ import {
|
||||||
JiraParamsSchema,
|
JiraParamsSchema,
|
||||||
PagerdutyParamsSchema,
|
PagerdutyParamsSchema,
|
||||||
SlackApiParamsSchema,
|
SlackApiParamsSchema,
|
||||||
SlackParamsSchema,
|
|
||||||
WebhookParamsSchema,
|
WebhookParamsSchema,
|
||||||
} from '@kbn/stack-connectors-plugin/server';
|
} from '@kbn/stack-connectors-plugin/server';
|
||||||
import { ObservabilityAIAssistantRouteHandlerResources } from '@kbn/observability-ai-assistant-plugin/server/routes/types';
|
import { ObservabilityAIAssistantRouteHandlerResources } from '@kbn/observability-ai-assistant-plugin/server/routes/types';
|
||||||
|
@ -37,14 +36,26 @@ import { CompatibleJSONSchema } from '@kbn/observability-ai-assistant-plugin/com
|
||||||
import { AlertDetailsContextualInsightsService } from '@kbn/observability-plugin/server/services';
|
import { AlertDetailsContextualInsightsService } from '@kbn/observability-plugin/server/services';
|
||||||
import { getSystemMessageFromInstructions } from '@kbn/observability-ai-assistant-plugin/server/service/util/get_system_message_from_instructions';
|
import { getSystemMessageFromInstructions } from '@kbn/observability-ai-assistant-plugin/server/service/util/get_system_message_from_instructions';
|
||||||
import { AdHocInstruction } from '@kbn/observability-ai-assistant-plugin/common/types';
|
import { AdHocInstruction } from '@kbn/observability-ai-assistant-plugin/common/types';
|
||||||
|
import { EXECUTE_CONNECTOR_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/execute_connector';
|
||||||
import { convertSchemaToOpenApi } from './convert_schema_to_open_api';
|
import { convertSchemaToOpenApi } from './convert_schema_to_open_api';
|
||||||
import { OBSERVABILITY_AI_ASSISTANT_CONNECTOR_ID } from '../../common/rule_connector';
|
import { OBSERVABILITY_AI_ASSISTANT_CONNECTOR_ID } from '../../common/rule_connector';
|
||||||
|
|
||||||
const CONNECTOR_PRIVILEGES = ['api:observabilityAIAssistant', 'app:observabilityAIAssistant'];
|
const CONNECTOR_PRIVILEGES = ['api:observabilityAIAssistant', 'app:observabilityAIAssistant'];
|
||||||
|
|
||||||
const connectorParamsSchemas: Record<string, CompatibleJSONSchema> = {
|
const connectorParamsSchemas: Record<string, CompatibleJSONSchema> = {
|
||||||
|
'.slack': {
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
id: { type: 'string' },
|
||||||
|
params: {
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
message: { type: 'string' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
'.slack_api': convertSchemaToOpenApi(SlackApiParamsSchema),
|
'.slack_api': convertSchemaToOpenApi(SlackApiParamsSchema),
|
||||||
'.slack': convertSchemaToOpenApi(SlackParamsSchema),
|
|
||||||
'.email': convertSchemaToOpenApi(EmailParamsSchema),
|
'.email': convertSchemaToOpenApi(EmailParamsSchema),
|
||||||
'.webhook': convertSchemaToOpenApi(WebhookParamsSchema),
|
'.webhook': convertSchemaToOpenApi(WebhookParamsSchema),
|
||||||
'.jira': convertSchemaToOpenApi(JiraParamsSchema),
|
'.jira': convertSchemaToOpenApi(JiraParamsSchema),
|
||||||
|
@ -189,6 +200,24 @@ If available, include the link of the conversation at the end of your answer.`
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const hasSlackConnector = !!connectorsList.filter(
|
||||||
|
(connector) => connector.actionTypeId === '.slack'
|
||||||
|
).length;
|
||||||
|
|
||||||
|
if (hasSlackConnector && functionClient.hasFunction(EXECUTE_CONNECTOR_FUNCTION_NAME)) {
|
||||||
|
const slackConnectorInstruction: AdHocInstruction = {
|
||||||
|
instruction_type: 'application_instruction',
|
||||||
|
text: dedent(
|
||||||
|
`The execute_connector function can be used to invoke Kibana connectors.
|
||||||
|
To send to the Slack connector, you need the following arguments:
|
||||||
|
- the "id" of the connector
|
||||||
|
- the "params" parameter that you will fill with the message
|
||||||
|
Please include both "id" and "params.message" in the function arguments when executing the Slack connector..`
|
||||||
|
),
|
||||||
|
};
|
||||||
|
functionClient.registerAdhocInstruction(slackConnectorInstruction);
|
||||||
|
}
|
||||||
|
|
||||||
const alertsContext = await getAlertsContext(
|
const alertsContext = await getAlertsContext(
|
||||||
execOptions.params.rule,
|
execOptions.params.rule,
|
||||||
execOptions.params.alerts,
|
execOptions.params.alerts,
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue