mirror of
https://github.com/elastic/kibana.git
synced 2025-06-27 18:51:07 -04:00
## Summary Introduces a new `security_solution/gen_ai_evals.yml` BuildKite pipeline for automatically running our Assistant and Attack Discovery evaluation suites weekly. ### To Run Locally: Ensure you are authenticated with vault for LLM + LangSmith creds: > See [internal docs](https://github.com/elastic/infra/blob/master/docs/vault/README.md#login-with-your-okta) for setup/login instructions. Fetch Connectors and LangSmith creds: > [!NOTE] > In discussion with @elastic/kibana-operations it was preferred to use the ci-prod secrets vault, so we cannot self-manage the secrets. To test this locally though, you can grab the secrets and follow the instructions in this [paste bin](https://p.elstc.co/paste/q7k+zYOc#PN0kasw11u2J0XWC2Ls5PMNWreKzKTpgWA1wtsPzeH+). ``` cd x-pack/test/security_solution_api_integration node scripts/genai/vault/retrieve_secrets.js ``` Navigate to api integration directory, load the env vars, and start server: ``` cd x-pack/test/security_solution_api_integration export KIBANA_SECURITY_TESTING_AI_CONNECTORS=$(base64 -w 0 < scripts/genai/vault/connector_config.json) && export KIBANA_SECURITY_TESTING_LANGSMITH_KEY=$(base64 -w 0 < scripts/genai/vault/langsmith_key.txt) yarn genai_evals:server:ess ``` Then in another terminal, load vars and run the tests: ``` cd x-pack/test/security_solution_api_integration export KIBANA_SECURITY_TESTING_AI_CONNECTORS=$(base64 -w 0 < scripts/genai/vault/connector_config.json) && export KIBANA_SECURITY_TESTING_LANGSMITH_KEY=$(base64 -w 0 < scripts/genai/vault/langsmith_key.txt) yarn genai_evals🏃ess ``` ### To manually run on BuildKite: Navigate to [BuildKite](https://buildkite.com/elastic?filter=ftr-security-solution-gen-ai-evaluations) and run `ftr-security-solution-gen-ai-evaluations` pipeline. ### To manually run on BuildKite for specific PR: In `.buildkite/ftr_security_stateful_configs.yml`, temporarily move the `genai/evaluations/trial_license_complete_tier/configs/ess.config.ts` line down to the `enabled` section. Will see if we can do this without requiring a commit. @elastic/kibana-operations is it possible to set a buildkite env var that can be read in FTR tests when a specific GitHub label is added to the PR? I.e. can I create a `SecurityGenAI:Run Evals` label that when added will run this suite as part of the build? > [!NOTE] > Currently the connectors secrets only include `gpt-4o` and `gpt-4o-mini`. Waiting on finalized list w/ credentials from @jamesspi and @peluja1012 and then we can have ops update using the scripts included in this PR. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Patryk Kopycinski <patryk.kopycinski@elastic.co>
183 lines
6.7 KiB
TypeScript
Executable file
183 lines
6.7 KiB
TypeScript
Executable file
/*
|
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
|
* or more contributor license agreements. Licensed under the Elastic License
|
|
* 2.0; you may not use this file except in compliance with the Elastic License
|
|
* 2.0.
|
|
*/
|
|
|
|
import { PluginInitializerContext, CoreStart, Plugin, Logger } from '@kbn/core/server';
|
|
|
|
import {
|
|
ATTACK_DISCOVERY_SCHEDULES_ENABLED_FEATURE_FLAG,
|
|
AssistantFeatures,
|
|
} from '@kbn/elastic-assistant-common';
|
|
import { ReplaySubject, type Subject } from 'rxjs';
|
|
import { events } from './lib/telemetry/event_based_telemetry';
|
|
import {
|
|
AssistantTool,
|
|
ElasticAssistantPluginCoreSetupDependencies,
|
|
ElasticAssistantPluginSetup,
|
|
ElasticAssistantPluginSetupDependencies,
|
|
ElasticAssistantPluginStart,
|
|
ElasticAssistantPluginStartDependencies,
|
|
ElasticAssistantRequestHandlerContext,
|
|
} from './types';
|
|
import { AIAssistantService } from './ai_assistant_service';
|
|
import { RequestContextFactory } from './routes/request_context_factory';
|
|
import { createEventLogger } from './create_event_logger';
|
|
import { PLUGIN_ID } from '../common/constants';
|
|
import { registerEventLogProvider } from './register_event_log_provider';
|
|
import { registerRoutes } from './routes/register_routes';
|
|
import { CallbackIds, appContextService } from './services/app_context';
|
|
import { removeLegacyQuickPrompt } from './ai_assistant_service/helpers';
|
|
import { getAttackDiscoveryScheduleType } from './lib/attack_discovery/schedules/register_schedule/definition';
|
|
import type { ConfigSchema } from './config_schema';
|
|
|
|
export class ElasticAssistantPlugin
|
|
implements
|
|
Plugin<
|
|
ElasticAssistantPluginSetup,
|
|
ElasticAssistantPluginStart,
|
|
ElasticAssistantPluginSetupDependencies,
|
|
ElasticAssistantPluginStartDependencies
|
|
>
|
|
{
|
|
private readonly logger: Logger;
|
|
private assistantService: AIAssistantService | undefined;
|
|
private pluginStop$: Subject<void>;
|
|
private readonly kibanaVersion: PluginInitializerContext['env']['packageInfo']['version'];
|
|
private readonly config: ConfigSchema;
|
|
|
|
constructor(initializerContext: PluginInitializerContext) {
|
|
this.pluginStop$ = new ReplaySubject(1);
|
|
this.logger = initializerContext.logger.get();
|
|
this.kibanaVersion = initializerContext.env.packageInfo.version;
|
|
this.config = initializerContext.config.get<ConfigSchema>();
|
|
}
|
|
|
|
public setup(
|
|
core: ElasticAssistantPluginCoreSetupDependencies,
|
|
plugins: ElasticAssistantPluginSetupDependencies
|
|
) {
|
|
this.logger.debug('elasticAssistant: Setup');
|
|
|
|
registerEventLogProvider(plugins.eventLog);
|
|
const eventLogger = createEventLogger(plugins.eventLog); // must be created during setup phase
|
|
|
|
this.assistantService = new AIAssistantService({
|
|
logger: this.logger.get('service'),
|
|
ml: plugins.ml,
|
|
taskManager: plugins.taskManager,
|
|
kibanaVersion: this.kibanaVersion,
|
|
elserInferenceId: this.config.elserInferenceId,
|
|
elasticsearchClientPromise: core
|
|
.getStartServices()
|
|
.then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser),
|
|
productDocManager: core
|
|
.getStartServices()
|
|
.then(([_, { productDocBase }]) => productDocBase.management),
|
|
pluginStop$: this.pluginStop$,
|
|
savedAttackDiscoveries: true,
|
|
});
|
|
|
|
const requestContextFactory = new RequestContextFactory({
|
|
logger: this.logger,
|
|
core,
|
|
plugins,
|
|
kibanaVersion: this.kibanaVersion,
|
|
assistantService: this.assistantService,
|
|
});
|
|
|
|
const router = core.http.createRouter<ElasticAssistantRequestHandlerContext>();
|
|
core.http.registerRouteHandlerContext<ElasticAssistantRequestHandlerContext, typeof PLUGIN_ID>(
|
|
PLUGIN_ID,
|
|
(context, request) =>
|
|
requestContextFactory.create(
|
|
context,
|
|
request,
|
|
plugins.eventLog.getIndexPattern(),
|
|
eventLogger
|
|
)
|
|
);
|
|
events.forEach((eventConfig) => core.analytics.registerEventType(eventConfig));
|
|
|
|
registerRoutes(router, this.logger, this.config);
|
|
|
|
// The featureFlags service is not available in the core setup, so we need
|
|
// to wait for the start services to be available to read the feature flags.
|
|
// This can take a while, but the plugin setup phase cannot run for a long time.
|
|
// As a workaround, this promise does not block the setup phase.
|
|
core
|
|
.getStartServices()
|
|
.then(([{ featureFlags }]) => {
|
|
// read all feature flags:
|
|
void Promise.all([
|
|
featureFlags.getBooleanValue(ATTACK_DISCOVERY_SCHEDULES_ENABLED_FEATURE_FLAG, false),
|
|
// add more feature flags here
|
|
]).then(([assistantAttackDiscoverySchedulingEnabled]) => {
|
|
if (assistantAttackDiscoverySchedulingEnabled) {
|
|
// Register Attack Discovery Schedule type
|
|
plugins.alerting.registerType(
|
|
getAttackDiscoveryScheduleType({
|
|
logger: this.logger,
|
|
})
|
|
);
|
|
}
|
|
});
|
|
})
|
|
.catch((error) => {
|
|
this.logger.error(`error in security assistant plugin setup: ${error}`);
|
|
});
|
|
|
|
return {
|
|
actions: plugins.actions,
|
|
getRegisteredFeatures: (pluginName: string) => {
|
|
return appContextService.getRegisteredFeatures(pluginName);
|
|
},
|
|
getRegisteredTools: (pluginName: string) => {
|
|
return appContextService.getRegisteredTools(pluginName);
|
|
},
|
|
};
|
|
}
|
|
|
|
public start(
|
|
core: CoreStart,
|
|
plugins: ElasticAssistantPluginStartDependencies
|
|
): ElasticAssistantPluginStart {
|
|
this.logger.debug('elasticAssistant: Started');
|
|
appContextService.start({ logger: this.logger });
|
|
|
|
removeLegacyQuickPrompt(core.elasticsearch.client.asInternalUser)
|
|
.then((res) => {
|
|
if (res?.total)
|
|
this.logger.info(`Removed ${res.total} legacy quick prompts from AI Assistant`);
|
|
})
|
|
.catch(() => {});
|
|
|
|
return {
|
|
actions: plugins.actions,
|
|
inference: plugins.inference,
|
|
getRegisteredFeatures: (pluginName: string) => {
|
|
return appContextService.getRegisteredFeatures(pluginName);
|
|
},
|
|
getRegisteredTools: (pluginName: string) => {
|
|
return appContextService.getRegisteredTools(pluginName);
|
|
},
|
|
registerFeatures: (pluginName: string, features: Partial<AssistantFeatures>) => {
|
|
return appContextService.registerFeatures(pluginName, features);
|
|
},
|
|
registerTools: (pluginName: string, tools: AssistantTool[]) => {
|
|
return appContextService.registerTools(pluginName, tools);
|
|
},
|
|
registerCallback: (callbackId: CallbackIds, callback: Function) => {
|
|
return appContextService.registerCallback(callbackId, callback);
|
|
},
|
|
};
|
|
}
|
|
|
|
public stop() {
|
|
appContextService.stop();
|
|
this.pluginStop$.next();
|
|
this.pluginStop$.complete();
|
|
}
|
|
}
|