mirror of
https://github.com/elastic/kibana.git
synced 2025-06-27 18:51:07 -04:00
[Security Assistant] Simplifies Security Gen AI Evaluation secret management (#219885)
## Summary Simplifies secret management for running the Security Gen AI Evaluations. See updated README.md for full details, but includes: * Consolidation of multiple vault keys to a single `KIBANA_SECURITY_GEN_AI_CONFIG` key, which contains all connectors, langsmith creds and now a way to specify `evaluatorConnectorId`. * Added `vault` params to both `retrieve_secrets.js` and `upload_secrets.js` for specifying the vault. Defaults to `sieam-team` secrets.elastic.co for ease of use by developers. * Introduces `get_commands.js` script for fetching commands to hand off to either Kibana Ops for updating, or specifying config overrides when manually running BuildKite pipelines. * Deleted `export_env_secrets.js` as it couldn't be used for setting env vars locally for the dev testing experience. * Updated `connectors` as per team discussion to include: GPT-4.1, Claude 3.5/3.7, and Gemini 2.5 Pro. This was a config change made by Kibana Ops, so no code change present. But you can confirm by running `retrieve_secrets.js`. And finally, a much more detailed `README.md` for testing locally, on PR's and CI, and the process for updating secrets. See full [README.md](https://github.com/spong/kibana/blob/ci-eval-tweaks/x-pack/test/security_solution_api_integration/test_suites/genai/evaluations/README.md) Example LangSmith Runs: * `ES|QL Generation Regression Suite`: [Run 298372](261dcc59-fbe7-4397-a662-ff94042f666c) * `Alerts RAG Regression (Episodes 1-8)`: [Run 298372](bd5bba1d-97aa-4512-bce7-b09aa943c651) * `Assistant Eval: Custom Knowledge`: [Run 298372](2d5f7c18-4bf4-4cdb-97a1-16e39a865cab) * `Eval AD: All Scenarios`: [Run 300138](4690ee16-9df5-416c-8bf0-b62bc2f2aba9/compare?selectedSessions=6d44134b-6492-4f2d-9b28-6d4a82a0e9ae&baseline=undefined) Note: there is currently a timing bug with Alerts/KB entries being cleaned up before the server is complete, so you may see poor evals for `Alerts RAG Regression (Episodes 1-8)` and `Assistant Eval: Custom Knowledge` until that is fixed. I'll address this in a follow-up PR since it is unrelated to this change-set.
This commit is contained in:
Garrett Spong
GitHub
parent
04f60148aa
commit
e9a8909fad
10 changed files with 245 additions and 147 deletions
|
|
@ -144,8 +144,7 @@ EOF
|
|||
{
|
||||
if [[ "${FTR_SECURITY_GEN_AI:-}" =~ ^(1|true)$ ]]; then
|
||||
echo "FTR_SECURITY_GEN_AI was set - exposing LLM connectors"
|
||||
export KIBANA_SECURITY_TESTING_AI_CONNECTORS="$(vault_get security-gen-ai/connectors config)"
|
||||
export KIBANA_SECURITY_TESTING_LANGSMITH_KEY="$(vault_get security-gen-ai/langsmith key)"
|
||||
export KIBANA_SECURITY_GEN_AI_CONFIG="$(vault_get security-gen-ai config)"
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
connector_config.json
|
||||
langsmith_key.txt
|
||||
config.json
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
require('@kbn/babel-register').install();
|
||||
require('./manage_secrets').exportToEnvVars();
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
require('@kbn/babel-register').install();
|
||||
const { getCommand } = require('./manage_secrets');
|
||||
const minimist = require('minimist');
|
||||
|
||||
/**
|
||||
* Gets a command for working with Security Gen AI secrets either as a vault write command or environment variable.
|
||||
* By default, the command is formatted as a 'vault-write' command, but it can be overridden with the --format parameter
|
||||
* to use 'env-var' format.
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function run() {
|
||||
const argv = minimist(process.argv.slice(2));
|
||||
const format = argv.format || 'vault-write';
|
||||
const vault = argv.vault || 'ci-prod';
|
||||
|
||||
if (format !== 'vault-write' && format !== 'env-var') {
|
||||
console.error('Error: format parameter must be either "vault-write" or "env-var"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (format === 'vault-write' && vault !== 'siem-team' && vault !== 'ci-prod') {
|
||||
console.error('Error: vault parameter must be either "siem-team" or "ci-prod"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(await getCommand(format, vault));
|
||||
}
|
||||
|
||||
run();
|
||||
|
|
@ -11,36 +11,42 @@ import { writeFile, readFile } from 'fs/promises';
|
|||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import { schema } from '@kbn/config-schema';
|
||||
|
||||
const SECURITY_GEN_AI_CONNECTORS_ENV_VAR = 'KIBANA_SECURITY_TESTING_AI_CONNECTORS';
|
||||
const SECURITY_GEN_AI_LANGSMITH_KEY_ENV_VAR = 'KIBANA_SECURITY_TESTING_LANGSMITH_KEY';
|
||||
// Environment variable set within BuildKite and read from in FTR tests
|
||||
// CI env vars are set by .buildkite/scripts/common/setup_job_env.sh
|
||||
const KIBANA_SECURITY_GEN_AI_CONFIG = 'KIBANA_SECURITY_GEN_AI_CONFIG';
|
||||
|
||||
// siem-team secrets discussed w/ operations and we will mirror them here
|
||||
// const SECURITY_GEN_AI_VAULT = 'secret/siem-team/security-gen-ai';
|
||||
// Vault paths
|
||||
// siem-team users (secrets.elastic.co vault) do not have access to the ci-prod vault, so secrets
|
||||
// are mirrored between the two vaults
|
||||
type VaultType = 'siem-team' | 'ci-prod';
|
||||
const VAULT_PATHS: Record<VaultType, string> = {
|
||||
'siem-team': 'secret/siem-team/security-gen-ai',
|
||||
'ci-prod': 'secret/ci/elastic-kibana/security-gen-ai',
|
||||
};
|
||||
|
||||
// CI Vault
|
||||
const SECURITY_GEN_AI_VAULT = 'secret/ci/elastic-kibana/security-gen-ai';
|
||||
const SECURITY_GEN_AI_VAULT_CONNECTORS = `${SECURITY_GEN_AI_VAULT}/connectors`;
|
||||
const SECURITY_GEN_AI_VAULT_LANGSMITH = `${SECURITY_GEN_AI_VAULT}/langsmith`;
|
||||
const SECURITY_GEN_AI_CONNECTORS_FIELD = 'config';
|
||||
const SECURITY_GEN_AI_LANGSMITH_FIELD = 'key';
|
||||
const CONNECTOR_FILE = Path.join(
|
||||
const getVaultPath = (vault: VaultType = 'siem-team') => {
|
||||
return VAULT_PATHS[vault];
|
||||
};
|
||||
|
||||
const SECURITY_GEN_AI_CONFIG_FIELD = 'config';
|
||||
const SECURITY_GEN_AI_CONFIG_FILE = Path.join(
|
||||
REPO_ROOT,
|
||||
'x-pack/test/security_solution_api_integration/scripts/genai/vault/connector_config.json'
|
||||
);
|
||||
const LANGSMITH_FILE = Path.join(
|
||||
REPO_ROOT,
|
||||
'x-pack/test/security_solution_api_integration/scripts/genai/vault/langsmith_key.txt'
|
||||
'x-pack/test/security_solution_api_integration/scripts/genai/vault/config.json'
|
||||
);
|
||||
|
||||
const connectorsSchema = schema.recordOf(
|
||||
schema.string(),
|
||||
schema.object({
|
||||
name: schema.string(),
|
||||
actionTypeId: schema.string(),
|
||||
config: schema.recordOf(schema.string(), schema.any()),
|
||||
secrets: schema.recordOf(schema.string(), schema.any()),
|
||||
})
|
||||
);
|
||||
const configSchema = schema.object({
|
||||
evaluatorConnectorId: schema.string(),
|
||||
langsmithKey: schema.string(),
|
||||
connectors: schema.recordOf(
|
||||
schema.string(),
|
||||
schema.object({
|
||||
name: schema.string(),
|
||||
actionTypeId: schema.string(),
|
||||
config: schema.recordOf(schema.string(), schema.any()),
|
||||
secrets: schema.recordOf(schema.string(), schema.any()),
|
||||
})
|
||||
),
|
||||
});
|
||||
|
||||
export interface AvailableConnector {
|
||||
name: string;
|
||||
|
|
@ -49,50 +55,75 @@ export interface AvailableConnector {
|
|||
secrets: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export const retrieveFromVault = async (
|
||||
vault: string,
|
||||
filePath: string,
|
||||
field: string,
|
||||
isJson = true
|
||||
) => {
|
||||
/**
|
||||
* Retrieve generic value from vault and write to file
|
||||
*
|
||||
* @param vault
|
||||
* @param filePath
|
||||
* @param field
|
||||
*/
|
||||
export const retrieveFromVault = async (vault: string, filePath: string, field: string) => {
|
||||
const { stdout } = await execa('vault', ['read', `-field=${field}`, vault], {
|
||||
cwd: REPO_ROOT,
|
||||
buffer: true,
|
||||
});
|
||||
|
||||
const value = Buffer.from(stdout, 'base64').toString('utf-8').trim();
|
||||
const config = isJson ? JSON.stringify(JSON.parse(value), null, 2) : value;
|
||||
const config = JSON.stringify(JSON.parse(value), null, 2);
|
||||
|
||||
await writeFile(filePath, config);
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Config dumped into ${filePath}`);
|
||||
console.log(`Config written to: ${filePath}`);
|
||||
};
|
||||
|
||||
export const retrieveConnectorConfig = async () => {
|
||||
/**
|
||||
* Retrieve Security Gen AI secrets config from vault and write to file
|
||||
* @param vault
|
||||
*/
|
||||
export const retrieveConfigFromVault = async (vault: VaultType = 'siem-team') => {
|
||||
await retrieveFromVault(
|
||||
SECURITY_GEN_AI_VAULT_CONNECTORS,
|
||||
CONNECTOR_FILE,
|
||||
SECURITY_GEN_AI_CONNECTORS_FIELD
|
||||
getVaultPath(vault),
|
||||
SECURITY_GEN_AI_CONFIG_FILE,
|
||||
SECURITY_GEN_AI_CONFIG_FIELD
|
||||
);
|
||||
};
|
||||
|
||||
export const retrieveLangsmithKey = async () => {
|
||||
await retrieveFromVault(
|
||||
SECURITY_GEN_AI_VAULT_LANGSMITH,
|
||||
LANGSMITH_FILE,
|
||||
SECURITY_GEN_AI_LANGSMITH_FIELD,
|
||||
false
|
||||
);
|
||||
};
|
||||
/**
|
||||
* Returns command for manually working with secrets from `config.json`.
|
||||
* Format can be either 'vault-write' (for vault command) or 'env-var' (for environment variable).
|
||||
* Run this command and share with @kibana-ops via https://p.elstc.co to make updating secrets easier, or for pasting
|
||||
* custom configs into the BuildKite pipeline: https://buildkite.com/elastic/kibana-ess-security-solution-gen-ai-evals
|
||||
|
||||
export const formatCurrentConfig = async (filePath: string) => {
|
||||
const config = await readFile(filePath, 'utf-8');
|
||||
* Alternatively, have @kibana-ops run the following to update the secrets for CI:
|
||||
*
|
||||
* node retrieve_secrets.js --vault siem-team
|
||||
* node upload_secrets.js --vault ci-prod
|
||||
*
|
||||
* @param format - The format of the command to return ('vault-write' or 'env-var')
|
||||
* @param vault - The vault to use (only applicable for 'vault-write' format)
|
||||
*/
|
||||
export const getCommand = async (
|
||||
format: 'vault-write' | 'env-var' = 'vault-write',
|
||||
vault: VaultType = 'ci-prod'
|
||||
) => {
|
||||
const config = await readFile(SECURITY_GEN_AI_CONFIG_FILE, 'utf-8');
|
||||
const asB64 = Buffer.from(config).toString('base64');
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(asB64);
|
||||
|
||||
if (format === 'vault-write') {
|
||||
return `vault write ${getVaultPath(vault)} ${SECURITY_GEN_AI_CONFIG_FIELD}=${asB64}`;
|
||||
} else {
|
||||
return `${KIBANA_SECURITY_GEN_AI_CONFIG}=${asB64}`;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Write generic value to vault from a file
|
||||
*
|
||||
* @param vault
|
||||
* @param filePath
|
||||
* @param field
|
||||
*/
|
||||
export const uploadToVault = async (vault: string, filePath: string, field: string) => {
|
||||
const config = await readFile(filePath, 'utf-8');
|
||||
const asB64 = Buffer.from(config).toString('base64');
|
||||
|
|
@ -103,69 +134,35 @@ export const uploadToVault = async (vault: string, filePath: string, field: stri
|
|||
});
|
||||
};
|
||||
|
||||
export const uploadConnectorConfigToVault = async () => {
|
||||
/**
|
||||
* Read Security Gen AI secrets from `config.json` and upload to vault
|
||||
* @param vault
|
||||
*/
|
||||
export const uploadConfigToVault = async (vault: VaultType = 'siem-team') => {
|
||||
await uploadToVault(
|
||||
SECURITY_GEN_AI_VAULT_CONNECTORS,
|
||||
CONNECTOR_FILE,
|
||||
SECURITY_GEN_AI_CONNECTORS_FIELD
|
||||
);
|
||||
};
|
||||
|
||||
export const uploadLangsmithKeyToVault = async () => {
|
||||
await uploadToVault(
|
||||
SECURITY_GEN_AI_VAULT_LANGSMITH,
|
||||
LANGSMITH_FILE,
|
||||
SECURITY_GEN_AI_LANGSMITH_FIELD
|
||||
getVaultPath(vault),
|
||||
SECURITY_GEN_AI_CONFIG_FILE,
|
||||
SECURITY_GEN_AI_CONFIG_FIELD
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* FOR LOCAL USE ONLY! Export connectors and langsmith secrets from vault to env vars before manually
|
||||
* running evaluations. CI env vars are set by .buildkite/scripts/common/setup_job_env.sh
|
||||
* Returns parsed config from environment variable
|
||||
*/
|
||||
export const exportToEnvVars = async () => {
|
||||
const { stdout: connectors } = await execa(
|
||||
'vault',
|
||||
['read', `-field=${SECURITY_GEN_AI_CONNECTORS_FIELD}`, SECURITY_GEN_AI_VAULT_CONNECTORS],
|
||||
{
|
||||
cwd: REPO_ROOT,
|
||||
buffer: true,
|
||||
}
|
||||
);
|
||||
const { stdout: langsmithKey } = await execa(
|
||||
'vault',
|
||||
['read', `-field=${SECURITY_GEN_AI_LANGSMITH_FIELD}`, SECURITY_GEN_AI_VAULT_LANGSMITH],
|
||||
{
|
||||
cwd: REPO_ROOT,
|
||||
buffer: true,
|
||||
}
|
||||
);
|
||||
process.env[SECURITY_GEN_AI_CONNECTORS_ENV_VAR] = connectors;
|
||||
process.env[SECURITY_GEN_AI_LANGSMITH_KEY_ENV_VAR] = langsmithKey;
|
||||
};
|
||||
|
||||
export const loadConnectorsFromEnvVar = (): Record<string, AvailableConnector> => {
|
||||
const connectorsValue = process.env[SECURITY_GEN_AI_CONNECTORS_ENV_VAR];
|
||||
if (!connectorsValue) {
|
||||
return {};
|
||||
export const getSecurityGenAIConfigFromEnvVar = () => {
|
||||
const configValue = process.env[KIBANA_SECURITY_GEN_AI_CONFIG];
|
||||
if (!configValue) {
|
||||
throw new Error(`Environment variable ${KIBANA_SECURITY_GEN_AI_CONFIG} does not exist!`);
|
||||
}
|
||||
|
||||
let connectors: Record<string, AvailableConnector>;
|
||||
let config: typeof configSchema;
|
||||
try {
|
||||
connectors = JSON.parse(Buffer.from(connectorsValue, 'base64').toString('utf-8'));
|
||||
config = JSON.parse(Buffer.from(configValue, 'base64').toString('utf-8'));
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Error trying to parse value from ${SECURITY_GEN_AI_CONNECTORS_ENV_VAR} environment variable: ${e.message}`
|
||||
`Error trying to parse value from ${KIBANA_SECURITY_GEN_AI_CONFIG} environment variable: ${e.message}`
|
||||
);
|
||||
}
|
||||
return connectorsSchema.validate(connectors);
|
||||
};
|
||||
|
||||
export const loadLangSmithKeyFromEnvVar = (): string | undefined => {
|
||||
const langsmithKeyValue = process.env[SECURITY_GEN_AI_LANGSMITH_KEY_ENV_VAR];
|
||||
if (!langsmithKeyValue) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return Buffer.from(langsmithKeyValue, 'base64').toString('utf-8').trim();
|
||||
return configSchema.validate(config);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,11 +6,27 @@
|
|||
*/
|
||||
|
||||
require('@kbn/babel-register').install();
|
||||
const { retrieveConnectorConfig, retrieveLangsmithKey } = require('./manage_secrets');
|
||||
const { retrieveConfigFromVault } = require('./manage_secrets');
|
||||
const minimist = require('minimist');
|
||||
|
||||
async function retrieveConfigs() {
|
||||
await retrieveConnectorConfig();
|
||||
await retrieveLangsmithKey();
|
||||
/**
|
||||
* Retrieves Security Gen AI secrets for testing from vault. By default, the 'siem-team' accessible
|
||||
* vault from secrets.elastic.co is used, but it can be overridden with the --vault parameter to use
|
||||
* the 'ci-prod' vault.
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function retrieveSecrets() {
|
||||
const argv = minimist(process.argv.slice(2));
|
||||
const vault = argv.vault || 'siem-team';
|
||||
|
||||
if (vault !== 'siem-team' && vault !== 'ci-prod') {
|
||||
console.error('Error: vault parameter must be either "siem-team" or "ci-prod"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Using ${vault} vault...`);
|
||||
await retrieveConfigFromVault(vault);
|
||||
}
|
||||
|
||||
retrieveConfigs();
|
||||
retrieveSecrets();
|
||||
|
|
|
|||
|
|
@ -6,12 +6,27 @@
|
|||
*/
|
||||
|
||||
require('@kbn/babel-register').install();
|
||||
const { uploadConfigToVault } = require('./manage_secrets');
|
||||
const minimist = require('minimist');
|
||||
|
||||
const { uploadConnectorConfigToVault, uploadLangsmithKeyToVault } = require('./manage_secrets');
|
||||
/**
|
||||
* Uploads Security Gen AI secrets for testing from local `config.json` to vault. By default, the 'siem-team' accessible
|
||||
* vault from secrets.elastic.co is used, but it can be overridden with the --vault parameter to use the 'ci-prod' vault.
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function uploadSecrets() {
|
||||
const argv = minimist(process.argv.slice(2));
|
||||
const vault = argv.vault || 'siem-team';
|
||||
|
||||
async function uploadConfigs() {
|
||||
await uploadConnectorConfigToVault();
|
||||
await uploadLangsmithKeyToVault();
|
||||
if (vault !== 'siem-team' && vault !== 'ci-prod') {
|
||||
console.error('Error: vault parameter must be either "siem-team" or "ci-prod"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Using ${vault} vault...`);
|
||||
await uploadConfigToVault(vault);
|
||||
console.log(`Secret upload complete!`);
|
||||
}
|
||||
|
||||
uploadConfigs();
|
||||
uploadSecrets();
|
||||
|
|
|
|||
|
|
@ -1,39 +1,82 @@
|
|||
## Summary
|
||||
|
||||
Introduces a new `security_solution/gen_ai_evals.yml` BuildKite pipeline for automatically running our Assistant and Attack Discovery evaluation suites weekly.
|
||||
This FTR suite is for running the Security GenAI Assistant and Attack Discovery evaluation suites. Evaluations can either be run locally, or on CI by adding the `ci:security-genai-run-evals` GitHub label to a PR. CI evaluations are also run weekly by means of the `kibana-ess-security-solution-gen-ai-evals` BuildKite pipeline (located in `security_solution/gen_ai_evals.yml`), and can also be manually triggered from the [pipeline](https://buildkite.com/elastic/kibana-ess-security-solution-gen-ai-evals) directly on BuildKite.
|
||||
|
||||
Most pre-requisites for running the evaluations are managed for you. Connector/LangSmith secrets are stored in vault, and managed via the scripts in `x-pack/test/security_solution_api_integration/scripts/genai/vault`. They are then read from ENV variables at test-time. Data pre-requisites are managed on test setup and include the installation of ptTinyElser, setup of the Knowledge Base, and ingestion of Attack Discovery alerts and KB entries.
|
||||
|
||||
> [!NOTE]
|
||||
> In discussion with @elastic/kibana-operations it was preferred to use the ci-prod vault for which we do not have access. so they are also mirrored to the `secrets.elastic.co` vault which can be modified via manage_secrets.ts and surrounding scripts so we can self-manage to a degree.
|
||||
|
||||
### To Run Locally:
|
||||
Ensure you are authenticated with vault for LLM + LangSmith creds:
|
||||
|
||||
All commands can be run from security test root:
|
||||
|
||||
```
|
||||
cd x-pack/test/security_solution_api_integration
|
||||
```
|
||||
|
||||
Ensure you are authenticated with vault for Connector + LangSmith creds:
|
||||
|
||||
> See [internal docs](https://github.com/elastic/infra/blob/master/docs/vault/README.md#login-with-your-okta) for setup/login instructions.
|
||||
|
||||
Fetch Connectors and LangSmith creds:
|
||||
|
||||
> [!NOTE]
|
||||
> In discussion with @elastic/kibana-operations it was preferred to use the ci-prod vault, but they are currently mirrored to `SECURITY_GEN_AI_VAULT` which can be modified manage_secrets.ts so we can self-manage to a degree.
|
||||
Fetch config, which includes Connectors and LangSmith creds:
|
||||
|
||||
```
|
||||
cd x-pack/test/security_solution_api_integration
|
||||
node scripts/genai/vault/retrieve_secrets.js
|
||||
node scripts/genai/vault/retrieve_secrets
|
||||
```
|
||||
|
||||
|
||||
Navigate to api integration directory, load the env vars, and start server:
|
||||
Load the env vars, and start server:
|
||||
```
|
||||
cd x-pack/test/security_solution_api_integration
|
||||
export KIBANA_SECURITY_TESTING_AI_CONNECTORS=$(base64 -w 0 < scripts/genai/vault/connector_config.json) && export KIBANA_SECURITY_TESTING_LANGSMITH_KEY=$(base64 -w 0 < scripts/genai/vault/langsmith_key.txt)
|
||||
export KIBANA_SECURITY_GEN_AI_CONFIG=$(base64 -w 0 < scripts/genai/vault/config.json)
|
||||
yarn genai_evals:server:ess
|
||||
```
|
||||
|
||||
Then in another terminal, load vars and run the tests:
|
||||
```
|
||||
cd x-pack/test/security_solution_api_integration
|
||||
export KIBANA_SECURITY_TESTING_AI_CONNECTORS=$(base64 -w 0 < scripts/genai/vault/connector_config.json) && export KIBANA_SECURITY_TESTING_LANGSMITH_KEY=$(base64 -w 0 < scripts/genai/vault/langsmith_key.txt)
|
||||
export KIBANA_SECURITY_GEN_AI_CONFIG=$(base64 -w 0 < scripts/genai/vault/config.json)
|
||||
yarn genai_evals:runner:ess
|
||||
```
|
||||
|
||||
### To manually run on BuildKite:
|
||||
Navigate to [BuildKite](https://buildkite.com/elastic?filter=ftr-security-solution-gen-ai-evaluations) and run `ftr-security-solution-gen-ai-evaluations` pipeline.
|
||||
Navigate to [BuildKite](https://buildkite.com/elastic/kibana-ess-security-solution-gen-ai-evals) and run `ftr-security-solution-gen-ai-evaluations` pipeline. If you want to run with a custom config, first modify `x-pack/test/security_solution_api_integration/scripts/genai/vault/config.json` and then run:
|
||||
|
||||
```
|
||||
node scripts/genai/vault/get_command --format env-var
|
||||
```
|
||||
|
||||
which can then be pasted into `Environment Variables` section of the BuildKite pipeline. This is helpful for running evals just against a specific model or to change the evaluator model.
|
||||
|
||||
### To manually run on BuildKite for specific PR:
|
||||
Add `ci:security-genai-run-evals` label to PR
|
||||
|
||||
### To update secrets
|
||||
|
||||
As mentioned above, secrets are mirrored between two different vaults since access differs between local development and CI. If you need to modify either the list of connectors, the LangSmith API key, or the preferred evaluatorConnectorId, perform the following steps:
|
||||
|
||||
Navigate to the test directory and fetch the latest secrets from our `siem-team` vault:
|
||||
|
||||
```
|
||||
cd x-pack/test/security_solution_api_integration
|
||||
node scripts/genai/vault/retrieve_secrets
|
||||
```
|
||||
|
||||
Modify `x-pack/test/security_solution_api_integration/scripts/genai/vault/config.json` accordingly.
|
||||
|
||||
Then, run the following command to upload the secrets back to the `siem-team` vault:
|
||||
|
||||
```
|
||||
node scripts/genai/vault/upload_secrets --vault siem-team
|
||||
```
|
||||
|
||||
Then finally, you must contact @elastic/kibana-operations and have them upload the secrets to the `ci-prod` vault. For this you can either have them run the following commands:
|
||||
|
||||
```
|
||||
node scripts/genai/vault/retrieve_secrets.js --vault siem-team
|
||||
node scripts/genai/vault/upload_secrets.js --vault ci-prod
|
||||
```
|
||||
|
||||
Or you can run the below command and paste the results into https://p.elstc.co and share the link with them to make updating secrets a little easier:
|
||||
|
||||
```
|
||||
node scripts/genai/vault/get_command --format vault-write --vault ci-prod
|
||||
```
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { FtrConfigProviderContext } from '@kbn/test';
|
||||
import { loadConnectorsFromEnvVar } from '../../../../../scripts/genai/vault/manage_secrets';
|
||||
import { getSecurityGenAIConfigFromEnvVar } from '../../../../../scripts/genai/vault/manage_secrets';
|
||||
import { getTinyElserServerArgs } from '../../../knowledge_base/entries/utils/helpers';
|
||||
|
||||
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
||||
|
|
@ -14,7 +14,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
|||
require.resolve('../../../../../config/ess/config.base.trial')
|
||||
);
|
||||
|
||||
const preconfiguredConnectors = loadConnectorsFromEnvVar();
|
||||
const preconfiguredConnectors = getSecurityGenAIConfigFromEnvVar().connectors;
|
||||
|
||||
return {
|
||||
...functionalConfig.getAll(),
|
||||
|
|
|
|||
|
|
@ -11,10 +11,8 @@ import {
|
|||
PostEvaluateBody,
|
||||
} from '@kbn/elastic-assistant-common';
|
||||
import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common';
|
||||
import {
|
||||
loadConnectorsFromEnvVar,
|
||||
loadLangSmithKeyFromEnvVar,
|
||||
} from '../../../../scripts/genai/vault/manage_secrets';
|
||||
import os from 'os';
|
||||
import { getSecurityGenAIConfigFromEnvVar } from '../../../../scripts/genai/vault/manage_secrets';
|
||||
import { FtrProviderContext } from '../../../../ftr_provider_context';
|
||||
|
||||
import {
|
||||
|
|
@ -76,20 +74,21 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
});
|
||||
|
||||
describe('Run Evaluations', () => {
|
||||
const buildNumber = process.env.BUILDKITE_BUILD_NUMBER;
|
||||
const buildNumber = process.env.BUILDKITE_BUILD_NUMBER || os.hostname();
|
||||
const config = getSecurityGenAIConfigFromEnvVar();
|
||||
const defaultEvalPayload: PostEvaluateBody = {
|
||||
runName: `Eval Automation${buildNumber ? ' - ' + buildNumber : ''}`,
|
||||
graphs: ['DefaultAssistantGraph'],
|
||||
datasetName: 'Sample Dataset',
|
||||
connectorIds: Object.keys(loadConnectorsFromEnvVar()),
|
||||
evaluatorConnectorId: 'gpt-4o',
|
||||
connectorIds: Object.keys(config.connectors),
|
||||
evaluatorConnectorId: config.evaluatorConnectorId,
|
||||
alertsIndexPattern: '.alerts-security.alerts-default',
|
||||
replacements: {},
|
||||
screenContext: {
|
||||
timeZone: 'America/Denver',
|
||||
},
|
||||
size: 100,
|
||||
langSmithApiKey: loadLangSmithKeyFromEnvVar(),
|
||||
langSmithApiKey: config.langsmithKey,
|
||||
};
|
||||
|
||||
describe('Security Assistant', () => {
|
||||
|
|
@ -147,7 +146,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
const evalPayload: PostEvaluateBody = {
|
||||
...defaultEvalPayload,
|
||||
graphs: ['DefaultAttackDiscoveryGraph'],
|
||||
datasetName: 'Attack Discovery: Episode 1',
|
||||
datasetName: 'Eval AD: All Scenarios',
|
||||
};
|
||||
const route = routeWithNamespace(ELASTIC_AI_ASSISTANT_EVALUATE_URL);
|
||||
await supertest
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue