github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 10:40:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security solution] Add github label for evals for prompt changes (#224488)

Browse source
This commit is contained in:
Steph Milovic 2025-06-23 17:25:05 -06:00 • committed by GitHub
parent f0b862f113
commit 63134aa8eb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 20 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.buildkite/scripts/common/env.sh
View file

@ -83,6 +83,11 @@ if is_pr; then
export ELASTIC_APM_CONTEXT_PROPAGATION_ONLY=true
fi
# value for security genai prompts evals
if is_pr_with_label "ci:security-genai-run-evals-local-prompts"; then
export IS_SECURITY_AI_PROMPT_TEST=true
fi
# These can be removed once we're not supporting Jenkins and Buildkite at the same time
# These are primarily used by github checks reporter and can be configured via /github_checks_api.json
export ghprbGhRepository="elastic/kibana"

5
.buildkite/scripts/pipelines/pull_request/pipeline.ts
View file

@ -470,7 +470,10 @@ const getPipeline = (filename: string, removeSteps = true) => {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_tests.yml'));
}
if (GITHUB_PR_LABELS.includes('ci:security-genai-run-evals')) {
if (
GITHUB_PR_LABELS.includes('ci:security-genai-run-evals') ||
GITHUB_PR_LABELS.includes('ci:security-genai-run-evals-local-prompts')
) {
pipeline.push(
getPipeline('.buildkite/pipelines/pull_request/security_solution/gen_ai_evals.yml')
);

12
x-pack/test/security_solution_api_integration/test_suites/genai/evaluations/trial_license_complete_tier/evaluations.ts
View file

@ -35,6 +35,7 @@ export default ({ getService }: FtrProviderContext) => {
const es = getService('es');
const ml = getService('ml') as ReturnType<typeof MachineLearningProvider>;
const esArchiver = getService('esArchiver');
const isEvalLocalPrompts = process.env.IS_SECURITY_AI_PROMPT_TEST === 'true';
/**
* Results will be written to LangSmith for project associated with the langSmithAPIKey, then later
@ -63,6 +64,12 @@ export default ({ getService }: FtrProviderContext) => {
await esArchiver.load(
'x-pack/test/functional/es_archives/security_solution/attack_discovery_alerts'
);
// if run is to test prompt changes, uninstall prompt integration to default to local prompts
if (isEvalLocalPrompts) {
// delete integration prompt saved objects
const route = routeWithNamespace(`/api/saved_objects/epm-packages/security_ai_prompts`);
await supertest.delete(route).set('kbn-xsrf', 'foo');
}
});
after(async () => {
@ -78,9 +85,12 @@ export default ({ getService }: FtrProviderContext) => {
describe('Run Evaluations', () => {
const buildNumber = process.env.BUILDKITE_BUILD_NUMBER || os.hostname();
const prNumber = process.env.BUILDKITE_PULL_REQUEST;
const config = getSecurityGenAIConfigFromEnvVar();
const defaultEvalPayload: PostEvaluateBody = {
runName: `Eval Automation${buildNumber ? ' - ' + buildNumber : ''}`,
runName: `Eval Automation${buildNumber ? ' | Build ' + buildNumber : ''}${
prNumber ? ' | PR ' + prNumber : ''
}${isEvalLocalPrompts ? ' | [Local Prompts]' : ''}`,
graphs: ['DefaultAssistantGraph'],
datasetName: 'Sample Dataset',
connectorIds: Object.keys(config.connectors),