[Security solution] Add github label for evals for prompt changes (#224488)

.buildkite/scripts/common/env.sh

@@ -83,6 +83,11 @@ if is_pr; then
     export ELASTIC_APM_CONTEXT_PROPAGATION_ONLY=true
   fi
 
+  # value for security genai prompts evals
+  if is_pr_with_label "ci:security-genai-run-evals-local-prompts"; then
+    export IS_SECURITY_AI_PROMPT_TEST=true
+  fi
+
   # These can be removed once we're not supporting Jenkins and Buildkite at the same time
   # These are primarily used by github checks reporter and can be configured via /github_checks_api.json
   export ghprbGhRepository="elastic/kibana"

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -470,7 +470,10 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_tests.yml'));
     }
 
-    if (GITHUB_PR_LABELS.includes('ci:security-genai-run-evals')) {
+    if (
+      GITHUB_PR_LABELS.includes('ci:security-genai-run-evals') ||
+      GITHUB_PR_LABELS.includes('ci:security-genai-run-evals-local-prompts')
+    ) {
       pipeline.push(
         getPipeline('.buildkite/pipelines/pull_request/security_solution/gen_ai_evals.yml')
       );

x-pack/test/security_solution_api_integration/test_suites/genai/evaluations/trial_license_complete_tier/evaluations.ts

@@ -35,6 +35,7 @@ export default ({ getService }: FtrProviderContext) => {
   const es = getService('es');
   const ml = getService('ml') as ReturnType<typeof MachineLearningProvider>;
   const esArchiver = getService('esArchiver');
+  const isEvalLocalPrompts = process.env.IS_SECURITY_AI_PROMPT_TEST === 'true';
 
   /**
    * Results will be written to LangSmith for project associated with the langSmithAPIKey, then later
@@ -63,6 +64,12 @@ export default ({ getService }: FtrProviderContext) => {
       await esArchiver.load(
         'x-pack/test/functional/es_archives/security_solution/attack_discovery_alerts'
       );
+      // if run is to test prompt changes, uninstall prompt integration to default to local prompts
+      if (isEvalLocalPrompts) {
+        // delete integration prompt saved objects
+        const route = routeWithNamespace(`/api/saved_objects/epm-packages/security_ai_prompts`);
+        await supertest.delete(route).set('kbn-xsrf', 'foo');
+      }
     });
 
     after(async () => {
@@ -78,9 +85,12 @@ export default ({ getService }: FtrProviderContext) => {
 
     describe('Run Evaluations', () => {
       const buildNumber = process.env.BUILDKITE_BUILD_NUMBER || os.hostname();
+      const prNumber = process.env.BUILDKITE_PULL_REQUEST;
       const config = getSecurityGenAIConfigFromEnvVar();
       const defaultEvalPayload: PostEvaluateBody = {
-        runName: `Eval Automation${buildNumber ? ' - ' + buildNumber : ''}`,
+        runName: `Eval Automation${buildNumber ? ' | Build ' + buildNumber : ''}${
+          prNumber ? ' | PR ' + prNumber : ''
+        }${isEvalLocalPrompts ? ' | [Local Prompts]' : ''}`,
         graphs: ['DefaultAssistantGraph'],
         datasetName: 'Sample Dataset',
         connectorIds: Object.keys(config.connectors),