[Security Solution] GenAI API Integration Tests (#176357)

.buildkite/ftr_configs.yml

@@ -521,6 +521,8 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/ess.config.ts

.buildkite/pipelines/security_solution/api_integration.yml

@@ -146,6 +146,17 @@ steps:
             - exit_status: "1"
               limit: 2
 
+      - label: Running genai:qa:serverless
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh genai:qa:serverless
+        key: genai:qa:serverless
+        agents:
+          queue: n2-4-spot
+        timeout_in_minutes: 120
+        retry:
+          automatic:
+            - exit_status: "1"
+              limit: 2
+
       - label: Running prebuilt_rules_management:qa:serverless
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_management:qa:serverless
         key: prebuilt_rules_management:qa:serverless

.github/CODEOWNERS

@@ -1501,6 +1501,9 @@ x-pack/plugins/security_solution/public/flyout/entity_details @elastic/security-
 x-pack/plugins/security_solution/common/api/entity_analytics @elastic/security-entity-analytics
 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/risk_score @elastic/security-entity-analytics
 
+## Security Solution sub teams - GenAI
+x-pack/test/security_solution_api_integration/test_suites/genai @elastic/security-generative-ai
+
 # Security Defend Workflows - OSQuery Ownership
 /x-pack/plugins/security_solution/common/api/detection_engine/model/rule_response_actions @elastic/security-defend-workflows
 /x-pack/plugins/security_solution/public/detection_engine/rule_response_actions @elastic/security-defend-workflows

x-pack/test/security_solution_api_integration/package.json

@@ -5,6 +5,9 @@
   "private": true,
   "license": "Elastic License 2.0",
   "scripts": {
+    "initialize-server:genai": "node ./scripts/index.js server genai trial_license_complete_tier",
+    "run-tests:genai": "node ./scripts/index.js runner genai trial_license_complete_tier",
+
     "initialize-server:ea": "node ./scripts/index.js server entity_analytics trial_license_complete_tier",
     "run-tests:ea": "node ./scripts/index.js runner entity_analytics trial_license_complete_tier",
 
@@ -26,6 +29,12 @@
     "initialize-server:lists:complete": "node ./scripts/index.js server lists_and_exception_lists trial_license_complete_tier",
     "run-tests:lists:complete": "node ./scripts/index.js runner lists_and_exception_lists trial_license_complete_tier",
 
+    "genai:server:serverless": "npm run initialize-server:genai invoke_ai serverless",
+    "genai:runner:serverless": "npm run run-tests:genai invoke_ai serverless serverlessEnv",
+    "genai:qa:serverless": "npm run run-tests:genai invoke_ai serverless qaEnv",
+    "genai:server:ess": "npm run initialize-server:genai invoke_ai ess",
+    "genai:runner:ess": "npm run run-tests:genai invoke_ai ess essEnv",
+
     "entity_analytics:server:serverless": "npm run initialize-server:ea risk_engine serverless",
     "entity_analytics:runner:serverless": "npm run run-tests:ea risk_engine serverless serverlessEnv",
     "entity_analytics:qa:serverless": "npm run run-tests:ea risk_engine serverless qaEnv",

x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/basic.ts

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+
+import { BedrockSimulator } from '@kbn/actions-simulators-plugin/server/bedrock_simulation';
+import { OpenAISimulator } from '@kbn/actions-simulators-plugin/server/openai_simulation';
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+import { postActionsClientExecute } from '../utils/post_actions_client_execute';
+import { ObjectRemover } from '../utils/object_remover';
+import { createConnector } from '../utils/create_connector';
+
+const mockRequest = {
+  params: {
+    subActionParams: {
+      messages: [
+        { role: 'user', content: '\\n\\n\\n\\nWhat is my name?' },
+        {
+          role: 'assistant',
+          content:
+            "I'm sorry, but I don't have the information about your name. You can tell me your name if you'd like, and we can continue our conversation from there.",
+        },
+        { role: 'user', content: '\\n\\nMy name is Andrew' },
+        {
+          role: 'assistant',
+          content: "Hello, Andrew! It's nice to meet you. What would you like to talk about today?",
+        },
+        { role: 'user', content: '\\n\\nDo you know my name?' },
+      ],
+    },
+    subAction: 'invokeAI',
+  },
+  isEnabledKnowledgeBase: false,
+  isEnabledRAGAlerts: false,
+  llmType: 'bedrock',
+};
+
+export default ({ getService }: FtrProviderContext) => {
+  const supertest = getService('supertest');
+  const objectRemover = new ObjectRemover(supertest);
+  const configService = getService('config');
+
+  // @skipInQA tag because the simulators do not work in the QA env
+  describe('@ess @serverless @skipInQA Basic Security AI Assistant Invoke AI [non-streaming, non-LangChain]', async () => {
+    after(() => {
+      objectRemover.removeAll();
+    });
+
+    describe('With Bedrock connector', () => {
+      const simulator = new BedrockSimulator({
+        proxy: {
+          config: configService.get('kbnTestServer.serverArgs'),
+        },
+      });
+      let apiUrl: string;
+      let bedrockActionId: string;
+
+      before(async () => {
+        apiUrl = await simulator.start();
+        bedrockActionId = await createConnector(supertest, objectRemover, apiUrl, 'bedrock');
+      });
+
+      after(() => {
+        simulator.close();
+      });
+      it('should execute a chat completion', async () => {
+        const response = await postActionsClientExecute(bedrockActionId, mockRequest, supertest);
+
+        const expected = {
+          connector_id: bedrockActionId,
+          data: 'Hello there! How may I assist you today?',
+          status: 'ok',
+        };
+
+        expect(response.body).to.eql(expected);
+      });
+    });
+
+    describe('With OpenAI connector', () => {
+      const simulator = new OpenAISimulator({
+        returnError: false,
+        proxy: {
+          config: configService.get('kbnTestServer.serverArgs'),
+        },
+      });
+      let apiUrl: string;
+      let openaiActionId: string;
+
+      before(async () => {
+        apiUrl = await simulator.start();
+        openaiActionId = await createConnector(supertest, objectRemover, apiUrl, 'openai');
+      });
+
+      after(() => {
+        simulator.close();
+      });
+      it('should execute a chat completion', async () => {
+        const response = await postActionsClientExecute(
+          openaiActionId,
+          { ...mockRequest, llmType: 'openai' },
+          supertest
+        );
+
+        const expected = {
+          connector_id: openaiActionId,
+          data: 'Hello there! How may I assist you today?',
+          status: 'ok',
+        };
+
+        expect(response.body).to.eql(expected);
+      });
+    });
+  });
+};

x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/ess.config.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import getPort from 'get-port';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(
+    require.resolve('../../../../../config/ess/config.base.trial')
+  );
+
+  const proxyPort = await getPort({ port: getPort.makeRange(6200, 6299) });
+
+  return {
+    ...functionalConfig.getAll(),
+    kbnTestServer: {
+      ...functionalConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...functionalConfig.get('kbnTestServer.serverArgs'),
+        // used for connector simulators
+        `--xpack.actions.proxyUrl=http://localhost:${proxyPort}`,
+        `--xpack.actions.enabledActionTypes=${JSON.stringify(['.bedrock', '.gen-ai'])}`,
+      ],
+    },
+    testFiles: [require.resolve('..')],
+    junit: {
+      reportName: 'GenAI - Invoke AI Tests - ESS Env - Trial License',
+    },
+  };
+}

x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/serverless.config.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createTestConfig } from '../../../../../config/serverless/config.base';
+
+export default createTestConfig({
+  kbnTestServerArgs: [
+    // used for connector simulators
+    `--xpack.actions.proxyUrl=http://localhost:6200`,
+  ],
+  testFiles: [require.resolve('..')],
+  junit: {
+    reportName: 'GenAI - Invoke AI Tests - Serverless Env - Complete Tier',
+  },
+});

x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  // this is the test suite for the inaptly named post_actions_connector_execute route
+  describe('GenAI - Invoke AI', function () {
+    loadTestFile(require.resolve('./basic'));
+  });
+}

x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/create_connector.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type SuperTest from 'supertest';
+import {
+  ELASTIC_HTTP_VERSION_HEADER,
+  X_ELASTIC_INTERNAL_ORIGIN_REQUEST,
+} from '@kbn/core-http-common';
+import { getUrlPrefix } from './space_test_utils';
+import { ObjectRemover } from './object_remover';
+
+const connectorSetup = {
+  bedrock: {
+    connectorTypeId: '.bedrock',
+    name: 'A bedrock action',
+    secrets: {
+      accessKey: 'bedrockAccessKey',
+      secret: 'bedrockSecret',
+    },
+    config: {
+      defaultModel: 'anthropic.claude-v2',
+    },
+  },
+  openai: {
+    connectorTypeId: '.gen-ai',
+    name: 'An openai action',
+    secrets: {
+      apiKey: 'genAiApiKey',
+    },
+    config: {
+      apiProvider: 'OpenAI',
+    },
+  },
+};
+
+/**
+ * Creates a connector
+ * @param supertest The supertest agent.
+ * @param apiUrl The url of the api
+ * @param connectorType The type of connector to create
+ * @param spaceId The space id
+ */
+export const createConnector = async (
+  supertest: SuperTest.SuperTest<SuperTest.Test>,
+  objectRemover: ObjectRemover,
+  apiUrl: string,
+  connectorType: 'bedrock' | 'openai',
+  spaceId?: string
+) => {
+  const { connectorTypeId, config, name, secrets } = connectorSetup[connectorType];
+  const result = await supertest
+    .post(`${getUrlPrefix(spaceId ?? 'default')}/api/actions/connector`)
+    .set('kbn-xsrf', 'foo')
+    .set(ELASTIC_HTTP_VERSION_HEADER, '1')
+    .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
+    .send({
+      name,
+      connector_type_id: connectorTypeId,
+      config: { ...config, apiUrl },
+      secrets,
+    })
+    .expect(200);
+
+  const { body } = result;
+
+  objectRemover.add(spaceId ?? 'default', body.id, 'connector', 'actions');
+
+  return body.id;
+};

x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/object_remover.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ELASTIC_HTTP_VERSION_HEADER,
+  X_ELASTIC_INTERNAL_ORIGIN_REQUEST,
+} from '@kbn/core-http-common';
+import { getUrlPrefix } from './space_test_utils';
+
+interface ObjectToRemove {
+  spaceId: string;
+  id: string;
+  type: string;
+  plugin: string;
+  isInternal?: boolean;
+}
+
+export class ObjectRemover {
+  private readonly supertest: any;
+  private objectsToRemove: ObjectToRemove[] = [];
+
+  constructor(supertest: any) {
+    this.supertest = supertest;
+  }
+
+  add(
+    spaceId: ObjectToRemove['spaceId'],
+    id: ObjectToRemove['id'],
+    type: ObjectToRemove['type'],
+    plugin: ObjectToRemove['plugin'],
+    isInternal?: ObjectToRemove['isInternal']
+  ) {
+    this.objectsToRemove.push({ spaceId, id, type, plugin, isInternal });
+  }
+
+  async removeAll() {
+    await Promise.all(
+      this.objectsToRemove.map(({ spaceId, id, type, plugin, isInternal }) => {
+        return this.supertest
+          .delete(
+            `${getUrlPrefix(spaceId)}/${isInternal ? 'internal' : 'api'}/${plugin}/${type}/${id}`
+          )
+          .set('kbn-xsrf', 'foo')
+          .set(ELASTIC_HTTP_VERSION_HEADER, '1')
+          .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
+          .expect(plugin === 'saved_objects' ? 200 : 204);
+      })
+    );
+    this.objectsToRemove = [];
+  }
+}

x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/post_actions_client_execute.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type SuperTest from 'supertest';
+import {
+  ELASTIC_HTTP_VERSION_HEADER,
+  X_ELASTIC_INTERNAL_ORIGIN_REQUEST,
+} from '@kbn/core-http-common';
+import { Response } from 'superagent';
+
+/**
+ * Executes an invoke AI action
+ * @param connectorId The connector id
+ * @param args The arguments to pass to the action
+ * @param supertest The supertest agent
+ */
+export const postActionsClientExecute = async (
+  connectorId: string,
+  args: any,
+  supertest: SuperTest.SuperTest<SuperTest.Test>
+): Promise<Response> => {
+  const response = await supertest
+    .post(`/internal/elastic_assistant/actions/connector/${connectorId}/_execute`)
+    .set('kbn-xsrf', 'true')
+    .set(ELASTIC_HTTP_VERSION_HEADER, '1')
+    .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
+    .send(args);
+
+  return response;
+};

x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/utils/space_test_utils.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export function getUrlPrefix(spaceId: string) {
+  return spaceId && spaceId !== 'default' ? `/s/${spaceId}` : ``;
+}