[Security Solution] [Elastic AI Assistant] Adds internal Get Evaluate API and migrates Post Evaluate API to OAS (#175338)

## Summary In https://github.com/elastic/kibana/pull/174317 we added support for OpenAPI codegen, this PR builds on that functionality by migrating the `Post Evaluate` route `/internal/elastic_assistant/evaluate` to be backed by an OAS, and adds a basic `Get Evaluate` route for rounding out the enhancements outlined in https://github.com/elastic/security-team/issues/8167 (to be in a subsequent PR). Changes include: * Migration of `Post Evaluate` route to OAS * Migration of `Post Evaluate` route to use versioned router * Extracted `evaluate` API calls from `x-pack/packages/kbn-elastic-assistant/impl/assistant/api/api.tsx` to `x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx` * Co-located relevant `use_perform_evaluation` hook * Adds `Get Evaluate` route, and corresponding `use_evaluation_data` hook. Currently only returns `agentExecutors` to be selected for evaluation. * API versioning constants added to `x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts` * Adds new `buildRouteValidationWithZod` function to `x-pack/plugins/elastic_assistant/server/schemas/common.ts` for validating routes against OAS generated zod schemas. ### Checklist Delete any items that are not applicable to this PR. - [X] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [X] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
2025-04-24 01:38:56 -04:00 · 2024-01-30 16:03:06 -07:00 · 2024-01-30 16:03:06 -07:00 · 38f0a7aa46
commit 38f0a7aa46
parent 3b4c0d2466
29 changed files with 893 additions and 417 deletions
--- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts
+++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts
@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get Evaluate API endpoint
+ *   version: 1
+ */
+
+export type GetEvaluateResponse = z.infer<typeof GetEvaluateResponse>;
+export const GetEvaluateResponse = z.object({
+  agentExecutors: z.array(z.string()),
+});
--- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml
+++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml
@ -0,0 +1,40 @@
+openapi: 3.0.0
+info:
+  title: Get Evaluate API endpoint
+  version: '1'
+paths:
+  /internal/elastic_assistant/evaluate:
+    get:
+      operationId: GetEvaluate
+      x-codegen-enabled: true
+      description: Get relevant data for performing an evaluation like available sample data, agents, and evaluators
+      summary: Get relevant data for performing an evaluation
+      tags:
+        - Evaluation API
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  agentExecutors:
+                    type: array
+                    items:
+                      type: string
+                required:
+                  - agentExecutors
+        '400':
+          description: Generic Error
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  statusCode:
+                    type: number
+                  error:
+                    type: string
+                  message:
+                    type: string
--- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts
+++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts
@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Post Evaluate API endpoint
+ *   version: 1
+ */
+
+export type OutputIndex = z.infer<typeof OutputIndex>;
+export const OutputIndex = z.string().regex(/^.kibana-elastic-ai-assistant-/);
+
+export type DatasetItem = z.infer<typeof DatasetItem>;
+export const DatasetItem = z.object({
+  id: z.string().optional(),
+  input: z.string(),
+  prediction: z.string().optional(),
+  reference: z.string(),
+  tags: z.array(z.string()).optional(),
+});
+
+export type Dataset = z.infer<typeof Dataset>;
+export const Dataset = z.array(DatasetItem).default([]);
+
+export type PostEvaluateBody = z.infer<typeof PostEvaluateBody>;
+export const PostEvaluateBody = z.object({
+  dataset: Dataset.optional(),
+  evalPrompt: z.string().optional(),
+});
+
+export type PostEvaluateRequestQuery = z.infer<typeof PostEvaluateRequestQuery>;
+export const PostEvaluateRequestQuery = z.object({
+  /**
+   * Agents parameter description
+   */
+  agents: z.string(),
+  /**
+   * Dataset Name parameter description
+   */
+  datasetName: z.string().optional(),
+  /**
+   * Evaluation Type parameter description
+   */
+  evaluationType: z.string().optional(),
+  /**
+   * Eval Model parameter description
+   */
+  evalModel: z.string().optional(),
+  /**
+   * Models parameter description
+   */
+  models: z.string(),
+  /**
+   * Output Index parameter description
+   */
+  outputIndex: OutputIndex,
+  /**
+   * Project Name parameter description
+   */
+  projectName: z.string().optional(),
+  /**
+   * Run Name parameter description
+   */
+  runName: z.string().optional(),
+});
+export type PostEvaluateRequestQueryInput = z.input<typeof PostEvaluateRequestQuery>;
+
+export type PostEvaluateRequestBody = z.infer<typeof PostEvaluateRequestBody>;
+export const PostEvaluateRequestBody = PostEvaluateBody;
+export type PostEvaluateRequestBodyInput = z.input<typeof PostEvaluateRequestBody>;
+
+export type PostEvaluateResponse = z.infer<typeof PostEvaluateResponse>;
+export const PostEvaluateResponse = z.object({
+  evaluationId: z.string(),
+  success: z.boolean(),
+});
--- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml
+++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml
@ -0,0 +1,126 @@
+openapi: 3.0.0
+info:
+  title: Post Evaluate API endpoint
+  version: '1'
+paths:
+  /internal/elastic_assistant/evaluate:
+    post:
+      operationId: PostEvaluate
+      x-codegen-enabled: true
+      description: Perform an evaluation using sample data against a combination of Agents and Connectors
+      summary: Performs an evaluation of the Elastic Assistant
+      tags:
+        - Evaluation API
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/PostEvaluateBody'
+      parameters:
+        - name: agents
+          in: query
+          description: Agents parameter description
+          required: true
+          schema:
+            type: string
+        - name: datasetName
+          in: query
+          description: Dataset Name parameter description
+          schema:
+            type: string
+        - name: evaluationType
+          in: query
+          description: Evaluation Type parameter description
+          schema:
+            type: string
+        - name: evalModel
+          in: query
+          description: Eval Model parameter description
+          schema:
+            type: string
+        - name: models
+          in: query
+          description: Models parameter description
+          required: true
+          schema:
+            type: string
+        - name: outputIndex
+          in: query
+          description: Output Index parameter description
+          required: true
+          schema:
+            $ref: '#/components/schemas/OutputIndex'
+        - name: projectName
+          in: query
+          description: Project Name parameter description
+          schema:
+            type: string
+        - name: runName
+          in: query
+          description: Run Name parameter description
+          schema:
+            type: string
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  evaluationId:
+                    type: string
+                  success:
+                    type: boolean
+                required:
+                  - evaluationId
+                  - success
+        '400':
+          description: Generic Error
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  statusCode:
+                    type: number
+                  error:
+                    type: string
+                  message:
+                    type: string
+components:
+  schemas:
+    OutputIndex:
+      type: string
+      pattern: '^.kibana-elastic-ai-assistant-'
+    DatasetItem:
+      type: object
+      properties:
+        id:
+          type: string
+        input:
+          type: string
+        prediction:
+          type: string
+        reference:
+          type: string
+        tags:
+          type: array
+          items:
+            type: string
+      required:
+        - input
+        - reference
+    Dataset:
+      type: array
+      items:
+        $ref: '#/components/schemas/DatasetItem'
+      default: []
+    PostEvaluateBody:
+      type: object
+      properties:
+        dataset:
+          $ref: '#/components/schemas/Dataset'
+        evalPrompt:
+          type: string
--- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts
+++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts
@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// API versioning constants
+export const API_VERSIONS = {
+  public: {
+    v1: '2023-10-31',
+  },
+  internal: {
+    v1: '1',
+  },
+};
+
+export const PUBLIC_API_ACCESS = 'public';
+export const INTERNAL_API_ACCESS = 'internal';
+
+// Evaluation Schemas
+export * from './evaluation/post_evaluate_route.gen';
+export * from './evaluation/get_evaluate_route.gen';
+
+// Capabilities Schemas
+export * from './capabilities/get_capabilities_route.gen';
--- a/x-pack/packages/kbn-elastic-assistant-common/index.ts
+++ b/x-pack/packages/kbn-elastic-assistant-common/index.ts
@ -5,7 +5,8 @@
 * 2.0.
 */

-export { GetCapabilitiesResponse } from './impl/schemas/capabilities/get_capabilities_route.gen';
+// Schema constants
+export * from './impl/schemas';

 export { defaultAssistantFeatures } from './impl/capabilities';
 export type { AssistantFeatures } from './impl/capabilities';
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx
@ -13,7 +13,6 @@ import {
  fetchConnectorExecuteAction,
  FetchConnectorExecuteAction,
  getKnowledgeBaseStatus,
-  postEvaluation,
  postKnowledgeBase,
 } from './api';
 import type { Conversation, Message } from '../assistant_context/types';
@ -340,52 +339,4 @@ describe('API tests', () => {
      await expect(deleteKnowledgeBase(knowledgeBaseArgs)).resolves.toThrowError('simulated error');
    });
  });
-
-  describe('postEvaluation', () => {
-    it('calls the knowledge base API when correct resource path', async () => {
-      (mockHttp.fetch as jest.Mock).mockResolvedValue({ success: true });
-      const testProps = {
-        http: mockHttp,
-        evalParams: {
-          agents: ['not', 'alphabetical'],
-          dataset: '{}',
-          datasetName: 'Test Dataset',
-          projectName: 'Test Project Name',
-          runName: 'Test Run Name',
-          evalModel: ['not', 'alphabetical'],
-          evalPrompt: 'evalPrompt',
-          evaluationType: ['not', 'alphabetical'],
-          models: ['not', 'alphabetical'],
-          outputIndex: 'outputIndex',
-        },
-      };
-
-      await postEvaluation(testProps);
-
-      expect(mockHttp.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', {
-        method: 'POST',
-        body: '{"dataset":{},"evalPrompt":"evalPrompt"}',
-        headers: { 'Content-Type': 'application/json' },
-        query: {
-          models: 'alphabetical,not',
-          agents: 'alphabetical,not',
-          datasetName: 'Test Dataset',
-          evaluationType: 'alphabetical,not',
-          evalModel: 'alphabetical,not',
-          outputIndex: 'outputIndex',
-          projectName: 'Test Project Name',
-          runName: 'Test Run Name',
-        },
-        signal: undefined,
-      });
-    });
-    it('returns error when error is an error', async () => {
-      const error = 'simulated error';
-      (mockHttp.fetch as jest.Mock).mockImplementation(() => {
-        throw new Error(error);
-      });
-
-      await expect(postEvaluation(knowledgeBaseArgs)).resolves.toThrowError('simulated error');
-    });
-  });
 });
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx
@ -16,7 +16,6 @@ import {
  getOptionalRequestParams,
  hasParsableResponse,
 } from './helpers';
-import { PerformEvaluationParams } from './settings/evaluation_settings/use_perform_evaluation';

 export interface FetchConnectorExecuteAction {
  isEnabledRAGAlerts: boolean;
@ -335,61 +334,3 @@ export const deleteKnowledgeBase = async ({
    return error as IHttpFetchError;
  }
 };
-
-export interface PostEvaluationParams {
-  http: HttpSetup;
-  evalParams?: PerformEvaluationParams;
-  signal?: AbortSignal | undefined;
-}
-
-export interface PostEvaluationResponse {
-  evaluationId: string;
-  success: boolean;
-}
-
-/**
- * API call for evaluating models.
- *
- * @param {Object} options - The options object.
- * @param {HttpSetup} options.http - HttpSetup
- * @param {string} [options.evalParams] - Params necessary for evaluation
- * @param {AbortSignal} [options.signal] - AbortSignal
- *
- * @returns {Promise<PostEvaluationResponse | IHttpFetchError>}
- */
-export const postEvaluation = async ({
-  http,
-  evalParams,
-  signal,
-}: PostEvaluationParams): Promise<PostEvaluationResponse | IHttpFetchError> => {
-  try {
-    const path = `/internal/elastic_assistant/evaluate`;
-    const query = {
-      agents: evalParams?.agents.sort()?.join(','),
-      datasetName: evalParams?.datasetName,
-      evaluationType: evalParams?.evaluationType.sort()?.join(','),
-      evalModel: evalParams?.evalModel.sort()?.join(','),
-      outputIndex: evalParams?.outputIndex,
-      models: evalParams?.models.sort()?.join(','),
-      projectName: evalParams?.projectName,
-      runName: evalParams?.runName,
-    };
-
-    const response = await http.fetch(path, {
-      method: 'POST',
-      body: JSON.stringify({
-        dataset: JSON.parse(evalParams?.dataset ?? '[]'),
-        evalPrompt: evalParams?.evalPrompt ?? '',
-      }),
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      query,
-      signal,
-    });
-
-    return response as PostEvaluationResponse;
-  } catch (error) {
-    return error as IHttpFetchError;
-  }
-};
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.test.tsx
@ -13,7 +13,7 @@ import { API_ERROR } from '../../translations';
 jest.mock('@kbn/core-http-browser');

 const mockHttp = {
-  fetch: jest.fn(),
+  get: jest.fn(),
 } as unknown as HttpSetup;

 describe('Capabilities API tests', () => {
@ -25,15 +25,14 @@ describe('Capabilities API tests', () => {
    it('calls the internal assistant API for fetching assistant capabilities', async () => {
      await getCapabilities({ http: mockHttp });

-      expect(mockHttp.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', {
-        method: 'GET',
+      expect(mockHttp.get).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', {
        signal: undefined,
        version: '1',
      });
    });

    it('returns API_ERROR when the response status is error', async () => {
-      (mockHttp.fetch as jest.Mock).mockResolvedValue({ status: API_ERROR });
+      (mockHttp.get as jest.Mock).mockResolvedValue({ status: API_ERROR });

      const result = await getCapabilities({ http: mockHttp });

--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx
@ -6,7 +6,7 @@
 */

 import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser';
-import { GetCapabilitiesResponse } from '@kbn/elastic-assistant-common';
+import { API_VERSIONS, GetCapabilitiesResponse } from '@kbn/elastic-assistant-common';

 export interface GetCapabilitiesParams {
  http: HttpSetup;
@ -29,13 +29,10 @@ export const getCapabilities = async ({
  try {
    const path = `/internal/elastic_assistant/capabilities`;

-    const response = await http.fetch(path, {
-      method: 'GET',
+    return await http.get<GetCapabilitiesResponse>(path, {
      signal,
-      version: '1',
+      version: API_VERSIONS.internal.v1,
    });
-
-    return response as GetCapabilitiesResponse;
  } catch (error) {
    return error as IHttpFetchError;
  }
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.test.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.test.tsx
@ -11,11 +11,12 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 import type { ReactNode } from 'react';
 import React from 'react';
 import { useCapabilities, UseCapabilitiesParams } from './use_capabilities';
+import { API_VERSIONS } from '@kbn/elastic-assistant-common';

 const statusResponse = { assistantModelEvaluation: true, assistantStreamingEnabled: false };

 const http = {
-  fetch: jest.fn().mockResolvedValue(statusResponse),
+  get: jest.fn().mockResolvedValue(statusResponse),
 };
 const toasts = {
  addError: jest.fn(),
@ -36,14 +37,10 @@ describe('useFetchRelatedCases', () => {
      wrapper: createWrapper(),
    });

-    expect(defaultProps.http.fetch).toHaveBeenCalledWith(
-      '/internal/elastic_assistant/capabilities',
-      {
-        method: 'GET',
-        version: '1',
-        signal: new AbortController().signal,
-      }
-    );
+    expect(defaultProps.http.get).toHaveBeenCalledWith('/internal/elastic_assistant/capabilities', {
+      version: API_VERSIONS.internal.v1,
+      signal: new AbortController().signal,
+    });
    expect(toasts.addError).not.toHaveBeenCalled();
  });
 });
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.test.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.test.tsx
@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { postEvaluation } from './evaluate';
+import { HttpSetup } from '@kbn/core-http-browser';
+import { API_VERSIONS } from '@kbn/elastic-assistant-common';
+
+jest.mock('@kbn/core-http-browser');
+
+const mockHttp = {
+  post: jest.fn(),
+} as unknown as HttpSetup;
+
+describe('postEvaluation', () => {
+  it('calls the knowledge base API when correct resource path', async () => {
+    (mockHttp.post as jest.Mock).mockResolvedValue({ success: true });
+    const testProps = {
+      http: mockHttp,
+      evalParams: {
+        agents: ['not', 'alphabetical'],
+        dataset: '{}',
+        datasetName: 'Test Dataset',
+        projectName: 'Test Project Name',
+        runName: 'Test Run Name',
+        evalModel: ['not', 'alphabetical'],
+        evalPrompt: 'evalPrompt',
+        evaluationType: ['not', 'alphabetical'],
+        models: ['not', 'alphabetical'],
+        outputIndex: 'outputIndex',
+      },
+    };
+
+    await postEvaluation(testProps);
+
+    expect(mockHttp.post).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', {
+      body: '{"dataset":{},"evalPrompt":"evalPrompt"}',
+      headers: { 'Content-Type': 'application/json' },
+      query: {
+        models: 'alphabetical,not',
+        agents: 'alphabetical,not',
+        datasetName: 'Test Dataset',
+        evaluationType: 'alphabetical,not',
+        evalModel: 'alphabetical,not',
+        outputIndex: 'outputIndex',
+        projectName: 'Test Project Name',
+        runName: 'Test Run Name',
+      },
+      signal: undefined,
+      version: API_VERSIONS.internal.v1,
+    });
+  });
+  it('returns error when error is an error', async () => {
+    const error = 'simulated error';
+    (mockHttp.post as jest.Mock).mockImplementation(() => {
+      throw new Error(error);
+    });
+
+    const knowledgeBaseArgs = {
+      resource: 'a-resource',
+      http: mockHttp,
+    };
+
+    await expect(postEvaluation(knowledgeBaseArgs)).resolves.toThrowError('simulated error');
+  });
+});
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx
@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser';
+import {
+  API_VERSIONS,
+  GetEvaluateResponse,
+  PostEvaluateResponse,
+} from '@kbn/elastic-assistant-common';
+import { PerformEvaluationParams } from './use_perform_evaluation';
+
+export interface PostEvaluationParams {
+  http: HttpSetup;
+  evalParams?: PerformEvaluationParams;
+  signal?: AbortSignal | undefined;
+}
+
+/**
+ * API call for evaluating models.
+ *
+ * @param {Object} options - The options object.
+ * @param {HttpSetup} options.http - HttpSetup
+ * @param {string} [options.evalParams] - Params necessary for evaluation
+ * @param {AbortSignal} [options.signal] - AbortSignal
+ *
+ * @returns {Promise<PostEvaluateResponse | IHttpFetchError>}
+ */
+export const postEvaluation = async ({
+  http,
+  evalParams,
+  signal,
+}: PostEvaluationParams): Promise<PostEvaluateResponse | IHttpFetchError> => {
+  try {
+    const path = `/internal/elastic_assistant/evaluate`;
+    const query = {
+      agents: evalParams?.agents.sort()?.join(','),
+      datasetName: evalParams?.datasetName,
+      evaluationType: evalParams?.evaluationType.sort()?.join(','),
+      evalModel: evalParams?.evalModel.sort()?.join(','),
+      outputIndex: evalParams?.outputIndex,
+      models: evalParams?.models.sort()?.join(','),
+      projectName: evalParams?.projectName,
+      runName: evalParams?.runName,
+    };
+
+    return await http.post<PostEvaluateResponse>(path, {
+      body: JSON.stringify({
+        dataset: JSON.parse(evalParams?.dataset ?? '[]'),
+        evalPrompt: evalParams?.evalPrompt ?? '',
+      }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      query,
+      signal,
+      version: API_VERSIONS.internal.v1,
+    });
+  } catch (error) {
+    return error as IHttpFetchError;
+  }
+};
+
+export interface GetEvaluationParams {
+  http: HttpSetup;
+  signal?: AbortSignal | undefined;
+}
+
+/**
+ * API call for fetching evaluation data.
+ *
+ * @param {Object} options - The options object.
+ * @param {HttpSetup} options.http - HttpSetup
+ * @param {AbortSignal} [options.signal] - AbortSignal
+ *
+ * @returns {Promise<GetEvaluateResponse | IHttpFetchError>}
+ */
+export const getEvaluation = async ({
+  http,
+  signal,
+}: GetEvaluationParams): Promise<GetEvaluateResponse | IHttpFetchError> => {
+  try {
+    const path = `/internal/elastic_assistant/evaluate`;
+
+    return await http.get<GetEvaluateResponse>(path, {
+      signal,
+      version: API_VERSIONS.internal.v1,
+    });
+  } catch (error) {
+    return error as IHttpFetchError;
+  }
+};
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_evaluation_data.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/use_evaluation_data.tsx
@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useQuery } from '@tanstack/react-query';
+import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser';
+import type { IToasts } from '@kbn/core-notifications-browser';
+import { i18n } from '@kbn/i18n';
+import { getEvaluation } from './evaluate';
+
+const EVALUATION_DATA_QUERY_KEY = ['elastic-assistant', 'evaluation-data'];
+
+export interface UseEvaluationDataParams {
+  http: HttpSetup;
+  toasts?: IToasts;
+}
+
+/**
+ * Hook for fetching evaluation data, like available agents, test data, etc
+ *
+ * @param {Object} options - The options object.
+ * @param {HttpSetup} options.http - HttpSetup
+ * @param {IToasts} [options.toasts] - IToasts
+ *
+ * @returns {useMutation} mutation hook for setting up the Knowledge Base
+ */
+export const useEvaluationData = ({ http, toasts }: UseEvaluationDataParams) => {
+  return useQuery({
+    queryKey: EVALUATION_DATA_QUERY_KEY,
+    queryFn: ({ signal }) => {
+      // Optional params workaround: see: https://github.com/TanStack/query/issues/1077#issuecomment-1431247266
+      return getEvaluation({ http, signal });
+    },
+    retry: false,
+    keepPreviousData: true,
+    // Deprecated, hoist to `queryCache` w/in `QueryClient. See: https://stackoverflow.com/a/76961109
+    onError: (error: IHttpFetchError<ResponseErrorBody>) => {
+      if (error.name !== 'AbortError') {
+        toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, {
+          title: i18n.translate('xpack.elasticAssistant.evaluation.fetchEvaluationDataError', {
+            defaultMessage: 'Error fetching evaluation data...',
+          }),
+        });
+      }
+    },
+  });
+};
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.test.tsx
@ -7,14 +7,15 @@

 import { act, renderHook } from '@testing-library/react-hooks';
 import { usePerformEvaluation, UsePerformEvaluationParams } from './use_perform_evaluation';
-import { postEvaluation as _postEvaluation } from '../../api';
+import { postEvaluation as _postEvaluation } from './evaluate';
 import { useMutation as _useMutation } from '@tanstack/react-query';
+import { API_VERSIONS } from '@kbn/elastic-assistant-common';

 const useMutationMock = _useMutation as jest.Mock;
 const postEvaluationMock = _postEvaluation as jest.Mock;

-jest.mock('../../api', () => {
-  const actual = jest.requireActual('../../api');
+jest.mock('./evaluate', () => {
+  const actual = jest.requireActual('./evaluate');
  return {
    ...actual,
    postEvaluation: jest.fn((...args) => actual.postEvaluation(...args)),
@ -37,7 +38,7 @@ const statusResponse = {
 };

 const http = {
-  fetch: jest.fn().mockResolvedValue(statusResponse),
+  post: jest.fn().mockResolvedValue(statusResponse),
 };
 const toasts = {
  addError: jest.fn(),
@ -53,20 +54,23 @@ describe('usePerformEvaluation', () => {
      const { waitForNextUpdate } = renderHook(() => usePerformEvaluation(defaultProps));
      await waitForNextUpdate();

-      expect(defaultProps.http.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', {
-        method: 'POST',
+      expect(defaultProps.http.post).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', {
        body: '{"dataset":[],"evalPrompt":""}',
        headers: {
          'Content-Type': 'application/json',
        },
        query: {
          agents: undefined,
+          datasetName: undefined,
          evalModel: undefined,
          evaluationType: undefined,
          models: undefined,
          outputIndex: undefined,
+          projectName: undefined,
+          runName: undefined,
        },
        signal: undefined,
+        version: API_VERSIONS.internal.v1,
      });
      expect(toasts.addError).not.toHaveBeenCalled();
    });
@ -82,6 +86,8 @@ describe('usePerformEvaluation', () => {
          evaluationType: ['f', 'e'],
          models: ['h', 'g'],
          outputIndex: 'outputIndex',
+          projectName: 'test project',
+          runName: 'test run',
        });
        return Promise.resolve(res);
      } catch (e) {
@ -92,20 +98,23 @@ describe('usePerformEvaluation', () => {
      const { waitForNextUpdate } = renderHook(() => usePerformEvaluation(defaultProps));
      await waitForNextUpdate();

-      expect(defaultProps.http.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', {
-        method: 'POST',
+      expect(defaultProps.http.post).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', {
        body: '{"dataset":["kewl"],"evalPrompt":"evalPrompt"}',
        headers: {
          'Content-Type': 'application/json',
        },
        query: {
          agents: 'c,d',
+          datasetName: undefined,
          evalModel: 'a,b',
          evaluationType: 'e,f',
          models: 'g,h',
          outputIndex: 'outputIndex',
+          projectName: 'test project',
+          runName: 'test run',
        },
        signal: undefined,
+        version: API_VERSIONS.internal.v1,
      });
    });
  });
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/use_perform_evaluation.tsx
@ -9,7 +9,7 @@ import { useMutation } from '@tanstack/react-query';
 import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser';
 import type { IToasts } from '@kbn/core-notifications-browser';
 import { i18n } from '@kbn/i18n';
-import { postEvaluation } from '../../api';
+import { postEvaluation } from './evaluate';

 const PERFORM_EVALUATION_MUTATION_KEY = ['elastic-assistant', 'perform-evaluation'];

--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx
@ -27,20 +27,16 @@ import {

 import { css } from '@emotion/react';
 import { FormattedMessage } from '@kbn/i18n-react';
+import type { GetEvaluateResponse, PostEvaluateResponse } from '@kbn/elastic-assistant-common';
 import * as i18n from './translations';
 import { useAssistantContext } from '../../../assistant_context';
 import { useLoadConnectors } from '../../../connectorland/use_load_connectors';
 import { getActionTypeTitle, getGenAiConfig } from '../../../connectorland/helpers';
 import { PRECONFIGURED_CONNECTOR } from '../../../connectorland/translations';
-import { usePerformEvaluation } from './use_perform_evaluation';
+import { usePerformEvaluation } from '../../api/evaluate/use_perform_evaluation';
 import { getApmLink, getDiscoverLink } from './utils';
-import { PostEvaluationResponse } from '../../api';
+import { useEvaluationData } from '../../api/evaluate/use_evaluation_data';

-/**
- * See AGENT_EXECUTOR_MAP in `x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts`
- * for the agent name -> executor mapping
- */
-const DEFAULT_AGENTS = ['DefaultAgentExecutor', 'OpenAIFunctionsExecutor'];
 const DEFAULT_EVAL_TYPES_OPTIONS = [
  { label: 'correctness' },
  { label: 'esql-validator', disabled: true },
@ -65,6 +61,11 @@ export const EvaluationSettings: React.FC<Props> = React.memo(({ onEvaluationSet
  } = usePerformEvaluation({
    http,
  });
+  const { data: evalData } = useEvaluationData({ http });
+  const defaultAgents = useMemo(
+    () => (evalData as GetEvaluateResponse)?.agentExecutors ?? [],
+    [evalData]
+  );

  // Run Details
  // Project Name
@ -195,8 +196,8 @@ export const EvaluationSettings: React.FC<Props> = React.memo(({ onEvaluationSet
    [selectedAgentOptions]
  );
  const agentOptions = useMemo(() => {
-    return DEFAULT_AGENTS.map((label) => ({ label }));
-  }, []);
+    return defaultAgents.map((label) => ({ label }));
+  }, [defaultAgents]);

  // Evaluation
  // Evaluation Type
@ -283,12 +284,12 @@ export const EvaluationSettings: React.FC<Props> = React.memo(({ onEvaluationSet
  ]);

  const discoverLink = useMemo(
-    () => getDiscoverLink(basePath, (evalResponse as PostEvaluationResponse)?.evaluationId ?? ''),
+    () => getDiscoverLink(basePath, (evalResponse as PostEvaluateResponse)?.evaluationId ?? ''),
    [basePath, evalResponse]
  );

  const apmLink = useMemo(
-    () => getApmLink(basePath, (evalResponse as PostEvaluationResponse)?.evaluationId ?? ''),
+    () => getApmLink(basePath, (evalResponse as PostEvaluateResponse)?.evaluationId ?? ''),
    [basePath, evalResponse]
  );

--- a/x-pack/plugins/elastic_assistant/server/mocks/request.ts
+++ b/x-pack/plugins/elastic_assistant/server/mocks/request.ts
@ -7,9 +7,9 @@
 import { httpServerMock } from '@kbn/core/server/mocks';
 import { CAPABILITIES, EVALUATE, KNOWLEDGE_BASE } from '../../common/constants';
 import {
-  PostEvaluateBodyInputs,
-  PostEvaluatePathQueryInputs,
-} from '../schemas/evaluate/post_evaluate';
+  PostEvaluateRequestBodyInput,
+  PostEvaluateRequestQueryInput,
+} from '@kbn/elastic-assistant-common';

 export const requestMock = {
  create: httpServerMock.createKibanaRequest,
@ -46,8 +46,8 @@ export const getPostEvaluateRequest = ({
  body,
  query,
 }: {
-  body: PostEvaluateBodyInputs;
-  query: PostEvaluatePathQueryInputs;
+  body: PostEvaluateRequestBodyInput;
+  query: PostEvaluateRequestQueryInput;
 }) =>
  requestMock.create({
    body,
--- a/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/index.ts
+++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/index.ts
@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AgentExecutor } from './types';
+import { callAgentExecutor } from '../execute_custom_llm_chain';
+import { callOpenAIFunctionsExecutor } from './openai_functions_executor';
+
+/**
+ * To support additional Agent Executors from the UI, add them to this map
+ * and reference your specific AgentExecutor function
+ */
+export const AGENT_EXECUTOR_MAP: Record<string, AgentExecutor> = {
+  DefaultAgentExecutor: callAgentExecutor,
+  OpenAIFunctionsExecutor: callOpenAIFunctionsExecutor,
+};
--- a/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts
+++ b/x-pack/plugins/elastic_assistant/server/lib/model_evaluator/evaluation.ts
@ -12,8 +12,8 @@ import { chunk as createChunks } from 'lodash/fp';
 import { Logger } from '@kbn/core/server';
 import { ToolingLog } from '@kbn/tooling-log';
 import { LangChainTracer, RunCollectorCallbackHandler } from 'langchain/callbacks';
+import { Dataset } from '@kbn/elastic-assistant-common';
 import { AgentExecutorEvaluatorWithMetadata } from '../langchain/executors/types';
-import { Dataset } from '../../schemas/evaluate/post_evaluate';
 import { callAgentWithRetry, getMessageFromLangChainResponse } from './utils';
 import { ResponseBody } from '../langchain/types';
 import { isLangSmithEnabled, writeLangSmithFeedback } from '../../routes/evaluate/utils';
@ -102,7 +102,6 @@ export const performEvaluation = async ({
    const chunk = requestChunks.shift() ?? [];
    const chunkNumber = totalChunks - requestChunks.length;
    logger.info(`Prediction request chunk: ${chunkNumber} of ${totalChunks}`);
-    logger.debug(chunk);

    // Note, order is kept between chunk and dataset, and is preserved w/ Promise.allSettled
    const chunkResults = await Promise.allSettled(chunk.map((r) => r.request()));
--- a/x-pack/plugins/elastic_assistant/server/plugin.ts
+++ b/x-pack/plugins/elastic_assistant/server/plugin.ts
@ -43,6 +43,7 @@ import {
  GetRegisteredTools,
 } from './services/app_context';
 import { getCapabilitiesRoute } from './routes/capabilities/get_capabilities_route';
+import { getEvaluateRoute } from './routes/evaluate/get_evaluate';

 interface CreateRouteHandlerContextParams {
  core: CoreSetup<ElasticAssistantPluginStart, unknown>;
@ -124,6 +125,7 @@ export class ElasticAssistantPlugin
    postActionsConnectorExecuteRoute(router, getElserId);
    // Evaluate
    postEvaluateRoute(router, getElserId);
+    getEvaluateRoute(router);
    // Capabilities
    getCapabilitiesRoute(router);
    return {
--- a/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts
+++ b/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts
@ -8,12 +8,17 @@
 import { IKibanaResponse, IRouter } from '@kbn/core/server';
 import { transformError } from '@kbn/securitysolution-es-utils';

-import type { GetCapabilitiesResponse } from '@kbn/elastic-assistant-common';
+import {
+  API_VERSIONS,
+  GetCapabilitiesResponse,
+  INTERNAL_API_ACCESS,
+} from '@kbn/elastic-assistant-common';
 import { CAPABILITIES } from '../../../common/constants';
 import { ElasticAssistantRequestHandlerContext } from '../../types';

 import { buildResponse } from '../../lib/build_response';
 import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers';
+import { buildRouteValidationWithZod } from '../../schemas/common';

 /**
 * Get the assistant capabilities for the requesting plugin
@ -23,7 +28,7 @@ import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers';
 export const getCapabilitiesRoute = (router: IRouter<ElasticAssistantRequestHandlerContext>) => {
  router.versioned
    .get({
-      access: 'internal',
+      access: INTERNAL_API_ACCESS,
      path: CAPABILITIES,
      options: {
        tags: ['access:elasticAssistant'],
@ -31,8 +36,14 @@ export const getCapabilitiesRoute = (router: IRouter<ElasticAssistantRequestHand
    })
    .addVersion(
      {
-        version: '1',
-        validate: {},
+        version: API_VERSIONS.internal.v1,
+        validate: {
+          response: {
+            200: {
+              body: buildRouteValidationWithZod(GetCapabilitiesResponse),
+            },
+          },
+        },
      },
      async (context, request, response): Promise<IKibanaResponse<GetCapabilitiesResponse>> => {
        const resp = buildResponse(response);
--- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts
+++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/get_evaluate.ts
@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { type IKibanaResponse, IRouter } from '@kbn/core/server';
+import { transformError } from '@kbn/securitysolution-es-utils';
+
+import {
+  API_VERSIONS,
+  INTERNAL_API_ACCESS,
+  GetEvaluateResponse,
+} from '@kbn/elastic-assistant-common';
+import { buildResponse } from '../../lib/build_response';
+import { ElasticAssistantRequestHandlerContext } from '../../types';
+import { EVALUATE } from '../../../common/constants';
+import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers';
+import { buildRouteValidationWithZod } from '../../schemas/common';
+import { AGENT_EXECUTOR_MAP } from '../../lib/langchain/executors';
+
+export const getEvaluateRoute = (router: IRouter<ElasticAssistantRequestHandlerContext>) => {
+  router.versioned
+    .get({
+      access: INTERNAL_API_ACCESS,
+      path: EVALUATE,
+      options: {
+        tags: ['access:elasticAssistant'],
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.internal.v1,
+        validate: {
+          response: {
+            200: {
+              body: buildRouteValidationWithZod(GetEvaluateResponse),
+            },
+          },
+        },
+      },
+      async (context, request, response): Promise<IKibanaResponse<GetEvaluateResponse>> => {
+        const assistantContext = await context.elasticAssistant;
+        const logger = assistantContext.logger;
+
+        // Validate evaluation feature is enabled
+        const pluginName = getPluginNameFromRequest({
+          request,
+          defaultPluginName: DEFAULT_PLUGIN_NAME,
+          logger,
+        });
+        const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName);
+        if (!registeredFeatures.assistantModelEvaluation) {
+          return response.notFound();
+        }
+
+        try {
+          return response.ok({ body: { agentExecutors: Object.keys(AGENT_EXECUTOR_MAP) } });
+        } catch (err) {
+          logger.error(err);
+          const error = transformError(err);
+
+          const resp = buildResponse(response);
+          return resp.error({
+            body: { error: error.message },
+            statusCode: error.statusCode,
+          });
+        }
+      }
+    );
+};
--- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts
+++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.test.ts
@ -9,17 +9,17 @@ import { postEvaluateRoute } from './post_evaluate';
 import { serverMock } from '../../__mocks__/server';
 import { requestContextMock } from '../../__mocks__/request_context';
 import { getPostEvaluateRequest } from '../../__mocks__/request';
-import {
-  PostEvaluateBodyInputs,
-  PostEvaluatePathQueryInputs,
-} from '../../schemas/evaluate/post_evaluate';
+import type {
+  PostEvaluateRequestBodyInput,
+  PostEvaluateRequestQueryInput,
+} from '@kbn/elastic-assistant-common';

-const defaultBody: PostEvaluateBodyInputs = {
+const defaultBody: PostEvaluateRequestBodyInput = {
  dataset: undefined,
  evalPrompt: undefined,
 };

-const defaultQueryParams: PostEvaluatePathQueryInputs = {
+const defaultQueryParams: PostEvaluateRequestQueryInput = {
  agents: 'agents',
  datasetName: undefined,
  evaluationType: undefined,
--- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts
+++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts
@ -5,23 +5,23 @@
 * 2.0.
 */

-import { IRouter, KibanaRequest } from '@kbn/core/server';
+import { type IKibanaResponse, IRouter, KibanaRequest } from '@kbn/core/server';
 import { transformError } from '@kbn/securitysolution-es-utils';
 import { v4 as uuidv4 } from 'uuid';

+import {
+  API_VERSIONS,
+  INTERNAL_API_ACCESS,
+  PostEvaluateBody,
+  PostEvaluateRequestQuery,
+  PostEvaluateResponse,
+} from '@kbn/elastic-assistant-common';
 import { ESQL_RESOURCE } from '../knowledge_base/constants';
 import { buildResponse } from '../../lib/build_response';
-import { buildRouteValidation } from '../../schemas/common';
 import { ElasticAssistantRequestHandlerContext, GetElser } from '../../types';
 import { EVALUATE } from '../../../common/constants';
-import { PostEvaluateBody, PostEvaluatePathQuery } from '../../schemas/evaluate/post_evaluate';
 import { performEvaluation } from '../../lib/model_evaluator/evaluation';
-import { callAgentExecutor } from '../../lib/langchain/execute_custom_llm_chain';
-import { callOpenAIFunctionsExecutor } from '../../lib/langchain/executors/openai_functions_executor';
-import {
-  AgentExecutor,
-  AgentExecutorEvaluatorWithMetadata,
-} from '../../lib/langchain/executors/types';
+import { AgentExecutorEvaluatorWithMetadata } from '../../lib/langchain/executors/types';
 import { ActionsClientLlm } from '../../lib/langchain/llm/actions_client_llm';
 import {
  indexEvaluations,
@ -30,15 +30,8 @@ import {
 import { fetchLangSmithDataset, getConnectorName, getLangSmithTracer, getLlmType } from './utils';
 import { RequestBody } from '../../lib/langchain/types';
 import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers';
-
-/**
- * To support additional Agent Executors from the UI, add them to this map
- * and reference your specific AgentExecutor function
- */
-const AGENT_EXECUTOR_MAP: Record<string, AgentExecutor> = {
-  DefaultAgentExecutor: callAgentExecutor,
-  OpenAIFunctionsExecutor: callOpenAIFunctionsExecutor,
-};
+import { buildRouteValidationWithZod } from '../../schemas/common';
+import { AGENT_EXECUTOR_MAP } from '../../lib/langchain/executors';

 const DEFAULT_SIZE = 20;

@ -46,200 +39,215 @@ export const postEvaluateRoute = (
  router: IRouter<ElasticAssistantRequestHandlerContext>,
  getElser: GetElser
 ) => {
-  router.post(
-    {
+  router.versioned
+    .post({
+      access: INTERNAL_API_ACCESS,
      path: EVALUATE,
-      validate: {
-        body: buildRouteValidation(PostEvaluateBody),
-        query: buildRouteValidation(PostEvaluatePathQuery),
+      options: {
+        tags: ['access:elasticAssistant'],
      },
-    },
-    async (context, request, response) => {
-      const assistantContext = await context.elasticAssistant;
-      const logger = assistantContext.logger;
-      const telemetry = assistantContext.telemetry;
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.internal.v1,
+        validate: {
+          request: {
+            body: buildRouteValidationWithZod(PostEvaluateBody),
+            query: buildRouteValidationWithZod(PostEvaluateRequestQuery),
+          },
+          response: {
+            200: {
+              body: buildRouteValidationWithZod(PostEvaluateResponse),
+            },
+          },
+        },
+      },
+      async (context, request, response): Promise<IKibanaResponse<PostEvaluateResponse>> => {
+        const assistantContext = await context.elasticAssistant;
+        const logger = assistantContext.logger;
+        const telemetry = assistantContext.telemetry;

-      // Validate evaluation feature is enabled
-      const pluginName = getPluginNameFromRequest({
-        request,
-        defaultPluginName: DEFAULT_PLUGIN_NAME,
-        logger,
-      });
-      const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName);
-      if (!registeredFeatures.assistantModelEvaluation) {
-        return response.notFound();
-      }
-
-      try {
-        const evaluationId = uuidv4();
-        const {
-          evalModel,
-          evaluationType,
-          outputIndex,
-          datasetName,
-          projectName = 'default',
-          runName = evaluationId,
-        } = request.query;
-        const { dataset: customDataset = [], evalPrompt } = request.body;
-        const connectorIds = request.query.models?.split(',') || [];
-        const agentNames = request.query.agents?.split(',') || [];
-
-        const dataset =
-          datasetName != null ? await fetchLangSmithDataset(datasetName, logger) : customDataset;
-
-        logger.info('postEvaluateRoute:');
-        logger.info(`request.query:\n${JSON.stringify(request.query, null, 2)}`);
-        logger.info(`request.body:\n${JSON.stringify(request.body, null, 2)}`);
-        logger.info(`Evaluation ID: ${evaluationId}`);
-
-        const totalExecutions = connectorIds.length * agentNames.length * dataset.length;
-        logger.info('Creating agents:');
-        logger.info(`\tconnectors/models: ${connectorIds.length}`);
-        logger.info(`\tagents: ${agentNames.length}`);
-        logger.info(`\tdataset: ${dataset.length}`);
-        logger.warn(`\ttotal baseline agent executions: ${totalExecutions} `);
-        if (totalExecutions > 50) {
-          logger.warn(
-            `Total baseline agent executions >= 50! This may take a while, and cost some money...`
-          );
+        // Validate evaluation feature is enabled
+        const pluginName = getPluginNameFromRequest({
+          request,
+          defaultPluginName: DEFAULT_PLUGIN_NAME,
+          logger,
+        });
+        const registeredFeatures = assistantContext.getRegisteredFeatures(pluginName);
+        if (!registeredFeatures.assistantModelEvaluation) {
+          return response.notFound();
        }

-        // Get the actions plugin start contract from the request context for the agents
-        const actions = (await context.elasticAssistant).actions;
+        try {
+          const evaluationId = uuidv4();
+          const {
+            evalModel,
+            evaluationType,
+            outputIndex,
+            datasetName,
+            projectName = 'default',
+            runName = evaluationId,
+          } = request.query;
+          const { dataset: customDataset = [], evalPrompt } = request.body;
+          const connectorIds = request.query.models?.split(',') || [];
+          const agentNames = request.query.agents?.split(',') || [];

-        // Fetch all connectors from the actions plugin, so we can set the appropriate `llmType` on ActionsClientLlm
-        const actionsClient = await actions.getActionsClientWithRequest(request);
-        const connectors = await actionsClient.getBulk({
-          ids: connectorIds,
-          throwIfSystemAction: false,
-        });
+          const dataset =
+            datasetName != null ? await fetchLangSmithDataset(datasetName, logger) : customDataset;

-        // Fetch any tools registered by the request's originating plugin
-        const assistantTools = (await context.elasticAssistant).getRegisteredTools(
-          'securitySolution'
-        );
+          logger.info('postEvaluateRoute:');
+          logger.info(`request.query:\n${JSON.stringify(request.query, null, 2)}`);
+          logger.info(`request.body:\n${JSON.stringify(request.body, null, 2)}`);
+          logger.info(`Evaluation ID: ${evaluationId}`);

-        // Get a scoped esClient for passing to the agents for retrieval, and
-        // writing results to the output index
-        const esClient = (await context.core).elasticsearch.client.asCurrentUser;
+          const totalExecutions = connectorIds.length * agentNames.length * dataset.length;
+          logger.info('Creating agents:');
+          logger.info(`\tconnectors/models: ${connectorIds.length}`);
+          logger.info(`\tagents: ${agentNames.length}`);
+          logger.info(`\tdataset: ${dataset.length}`);
+          logger.warn(`\ttotal baseline agent executions: ${totalExecutions} `);
+          if (totalExecutions > 50) {
+            logger.warn(
+              `Total baseline agent executions >= 50! This may take a while, and cost some money...`
+            );
+          }

-        // Default ELSER model
-        const elserId = await getElser(request, (await context.core).savedObjects.getClient());
+          // Get the actions plugin start contract from the request context for the agents
+          const actions = (await context.elasticAssistant).actions;

-        // Skeleton request from route to pass to the agents
-        // params will be passed to the actions executor
-        const skeletonRequest: KibanaRequest<unknown, unknown, RequestBody> = {
-          ...request,
-          body: {
-            alertsIndexPattern: '',
-            allow: [],
-            allowReplacement: [],
-            params: {
-              subAction: 'invokeAI',
-              subActionParams: {
-                messages: [],
+          // Fetch all connectors from the actions plugin, so we can set the appropriate `llmType` on ActionsClientLlm
+          const actionsClient = await actions.getActionsClientWithRequest(request);
+          const connectors = await actionsClient.getBulk({
+            ids: connectorIds,
+            throwIfSystemAction: false,
+          });
+
+          // Fetch any tools registered by the request's originating plugin
+          const assistantTools = (await context.elasticAssistant).getRegisteredTools(
+            'securitySolution'
+          );
+
+          // Get a scoped esClient for passing to the agents for retrieval, and
+          // writing results to the output index
+          const esClient = (await context.core).elasticsearch.client.asCurrentUser;
+
+          // Default ELSER model
+          const elserId = await getElser(request, (await context.core).savedObjects.getClient());
+
+          // Skeleton request from route to pass to the agents
+          // params will be passed to the actions executor
+          const skeletonRequest: KibanaRequest<unknown, unknown, RequestBody> = {
+            ...request,
+            body: {
+              alertsIndexPattern: '',
+              allow: [],
+              allowReplacement: [],
+              params: {
+                subAction: 'invokeAI',
+                subActionParams: {
+                  messages: [],
+                },
              },
+              replacements: {},
+              size: DEFAULT_SIZE,
+              isEnabledKnowledgeBase: true,
+              isEnabledRAGAlerts: true,
            },
-            replacements: {},
-            size: DEFAULT_SIZE,
-            isEnabledKnowledgeBase: true,
-            isEnabledRAGAlerts: true,
-          },
-        };
+          };

-        // Create an array of executor functions to call in batches
-        // One for each connector/model + agent combination
-        // Hoist `langChainMessages` so they can be batched by dataset.input in the evaluator
-        const agents: AgentExecutorEvaluatorWithMetadata[] = [];
-        connectorIds.forEach((connectorId) => {
-          agentNames.forEach((agentName) => {
-            logger.info(`Creating agent: ${connectorId} + ${agentName}`);
-            const llmType = getLlmType(connectorId, connectors);
-            const connectorName =
-              getConnectorName(connectorId, connectors) ?? '[unknown connector]';
-            const detailedRunName = `${runName} - ${connectorName} + ${agentName}`;
-            agents.push({
-              agentEvaluator: (langChainMessages, exampleId) =>
-                AGENT_EXECUTOR_MAP[agentName]({
-                  actions,
-                  isEnabledKnowledgeBase: true,
-                  assistantTools,
-                  connectorId,
-                  esClient,
-                  elserId,
-                  langChainMessages,
-                  llmType,
-                  logger,
-                  request: skeletonRequest,
-                  kbResource: ESQL_RESOURCE,
-                  telemetry,
-                  traceOptions: {
-                    exampleId,
-                    projectName,
-                    runName: detailedRunName,
-                    evaluationId,
-                    tags: [
-                      'security-assistant-prediction',
-                      ...(connectorName != null ? [connectorName] : []),
-                      runName,
-                    ],
-                    tracers: getLangSmithTracer(detailedRunName, exampleId, logger),
-                  },
-                }),
-              metadata: {
-                connectorName,
-                runName: detailedRunName,
-              },
+          // Create an array of executor functions to call in batches
+          // One for each connector/model + agent combination
+          // Hoist `langChainMessages` so they can be batched by dataset.input in the evaluator
+          const agents: AgentExecutorEvaluatorWithMetadata[] = [];
+          connectorIds.forEach((connectorId) => {
+            agentNames.forEach((agentName) => {
+              logger.info(`Creating agent: ${connectorId} + ${agentName}`);
+              const llmType = getLlmType(connectorId, connectors);
+              const connectorName =
+                getConnectorName(connectorId, connectors) ?? '[unknown connector]';
+              const detailedRunName = `${runName} - ${connectorName} + ${agentName}`;
+              agents.push({
+                agentEvaluator: (langChainMessages, exampleId) =>
+                  AGENT_EXECUTOR_MAP[agentName]({
+                    actions,
+                    isEnabledKnowledgeBase: true,
+                    assistantTools,
+                    connectorId,
+                    esClient,
+                    elserId,
+                    langChainMessages,
+                    llmType,
+                    logger,
+                    request: skeletonRequest,
+                    kbResource: ESQL_RESOURCE,
+                    telemetry,
+                    traceOptions: {
+                      exampleId,
+                      projectName,
+                      runName: detailedRunName,
+                      evaluationId,
+                      tags: [
+                        'security-assistant-prediction',
+                        ...(connectorName != null ? [connectorName] : []),
+                        runName,
+                      ],
+                      tracers: getLangSmithTracer(detailedRunName, exampleId, logger),
+                    },
+                  }),
+                metadata: {
+                  connectorName,
+                  runName: detailedRunName,
+                },
+              });
            });
          });
-        });
-        logger.info(`Agents created: ${agents.length}`);
+          logger.info(`Agents created: ${agents.length}`);

-        // Evaluator Model is optional to support just running predictions
-        const evaluatorModel =
-          evalModel == null || evalModel === ''
-            ? undefined
-            : new ActionsClientLlm({
-                actions,
-                connectorId: evalModel,
-                request: skeletonRequest,
-                logger,
-              });
+          // Evaluator Model is optional to support just running predictions
+          const evaluatorModel =
+            evalModel == null || evalModel === ''
+              ? undefined
+              : new ActionsClientLlm({
+                  actions,
+                  connectorId: evalModel,
+                  request: skeletonRequest,
+                  logger,
+                });

-        const { evaluationResults, evaluationSummary } = await performEvaluation({
-          agentExecutorEvaluators: agents,
-          dataset,
-          evaluationId,
-          evaluatorModel,
-          evaluationPrompt: evalPrompt,
-          evaluationType,
-          logger,
-          runName,
-        });
+          const { evaluationResults, evaluationSummary } = await performEvaluation({
+            agentExecutorEvaluators: agents,
+            dataset,
+            evaluationId,
+            evaluatorModel,
+            evaluationPrompt: evalPrompt,
+            evaluationType,
+            logger,
+            runName,
+          });

-        logger.info(`Writing evaluation results to index: ${outputIndex}`);
-        await setupEvaluationIndex({ esClient, index: outputIndex, logger });
-        await indexEvaluations({
-          esClient,
-          evaluationResults,
-          evaluationSummary,
-          index: outputIndex,
-          logger,
-        });
+          logger.info(`Writing evaluation results to index: ${outputIndex}`);
+          await setupEvaluationIndex({ esClient, index: outputIndex, logger });
+          await indexEvaluations({
+            esClient,
+            evaluationResults,
+            evaluationSummary,
+            index: outputIndex,
+            logger,
+          });

-        return response.ok({
-          body: { evaluationId, success: true },
-        });
-      } catch (err) {
-        logger.error(err);
-        const error = transformError(err);
+          return response.ok({
+            body: { evaluationId, success: true },
+          });
+        } catch (err) {
+          logger.error(err);
+          const error = transformError(err);

-        const resp = buildResponse(response);
-        return resp.error({
-          body: { success: false, error: error.message },
-          statusCode: error.statusCode,
-        });
+          const resp = buildResponse(response);
+          return resp.error({
+            body: { success: false, error: error.message },
+            statusCode: error.statusCode,
+          });
+        }
      }
-    }
-  );
+    );
 };
--- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts
+++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/utils.ts
@ -12,7 +12,7 @@ import type { Logger } from '@kbn/core/server';
 import type { Run } from 'langsmith/schemas';
 import { ToolingLog } from '@kbn/tooling-log';
 import { LangChainTracer } from 'langchain/callbacks';
-import { Dataset } from '../../schemas/evaluate/post_evaluate';
+import { Dataset } from '@kbn/elastic-assistant-common';

 /**
 * Returns the LangChain `llmType` for the given connectorId/connectors
--- a/x-pack/plugins/elastic_assistant/server/schemas/common.ts
+++ b/x-pack/plugins/elastic_assistant/server/schemas/common.ts
@ -14,6 +14,8 @@ import type {
  RouteValidationResultFactory,
  RouteValidationError,
 } from '@kbn/core/server';
+import type { TypeOf, ZodType } from 'zod';
+import { stringifyZodError } from '@kbn/zod-helpers';

 type RequestValidationResult<T> =
  | {
@ -36,3 +38,14 @@ export const buildRouteValidation =
        (validatedInput: A) => validationResult.ok(validatedInput)
      )
    );
+
+export const buildRouteValidationWithZod =
+  <T extends ZodType, A = TypeOf<T>>(schema: T): RouteValidationFunction<A> =>
+  (inputValue: unknown, validationResult: RouteValidationResultFactory) => {
+    const decoded = schema.safeParse(inputValue);
+    if (decoded.success) {
+      return validationResult.ok(decoded.data);
+    } else {
+      return validationResult.badRequest(stringifyZodError(decoded.error));
+    }
+  };
--- a/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts
+++ b/x-pack/plugins/elastic_assistant/server/schemas/evaluate/post_evaluate.ts
@ -1,58 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import * as t from 'io-ts';
-
-/** Validates Output Index starts with `.kibana-elastic-ai-assistant-` */
-const outputIndex = new t.Type<string, string, unknown>(
-  'OutputIndexPrefixed',
-  (input): input is string =>
-    typeof input === 'string' && input.startsWith('.kibana-elastic-ai-assistant-'),
-  (input, context) =>
-    typeof input === 'string' && input.startsWith('.kibana-elastic-ai-assistant-')
-      ? t.success(input)
-      : t.failure(
-          input,
-          context,
-          `Type error: Output Index does not start with '.kibana-elastic-ai-assistant-'`
-        ),
-  t.identity
-);
-
-/** Validates the URL path of a POST request to the `/evaluate` endpoint */
-export const PostEvaluatePathQuery = t.type({
-  agents: t.string,
-  datasetName: t.union([t.string, t.undefined]),
-  evaluationType: t.union([t.string, t.undefined]),
-  evalModel: t.union([t.string, t.undefined]),
-  models: t.string,
-  outputIndex,
-  projectName: t.union([t.string, t.undefined]),
-  runName: t.union([t.string, t.undefined]),
-});
-
-export type PostEvaluatePathQueryInputs = t.TypeOf<typeof PostEvaluatePathQuery>;
-
-export type DatasetItem = t.TypeOf<typeof DatasetItem>;
-export const DatasetItem = t.type({
-  id: t.union([t.string, t.undefined]),
-  input: t.string,
-  reference: t.string,
-  tags: t.union([t.array(t.string), t.undefined]),
-  prediction: t.union([t.string, t.undefined]),
-});
-
-export type Dataset = t.TypeOf<typeof Dataset>;
-export const Dataset = t.array(DatasetItem);
-
-/** Validates the body of a POST request to the `/evaluate` endpoint */
-export const PostEvaluateBody = t.type({
-  dataset: t.union([Dataset, t.undefined]),
-  evalPrompt: t.union([t.string, t.undefined]),
-});
-
-export type PostEvaluateBodyInputs = t.TypeOf<typeof PostEvaluateBody>;
--- a/x-pack/plugins/elastic_assistant/tsconfig.json
+++ b/x-pack/plugins/elastic_assistant/tsconfig.json
@ -35,6 +35,7 @@
    "@kbn/core-analytics-server",
    "@kbn/elastic-assistant-common",
    "@kbn/core-http-router-server-mocks",
+    "@kbn/zod-helpers",
  ],
  "exclude": [
    "target/**/*",