[SecuritySolution][SIEM Migrations] Rule migrations storage (#197032)

## Summary

issue: https://github.com/elastic/security-team/issues/10654?reload=1

Implements the persistence layer for the rule migrations from other
vendors, as part of the SIEM Rule migrations effort.

### Changes

- Schemas created for `SiemRuleMigration` document entity, along with
`ElasticRule` and `OriginalRule`.

- New API `/internal/siem_migrations/rules` was created:
- `POST` -> Receives an array of (original) rules and stores them with
`status: pending` to be processed. Responds with the `migration_id` that
will be used to start the migration background task (implementation
details here: https://github.com/elastic/security-team/issues/10850).
  - `GET` -> (to be implemented later)

- New `SiemMigrationsService` added to the `securitySolution` route
context, to encapsulate all operations related to SIEM migrations (We
start with _rule_ migrations, but there are more "kinds" of SIEM
migrations in the pipeline: _dashboards_, _saved queries_...). It
contains:

- `SiemRuleMigrationsService` to encapsulate all operations related to
SIEM rule migrations.
- `RuleMigrationsDataStream` class to manage the
`.kibana.siem-rule-migrations-<spaceId>` data stream operations using
`DataStreamSpacesAdapter`.
- It exposes a client with abstracted operations that are exposed to the
API routes:
- `create`: indexes an array of _SiemRuleMigration_ documents to the
data stream
- `search`: searches _SiemRuleMigration_ documents by specific terms.

> [!NOTE]  
> Without `siemMigrationsEnabled` experimental flag the new API route
won't be registered, and the `SiemRuleMigrationsService` _setup_ won't
be called, so no index/component template will be installed to ES.

### Testing locally

Enable the flag
```
xpack.securitySolution.enableExperimental: ['siemMigrationsEnabled']
```

<details>
  <summary>Example curl request</summary>

```
curl --location 'http://elastic:changeme@localhost:5601/internal/siem_migrations/rules' \
--header 'kbn-xsrf;' \
--header 'x-elastic-internal-origin: security-solution' \
--header 'elastic-api-version: 1' \
--header 'Content-Type: application/json' \
--data '[
    {
        "id": "f8c325ea-506e-4105-8ccf-da1492e90115",
        "vendor": "splunk",
        "title": "Linux Auditd Add User Account Type",
        "description": "The following analytic detects the suspicious add user account type. This behavior is critical for a SOC to monitor because it may indicate attempts to gain unauthorized access or maintain control over a system. Such actions could be signs of malicious activity. If confirmed, this could lead to serious consequences, including a compromised system, unauthorized access to sensitive data, or even a wider breach affecting the entire network. Detecting and responding to these signs early is essential to prevent potential security incidents.",
        "query": "sourcetype=\"linux:audit\" type=ADD_USER \n| rename hostname as dest \n| stats count min(_time) as firstTime max(_time) as lastTime by exe pid dest res UID type \n| `security_content_ctime(firstTime)` \n| `security_content_ctime(lastTime)`\n| search *",
        "query_language":"spl",
        "mitre_attack_ids": [
            "T1136"
        ]
    },
    {
        "id": "7b87c556-0ca4-47e0-b84c-6cd62a0a3e90",
        "vendor": "splunk",
        "title": "Linux Auditd Change File Owner To Root",
        "description": "The following analytic detects the use of the '\''chown'\'' command to change a file owner to '\''root'\'' on a Linux system. It leverages Linux Auditd telemetry, specifically monitoring command-line executions and process details. This activity is significant as it may indicate an attempt to escalate privileges by adversaries, malware, or red teamers. If confirmed malicious, this action could allow an attacker to gain root-level access, leading to full control over the compromised host and potential persistence within the environment.",
        "query": "`linux_auditd` `linux_auditd_normalized_proctitle_process`\r\n| rename host as dest \r\n| where LIKE (process_exec, \"%chown %root%\") \r\n| stats count min(_time) as firstTime max(_time) as lastTime by process_exec proctitle normalized_proctitle_delimiter dest \r\n| `security_content_ctime(firstTime)` \r\n| `security_content_ctime(lastTime)`\r\n| `linux_auditd_change_file_owner_to_root_filter`",
        "query_language": "spl",
        "mitre_attack_ids": [
            "T1222"
        ]
    }
]'
```
</details>

The newly created documents can be retrieved using Kibana DevTools
console:
```
GET .kibana.siem-rule-migrations-default/_search
```

### Screenshots


![postman_screenshot](https://github.com/user-attachments/assets/9d3852d2-48ef-4955-b621-fdba6b249c65)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -1649,7 +1649,12 @@ x-pack/test/security_solution_api_integration/test_suites/sources @elastic/secur
 /x-pack/test/security_solution_playwright @elastic/security-engineering-productivity
 /x-pack/plugins/security_solution/scripts/run_cypress @MadameSheema @patrykkopycinski @maximpn @banderror
 
-## Security Solution sub teams - Threat Hunting Investigations
+## Security Solution sub teams - Threat Hunting
+
+/x-pack/plugins/security_solution/server/lib/siem_migrations @elastic/security-threat-hunting
+/x-pack/plugins/security_solution/common/siem_migrations @elastic/security-threat-hunting
+
+## Security Solution Threat Hunting areas - Threat Hunting Investigations
 
 /x-pack/plugins/security_solution/common/api/timeline @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/common/search_strategy/timeline @elastic/security-threat-hunting-investigations
@@ -1679,7 +1684,7 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 
 /x-pack/plugins/security_solution/server/lib/timeline @elastic/security-threat-hunting-investigations
 
-## Security Solution sub teams - Threat Hunting Explore
+## Security Solution Threat Hunting areas - Threat Hunting Explore
 /x-pack/plugins/security_solution/common/api/tags @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram @elastic/security-threat-hunting-explore

packages/kbn-data-stream-adapter/src/field_maps/types.ts

@@ -38,8 +38,9 @@ export interface EcsMetadata {
   properties?: Record<string, { type: string }>;
 }
 
-export interface FieldMap {
-  [key: string]: {
+export type FieldMap<T extends string = string> = Record<
+  T,
+  {
     type: string;
     required: boolean;
     array?: boolean;
@@ -53,5 +54,17 @@ export interface FieldMap {
     scaling_factor?: number;
     dynamic?: boolean | 'strict';
     properties?: Record<string, { type: string }>;
-  };
-}
+  }
+>;
+
+// This utility type flattens all the keys of a schema object and its nested objects as a union type.
+// Its purpose is to ensure that the FieldMap keys are always in sync with the schema object.
+// It assumes all optional fields of the schema are required in the field map, they can always be omitted from the resulting type.
+export type SchemaFieldMapKeys<
+  T extends Record<string, unknown>,
+  Key = keyof T
+> = Key extends string
+  ? NonNullable<T[Key]> extends Record<string, unknown>
+    ? `${Key}` | `${Key}.${SchemaFieldMapKeys<NonNullable<T[Key]>>}`
+    : `${Key}`
+  : never;

x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts

@@ -361,6 +361,11 @@ import type {
   ResolveTimelineRequestQueryInput,
   ResolveTimelineResponse,
 } from './timeline/resolve_timeline/resolve_timeline_route.gen';
+import type {
+  CreateRuleMigrationRequestBodyInput,
+  CreateRuleMigrationResponse,
+  GetRuleMigrationResponse,
+} from '../siem_migrations/model/api/rules/rules_migration.gen';
 
 export interface ClientOptions {
   kbnClient: KbnClient;
@@ -655,6 +660,22 @@ If a record already exists for the specified entity, that record is overwritten
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  /**
+   * Creates a new SIEM rules migration using the original vendor rules provided
+   */
+  async createRuleMigration(props: CreateRuleMigrationProps) {
+    this.log.info(`${new Date().toISOString()} Calling API CreateRuleMigration`);
+    return this.kbnClient
+      .request<CreateRuleMigrationResponse>({
+        path: '/internal/siem_migrations/rules',
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '1',
+        },
+        method: 'POST',
+        body: props.body,
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   /**
    * Create a new Timeline or Timeline template.
    */
@@ -1396,6 +1417,21 @@ finalize it.
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  /**
+   * Retrieves the rule migrations stored in the system
+   */
+  async getRuleMigration() {
+    this.log.info(`${new Date().toISOString()} Calling API GetRuleMigration`);
+    return this.kbnClient
+      .request<GetRuleMigrationResponse>({
+        path: '/internal/siem_migrations/rules',
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '1',
+        },
+        method: 'GET',
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   /**
    * Get the details of an existing saved Timeline or Timeline template.
    */
@@ -2039,6 +2075,9 @@ export interface CreateAssetCriticalityRecordProps {
 export interface CreateRuleProps {
   body: CreateRuleRequestBodyInput;
 }
+export interface CreateRuleMigrationProps {
+  body: CreateRuleMigrationRequestBodyInput;
+}
 export interface CreateTimelinesProps {
   body: CreateTimelinesRequestBodyInput;
 }

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -235,6 +235,11 @@ export const allowedExperimentalValues = Object.freeze({
    * can be disabled if necessary in a given environment.
    */
   entityStoreDisabled: false,
+
+  /**
+   * Enables the siem migrations feature
+   */
+  siemMigrationsEnabled: false,
 });
 
 type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;

x-pack/plugins/security_solution/common/siem_migrations/constants.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const SIEM_MIGRATIONS_PATH = '/internal/siem_migrations' as const;
+export const SIEM_RULE_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/rules` as const;
+
+export enum SiemMigrationsStatus {
+  PENDING = 'pending',
+  PROCESSING = 'processing',
+  FINISHED = 'finished',
+  ERROR = 'error',
+}

x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Common SIEM Migrations Attributes
+ *   version: not applicable
+ */
+
+import { z } from '@kbn/zod';
+
+/**
+ * The GenAI connector id to use.
+ */
+export type ConnectorId = z.infer<typeof ConnectorId>;
+export const ConnectorId = z.string();
+
+/**
+ * The LangSmith options object.
+ */
+export type LangSmithOptions = z.infer<typeof LangSmithOptions>;
+export const LangSmithOptions = z.object({
+  /**
+   * The project name.
+   */
+  project_name: z.string(),
+  /**
+   * The API key to use for tracing.
+   */
+  api_key: z.string(),
+});

x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml

@@ -0,0 +1,24 @@
+openapi: 3.0.3
+info:
+  title: Common SIEM Migrations Attributes
+  version: 'not applicable'
+paths: {}
+components:
+  x-codegen-enabled: true
+  schemas:
+    ConnectorId:
+      type: string
+      description: The GenAI connector id to use.
+    LangSmithOptions:
+      type: object
+      description: The LangSmith options object.
+      required:
+        - project_name
+        - api_key
+      properties:
+        project_name:
+          type: string
+          description: The project name.
+        api_key:
+          type: string
+          description: The API key to use for tracing.

x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: SIEM Rules Migration API endpoint
+ *   version: 1
+ */
+
+import { z } from '@kbn/zod';
+
+import { OriginalRule, RuleMigration } from '../../rule_migration.gen';
+
+export type CreateRuleMigrationRequestBody = z.infer<typeof CreateRuleMigrationRequestBody>;
+export const CreateRuleMigrationRequestBody = z.array(OriginalRule);
+export type CreateRuleMigrationRequestBodyInput = z.input<typeof CreateRuleMigrationRequestBody>;
+
+export type CreateRuleMigrationResponse = z.infer<typeof CreateRuleMigrationResponse>;
+export const CreateRuleMigrationResponse = z.object({
+  /**
+   * The migration id created.
+   */
+  migration_id: z.string(),
+});
+
+export type GetRuleMigrationResponse = z.infer<typeof GetRuleMigrationResponse>;
+export const GetRuleMigrationResponse = z.array(RuleMigration);

x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml

@@ -0,0 +1,52 @@
+openapi: 3.0.3
+info:
+  title: SIEM Rules Migration API endpoint
+  version: '1'
+paths:
+  /internal/siem_migrations/rules:
+    post:
+      summary: Creates a new rule migration
+      operationId: CreateRuleMigration
+      x-codegen-enabled: true
+      description: Creates a new SIEM rules migration using the original vendor rules provided
+      tags:
+        - SIEM Migrations
+        - Rule Migrations
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                $ref: '../../rule_migration.schema.yaml#/components/schemas/OriginalRule'
+      responses:
+        200:
+          description: Indicates migration have been created correctly.
+          content:
+            application/json:
+              schema:
+                type: object
+                required:
+                  - migration_id
+                properties:
+                  migration_id:
+                    type: string
+                    description: The migration id created.
+    get:
+      summary: Retrieves rule migrations
+      operationId: GetRuleMigration
+      x-codegen-enabled: true
+      description: Retrieves the rule migrations stored in the system
+      tags:
+        - SIEM Migrations
+        - Rule Migrations
+      responses:
+        200:
+          description: Indicates rule migrations have been retrieved correctly.
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigration'

x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Common Splunk Rules Attributes
+ *   version: not applicable
+ */
+
+import { z } from '@kbn/zod';
+
+/**
+ * The original rule to migrate.
+ */
+export type OriginalRule = z.infer<typeof OriginalRule>;
+export const OriginalRule = z.object({
+  /**
+   * The original rule id.
+   */
+  id: z.string(),
+  /**
+   * The original rule vendor identifier.
+   */
+  vendor: z.literal('splunk'),
+  /**
+   * The original rule name.
+   */
+  title: z.string(),
+  /**
+   * The original rule description.
+   */
+  description: z.string(),
+  /**
+   * The original rule query.
+   */
+  query: z.string(),
+  /**
+   * The original rule query language.
+   */
+  query_language: z.string(),
+  /**
+   * The original rule Mitre Attack technique IDs.
+   */
+  mitre_attack_ids: z.array(z.string()).optional(),
+});
+
+/**
+ * The migrated elastic rule.
+ */
+export type ElasticRule = z.infer<typeof ElasticRule>;
+export const ElasticRule = z.object({
+  /**
+   * The migrated rule title.
+   */
+  title: z.string(),
+  /**
+   * The migrated rule description.
+   */
+  description: z.string().optional(),
+  /**
+   * The migrated rule severity.
+   */
+  severity: z.string().optional(),
+  /**
+   * The translated elastic query.
+   */
+  query: z.string(),
+  /**
+   * The translated elastic query language.
+   */
+  query_language: z.literal('esql').default('esql'),
+  /**
+   * The Elastic prebuilt rule id matched.
+   */
+  prebuilt_rule_id: z.string().optional(),
+  /**
+   * The Elastic rule id installed as a result.
+   */
+  id: z.string().optional(),
+});
+
+/**
+ * The rule migration document object.
+ */
+export type RuleMigration = z.infer<typeof RuleMigration>;
+export const RuleMigration = z.object({
+  /**
+   * The moment of creation
+   */
+  '@timestamp': z.string(),
+  /**
+   * The migration id.
+   */
+  migration_id: z.string(),
+  original_rule: OriginalRule,
+  elastic_rule: ElasticRule.optional(),
+  /**
+   * The translation state.
+   */
+  translation_state: z.enum(['complete', 'partial', 'untranslatable']).optional(),
+  /**
+   * The status of the rule migration.
+   */
+  status: z.enum(['pending', 'processing', 'finished', 'error']).default('pending'),
+  /**
+   * The comments for the migration including a summary from the LLM in markdown.
+   */
+  comments: z.array(z.string()).optional(),
+  /**
+   * The moment of the last update
+   */
+  updated_at: z.string().optional(),
+  /**
+   * The user who last updated the migration
+   */
+  updated_by: z.string().optional(),
+});

x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml

@@ -0,0 +1,124 @@
+openapi: 3.0.3
+info:
+  title: Common Splunk Rules Attributes
+  version: 'not applicable'
+paths: {}
+components:
+  x-codegen-enabled: true
+  schemas:
+    OriginalRule:
+      type: object
+      description: The original rule to migrate.
+      required:
+        - id
+        - vendor
+        - title
+        - description
+        - query
+        - query_language
+      properties:
+        id:
+          type: string
+          description: The original rule id.
+        vendor:
+          type: string
+          description: The original rule vendor identifier.
+          enum:
+            - splunk
+        title:
+          type: string
+          description: The original rule name.
+        description:
+          type: string
+          description: The original rule description.
+        query:
+          type: string
+          description: The original rule query.
+        query_language:
+          type: string
+          description: The original rule query language.
+        mitre_attack_ids:
+          type: array
+          items:
+            type: string
+          description: The original rule Mitre Attack technique IDs.
+
+    ElasticRule:
+      type: object
+      description: The migrated elastic rule.
+      required:
+        - title
+        - query
+        - query_language
+      properties:
+        title:
+          type: string
+          description: The migrated rule title.
+        description:
+          type: string
+          description: The migrated rule description.
+        severity:
+          type: string
+          description: The migrated rule severity.
+        query:
+          type: string
+          description: The translated elastic query.
+        query_language:
+          type: string
+          description: The translated elastic query language.
+          enum:
+            - esql
+          default: esql
+        prebuilt_rule_id:
+          type: string
+          description: The Elastic prebuilt rule id matched.
+        id:
+          type: string
+          description: The Elastic rule id installed as a result.
+
+    RuleMigration:
+      type: object
+      description: The rule migration document object.
+      required:
+        - '@timestamp'
+        - migration_id
+        - original_rule
+        - status
+      properties:
+        "@timestamp":
+          type: string
+          description: The moment of creation
+        migration_id:
+          type: string
+          description: The migration id.
+        original_rule:
+          $ref: '#/components/schemas/OriginalRule'
+        elastic_rule:
+          $ref: '#/components/schemas/ElasticRule'
+        translation_state:
+          type: string
+          description: The translation state.
+          enum:
+            - complete
+            - partial
+            - untranslatable
+        status:
+          type: string
+          description: The status of the rule migration.
+          enum: # should match SiemMigrationsStatus enum at ../constants.ts
+            - pending
+            - processing
+            - finished
+            - error
+          default: pending
+        comments:
+          type: array
+          description: The comments for the migration including a summary from the LLM in markdown.
+          items:
+            type: string
+        updated_at:
+          type: string
+          description: The moment of the last update
+        updated_by:
+          type: string
+          description: The user who last updated the migration

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts

@@ -42,6 +42,7 @@ import { auditLoggerMock } from '@kbn/security-plugin/server/audit/mocks';
 import { detectionRulesClientMock } from '../../rule_management/logic/detection_rules_client/__mocks__/detection_rules_client';
 import { packageServiceMock } from '@kbn/fleet-plugin/server/services/epm/package_service.mock';
 import type { EndpointInternalFleetServicesInterface } from '../../../../endpoint/services/fleet';
+import { siemMigrationsServiceMock } from '../../../siem_migrations/__mocks__/mocks';
 
 export const createMockClients = () => {
   const core = coreMock.createRequestHandlerContext();
@@ -78,6 +79,7 @@ export const createMockClients = () => {
     internalFleetServices: {
       packages: packageServiceMock.createClient(),
     },
+    siemMigrationsClient: siemMigrationsServiceMock.createClient(),
   };
 };
 
@@ -163,6 +165,7 @@ const createSecuritySolutionRequestContextMock = (
     getAssetCriticalityDataClient: jest.fn(() => clients.assetCriticalityDataClient),
     getAuditLogger: jest.fn(() => mockAuditLogger),
     getEntityStoreDataClient: jest.fn(() => clients.entityStoreDataClient),
+    getSiemMigrationsClient: jest.fn(() => clients.siemMigrationsClient),
   };
 };
 

x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createRuleMigrationClient } from '../rules/__mocks__/mocks';
+
+const createClient = () => ({ rules: createRuleMigrationClient() });
+
+export const mockSetup = jest.fn().mockResolvedValue(undefined);
+export const mockCreateClient = jest.fn().mockReturnValue(createClient());
+export const mockStop = jest.fn();
+
+export const siemMigrationsServiceMock = {
+  create: () =>
+    jest.fn().mockImplementation(() => ({
+      setup: mockSetup,
+      createClient: mockCreateClient,
+      stop: mockStop,
+    })),
+  createClient: () => createClient(),
+};

x-pack/plugins/security_solution/server/lib/siem_migrations/routes.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import { registerSiemRuleMigrationsRoutes } from './rules/api';
+import type { SecuritySolutionPluginRouter } from '../../types';
+import type { ConfigType } from '../../config';
+
+export const registerSiemMigrationsRoutes = (
+  router: SecuritySolutionPluginRouter,
+  config: ConfigType,
+  logger: Logger
+) => {
+  if (config.experimentalFeatures.siemMigrationsEnabled) {
+    registerSiemRuleMigrationsRoutes(router, logger);
+  }
+};

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SiemRuleMigrationsClient } from '../types';
+
+export const createRuleMigrationClient = (): SiemRuleMigrationsClient => ({
+  create: jest.fn().mockResolvedValue({ success: true }),
+  search: jest.fn().mockResolvedValue([]),
+});
+
+export const mockSetup = jest.fn();
+export const mockGetClient = jest.fn().mockReturnValue(createRuleMigrationClient());
+
+export const MockSiemRuleMigrationsService = jest.fn().mockImplementation(() => ({
+  setup: mockSetup,
+  getClient: mockGetClient,
+}));

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_service.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MockSiemRuleMigrationsService } from './mocks';
+export const SiemRuleMigrationsService = MockSiemRuleMigrationsService;

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IKibanaResponse, Logger } from '@kbn/core/server';
+import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
+import { v4 as uuidV4 } from 'uuid';
+import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen';
+import type { CreateRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen';
+import { CreateRuleMigrationRequestBody } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen';
+import {
+  SIEM_RULE_MIGRATIONS_PATH,
+  SiemMigrationsStatus,
+} from '../../../../../common/siem_migrations/constants';
+import type { SecuritySolutionPluginRouter } from '../../../../types';
+
+export const registerSiemRuleMigrationsCreateRoute = (
+  router: SecuritySolutionPluginRouter,
+  logger: Logger
+) => {
+  router.versioned
+    .post({
+      path: SIEM_RULE_MIGRATIONS_PATH,
+      access: 'internal',
+      options: { tags: ['access:securitySolution'] },
+    })
+    .addVersion(
+      {
+        version: '1',
+        validate: {
+          request: { body: buildRouteValidationWithZod(CreateRuleMigrationRequestBody) },
+        },
+      },
+      async (context, req, res): Promise<IKibanaResponse<CreateRuleMigrationResponse>> => {
+        const originalRules = req.body;
+        try {
+          const ctx = await context.resolve(['core', 'actions', 'securitySolution']);
+
+          const siemMigrationClient = ctx.securitySolution.getSiemMigrationsClient();
+
+          const migrationId = uuidV4();
+          const timestamp = new Date().toISOString();
+
+          const ruleMigrations = originalRules.map<RuleMigration>((originalRule) => ({
+            '@timestamp': timestamp,
+            migration_id: migrationId,
+            original_rule: originalRule,
+            status: SiemMigrationsStatus.PENDING,
+          }));
+          await siemMigrationClient.rules.create(ruleMigrations);
+
+          return res.ok({ body: { migration_id: migrationId } });
+        } catch (err) {
+          logger.error(err);
+          return res.badRequest({
+            body: err.message,
+          });
+        }
+      }
+    );
+};

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import type { SecuritySolutionPluginRouter } from '../../../../types';
+import { registerSiemRuleMigrationsCreateRoute } from './create';
+
+export const registerSiemRuleMigrationsRoutes = (
+  router: SecuritySolutionPluginRouter,
+  logger: Logger
+) => {
+  registerSiemRuleMigrationsCreateRoute(router, logger);
+};

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const mockIndexName = 'mocked_data_stream_name';
+export const mockInstall = jest.fn().mockResolvedValue(undefined);
+export const mockInstallSpace = jest.fn().mockResolvedValue(mockIndexName);
+
+export const MockRuleMigrationsDataStream = jest.fn().mockImplementation(() => ({
+  install: mockInstall,
+  installSpace: mockInstallSpace,
+}));

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/rule_migrations_data_stream.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MockRuleMigrationsDataStream } from './mocks';
+export const RuleMigrationsDataStream = MockRuleMigrationsDataStream;

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { RuleMigrationsDataStream } from './rule_migrations_data_stream';
+import { Subject } from 'rxjs';
+import type { InstallParams } from '@kbn/data-stream-adapter';
+import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter';
+import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
+import { loggerMock } from '@kbn/logging-mocks';
+
+jest.mock('@kbn/data-stream-adapter');
+
+const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest.MockedClass<
+  typeof DataStreamSpacesAdapter
+>;
+
+const esClient = elasticsearchServiceMock.createStart().client.asInternalUser;
+
+describe('SiemRuleMigrationsDataStream', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('constructor', () => {
+    it('should create DataStreamSpacesAdapter', () => {
+      new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      expect(MockedDataStreamSpacesAdapter).toHaveBeenCalledTimes(1);
+    });
+
+    it('should create component templates', () => {
+      new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      expect(dataStreamSpacesAdapter.setComponentTemplate).toHaveBeenCalledWith(
+        expect.objectContaining({ name: '.kibana.siem-rule-migrations' })
+      );
+    });
+
+    it('should create index templates', () => {
+      new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      expect(dataStreamSpacesAdapter.setIndexTemplate).toHaveBeenCalledWith(
+        expect.objectContaining({ name: '.kibana.siem-rule-migrations' })
+      );
+    });
+  });
+
+  describe('install', () => {
+    it('should install data stream', async () => {
+      const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const params: InstallParams = {
+        esClient,
+        logger: loggerMock.create(),
+        pluginStop$: new Subject(),
+      };
+      await dataStream.install(params);
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(params);
+    });
+
+    it('should log error', async () => {
+      const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const params: InstallParams = {
+        esClient,
+        logger: loggerMock.create(),
+        pluginStop$: new Subject(),
+      };
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      const error = new Error('test-error');
+      (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error);
+
+      await dataStream.install(params);
+      expect(params.logger.error).toHaveBeenCalledWith(expect.any(String), error);
+    });
+  });
+
+  describe('installSpace', () => {
+    it('should install space data stream', async () => {
+      const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const params: InstallParams = {
+        esClient,
+        logger: loggerMock.create(),
+        pluginStop$: new Subject(),
+      };
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      (dataStreamSpacesAdapter.install as jest.Mock).mockResolvedValueOnce(undefined);
+
+      await dataStream.install(params);
+      await dataStream.installSpace('space1');
+
+      expect(dataStreamSpacesAdapter.getInstalledSpaceName).toHaveBeenCalledWith('space1');
+      expect(dataStreamSpacesAdapter.installSpace).toHaveBeenCalledWith('space1');
+    });
+
+    it('should not install space data stream if install not executed', async () => {
+      const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      await expect(dataStream.installSpace('space1')).rejects.toThrowError();
+    });
+
+    it('should throw error if main install had error', async () => {
+      const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const params: InstallParams = {
+        esClient,
+        logger: loggerMock.create(),
+        pluginStop$: new Subject(),
+      };
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      const error = new Error('test-error');
+      (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error);
+      await dataStream.install(params);
+
+      await expect(dataStream.installSpace('space1')).rejects.toThrowError(error);
+    });
+  });
+});

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DataStreamSpacesAdapter, type InstallParams } from '@kbn/data-stream-adapter';
+import { ruleMigrationsFieldMap } from './rule_migrations_field_map';
+
+const TOTAL_FIELDS_LIMIT = 2500;
+
+const DATA_STREAM_NAME = '.kibana.siem-rule-migrations';
+const ECS_COMPONENT_TEMPLATE_NAME = 'ecs';
+
+export class RuleMigrationsDataStream {
+  private readonly dataStream: DataStreamSpacesAdapter;
+  private installPromise?: Promise<void>;
+
+  constructor({ kibanaVersion }: { kibanaVersion: string }) {
+    this.dataStream = new DataStreamSpacesAdapter(DATA_STREAM_NAME, {
+      kibanaVersion,
+      totalFieldsLimit: TOTAL_FIELDS_LIMIT,
+    });
+    this.dataStream.setComponentTemplate({
+      name: DATA_STREAM_NAME,
+      fieldMap: ruleMigrationsFieldMap,
+    });
+
+    this.dataStream.setIndexTemplate({
+      name: DATA_STREAM_NAME,
+      componentTemplateRefs: [DATA_STREAM_NAME, ECS_COMPONENT_TEMPLATE_NAME],
+    });
+  }
+
+  async install(params: InstallParams) {
+    try {
+      this.installPromise = this.dataStream.install(params);
+      await this.installPromise;
+    } catch (err) {
+      params.logger.error(`Error installing siem rule migrations data stream. ${err.message}`, err);
+    }
+  }
+
+  async installSpace(spaceId: string): Promise<string> {
+    if (!this.installPromise) {
+      throw new Error('Siem rule migrations data stream not installed');
+    }
+    // wait for install to complete, may reject if install failed, routes should handle this
+    await this.installPromise;
+    let dataStreamName = await this.dataStream.getInstalledSpaceName(spaceId);
+    if (!dataStreamName) {
+      dataStreamName = await this.dataStream.installSpace(spaceId);
+    }
+    return dataStreamName;
+  }
+}

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { FieldMap, SchemaFieldMapKeys } from '@kbn/data-stream-adapter';
+import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen';
+
+export const ruleMigrationsFieldMap: FieldMap<SchemaFieldMapKeys<RuleMigration>> = {
+  '@timestamp': { type: 'date', required: false },
+  migration_id: { type: 'keyword', required: true },
+  status: { type: 'keyword', required: true },
+  original_rule: { type: 'nested', required: true },
+  'original_rule.vendor': { type: 'keyword', required: true },
+  'original_rule.id': { type: 'keyword', required: true },
+  'original_rule.title': { type: 'keyword', required: true },
+  'original_rule.description': { type: 'keyword', required: false },
+  'original_rule.query': { type: 'text', required: true },
+  'original_rule.query_language': { type: 'keyword', required: true },
+  'original_rule.mitre_attack_ids': { type: 'keyword', array: true, required: false },
+  elastic_rule: { type: 'nested', required: false },
+  'elastic_rule.title': { type: 'keyword', required: true },
+  'elastic_rule.query': { type: 'text', required: true },
+  'elastic_rule.query_language': { type: 'keyword', required: true },
+  'elastic_rule.description': { type: 'keyword', required: false },
+  'elastic_rule.severity': { type: 'keyword', required: false },
+  'elastic_rule.prebuilt_rule_id': { type: 'keyword', required: false },
+  'elastic_rule.id': { type: 'keyword', required: false },
+  translation_state: { type: 'keyword', required: false },
+  comments: { type: 'text', array: true, required: false },
+  updated_at: { type: 'date', required: false },
+  updated_by: { type: 'keyword', required: false },
+};

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  loggingSystemMock,
+  elasticsearchServiceMock,
+  httpServerMock,
+} from '@kbn/core/server/mocks';
+import { SiemRuleMigrationsService } from './siem_rule_migrations_service';
+import { Subject } from 'rxjs';
+import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen';
+import {
+  MockRuleMigrationsDataStream,
+  mockInstall,
+  mockInstallSpace,
+  mockIndexName,
+} from './data_stream/__mocks__/mocks';
+import type { KibanaRequest } from '@kbn/core/server';
+
+jest.mock('./data_stream/rule_migrations_data_stream');
+
+describe('SiemRuleMigrationsService', () => {
+  let ruleMigrationsService: SiemRuleMigrationsService;
+  const kibanaVersion = '8.16.0';
+
+  const esClusterClient = elasticsearchServiceMock.createClusterClient();
+  const logger = loggingSystemMock.createLogger();
+  const pluginStop$ = new Subject<void>();
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    ruleMigrationsService = new SiemRuleMigrationsService(logger, kibanaVersion);
+  });
+
+  it('should instantiate the rule migrations data stream adapter', () => {
+    expect(MockRuleMigrationsDataStream).toHaveBeenCalledWith({ kibanaVersion });
+  });
+
+  describe('when setup is called', () => {
+    it('should set esClusterClient and call dataStreamAdapter.install', () => {
+      ruleMigrationsService.setup({ esClusterClient, pluginStop$ });
+
+      expect(mockInstall).toHaveBeenCalledWith({
+        esClient: esClusterClient.asInternalUser,
+        logger,
+        pluginStop$,
+      });
+    });
+  });
+
+  describe('when getClient is called', () => {
+    let request: KibanaRequest;
+    beforeEach(() => {
+      request = httpServerMock.createKibanaRequest();
+    });
+
+    describe('without setup', () => {
+      it('should throw an error', () => {
+        expect(() => {
+          ruleMigrationsService.getClient({ spaceId: 'default', request });
+        }).toThrowError('ES client not available, please call setup first');
+      });
+    });
+
+    describe('with setup', () => {
+      beforeEach(() => {
+        ruleMigrationsService.setup({ esClusterClient, pluginStop$ });
+      });
+
+      it('should call installSpace', () => {
+        ruleMigrationsService.getClient({ spaceId: 'default', request });
+
+        expect(mockInstallSpace).toHaveBeenCalledWith('default');
+      });
+
+      it('should return a client with create and search methods after setup', () => {
+        const client = ruleMigrationsService.getClient({ spaceId: 'default', request });
+
+        expect(client).toHaveProperty('create');
+        expect(client).toHaveProperty('search');
+      });
+
+      it('should call ES bulk create API with the correct parameters with create is called', async () => {
+        const client = ruleMigrationsService.getClient({ spaceId: 'default', request });
+
+        const ruleMigrations = [{ migration_id: 'dummy_migration_id' } as RuleMigration];
+        await client.create(ruleMigrations);
+
+        expect(esClusterClient.asScoped().asCurrentUser.bulk).toHaveBeenCalledWith(
+          expect.objectContaining({
+            body: [{ create: { _index: mockIndexName } }, { migration_id: 'dummy_migration_id' }],
+            refresh: 'wait_for',
+          })
+        );
+      });
+
+      it('should call ES search API with the correct parameters with search is called', async () => {
+        const client = ruleMigrationsService.getClient({ spaceId: 'default', request });
+
+        const term = { migration_id: 'dummy_migration_id' };
+        await client.search(term);
+
+        expect(esClusterClient.asScoped().asCurrentUser.search).toHaveBeenCalledWith(
+          expect.objectContaining({
+            index: mockIndexName,
+            body: { query: { term } },
+          })
+        );
+      });
+    });
+  });
+});

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IClusterClient, Logger } from '@kbn/core/server';
+import { RuleMigrationsDataStream } from './data_stream/rule_migrations_data_stream';
+import type {
+  SiemRuleMigrationsClient,
+  SiemRulesMigrationsSetupParams,
+  SiemRuleMigrationsGetClientParams,
+} from './types';
+
+export class SiemRuleMigrationsService {
+  private dataStreamAdapter: RuleMigrationsDataStream;
+  private esClusterClient?: IClusterClient;
+
+  constructor(private logger: Logger, kibanaVersion: string) {
+    this.dataStreamAdapter = new RuleMigrationsDataStream({ kibanaVersion });
+  }
+
+  setup({ esClusterClient, ...params }: SiemRulesMigrationsSetupParams) {
+    this.esClusterClient = esClusterClient;
+    const esClient = esClusterClient.asInternalUser;
+    this.dataStreamAdapter.install({ ...params, esClient, logger: this.logger }).catch((err) => {
+      this.logger.error(`Error installing data stream for rule migrations: ${err.message}`);
+      throw err;
+    });
+  }
+
+  getClient({ spaceId, request }: SiemRuleMigrationsGetClientParams): SiemRuleMigrationsClient {
+    if (!this.esClusterClient) {
+      throw new Error('ES client not available, please call setup first');
+    }
+    // Installs the data stream for the specific space. it will only install if it hasn't been installed yet.
+    // The adapter stores the data stream name promise, it will return it directly when the data stream is known to be installed.
+    const dataStreamNamePromise = this.dataStreamAdapter.installSpace(spaceId);
+
+    const esClient = this.esClusterClient.asScoped(request).asCurrentUser;
+    return {
+      create: async (ruleMigrations) => {
+        const _index = await dataStreamNamePromise;
+        return esClient.bulk({
+          refresh: 'wait_for',
+          body: ruleMigrations.flatMap((ruleMigration) => [{ create: { _index } }, ruleMigration]),
+        });
+      },
+      search: async (term) => {
+        const index = await dataStreamNamePromise;
+        return esClient.search({ index, body: { query: { term } } });
+      },
+    };
+  }
+}

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { BulkResponse, SearchResponse } from '@elastic/elasticsearch/lib/api/types';
+import type { IClusterClient, KibanaRequest } from '@kbn/core/server';
+import type { Subject } from 'rxjs';
+import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen';
+
+export interface SiemRulesMigrationsSetupParams {
+  esClusterClient: IClusterClient;
+  pluginStop$: Subject<void>;
+  tasksTimeoutMs?: number;
+}
+
+export interface SiemRuleMigrationsGetClientParams {
+  request: KibanaRequest;
+  spaceId: string;
+}
+
+export interface RuleMigrationSearchParams {
+  migration_id?: string;
+}
+export interface SiemRuleMigrationsClient {
+  create: (body: RuleMigration[]) => Promise<BulkResponse>;
+  search: (params: RuleMigrationSearchParams) => Promise<SearchResponse>;
+}

x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  loggingSystemMock,
+  elasticsearchServiceMock,
+  httpServerMock,
+} from '@kbn/core/server/mocks';
+import { SiemMigrationsService } from './siem_migrations_service';
+import { MockSiemRuleMigrationsService, mockSetup, mockGetClient } from './rules/__mocks__/mocks';
+import type { ConfigType } from '../../config';
+
+jest.mock('./rules/siem_rule_migrations_service');
+
+const mockReplaySubject$ = { next: jest.fn(), complete: jest.fn() };
+jest.mock('rxjs', () => ({
+  ...jest.requireActual('rxjs'),
+  ReplaySubject: jest.fn().mockImplementation(() => mockReplaySubject$),
+}));
+
+describe('SiemMigrationsService', () => {
+  let siemMigrationsService: SiemMigrationsService;
+  const kibanaVersion = '8.16.0';
+
+  const esClusterClient = elasticsearchServiceMock.createClusterClient();
+  const logger = loggingSystemMock.createLogger();
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('with siemMigrationsEnabled flag', () => {
+    beforeEach(() => {
+      siemMigrationsService = new SiemMigrationsService(
+        { experimentalFeatures: { siemMigrationsEnabled: true } } as ConfigType,
+        logger,
+        kibanaVersion
+      );
+    });
+
+    it('should instantiate the rule migrations service', async () => {
+      expect(MockSiemRuleMigrationsService).toHaveBeenCalledWith(logger, kibanaVersion);
+    });
+
+    describe('when setup is called', () => {
+      it('should call siemRuleMigrationsService setup', async () => {
+        siemMigrationsService.setup({ esClusterClient, tasksTimeoutMs: 100 });
+
+        expect(mockSetup).toHaveBeenCalledWith({
+          esClusterClient,
+          tasksTimeoutMs: 100,
+          pluginStop$: mockReplaySubject$,
+        });
+      });
+    });
+
+    describe('when createClient is called', () => {
+      it('should create rules client', async () => {
+        const request = httpServerMock.createKibanaRequest();
+        siemMigrationsService.createClient({ spaceId: 'default', request });
+        expect(mockGetClient).toHaveBeenCalledWith({ spaceId: 'default', request });
+      });
+    });
+
+    describe('when stop is called', () => {
+      it('should trigger the pluginStop subject', async () => {
+        siemMigrationsService.stop();
+        expect(mockReplaySubject$.next).toHaveBeenCalled();
+        expect(mockReplaySubject$.complete).toHaveBeenCalled();
+      });
+    });
+  });
+
+  describe('without siemMigrationsEnabled flag', () => {
+    beforeEach(() => {
+      siemMigrationsService = new SiemMigrationsService(
+        { experimentalFeatures: { siemMigrationsEnabled: false } } as ConfigType,
+        logger,
+        kibanaVersion
+      );
+    });
+
+    it('should instantiate the rule migrations service', async () => {
+      expect(MockSiemRuleMigrationsService).toHaveBeenCalledWith(logger, kibanaVersion);
+    });
+
+    describe('when setup is called', () => {
+      it('should not call siemRuleMigrationsService setup', async () => {
+        siemMigrationsService.setup({ esClusterClient, tasksTimeoutMs: 100 });
+        expect(mockSetup).not.toHaveBeenCalled();
+      });
+    });
+  });
+});

x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import { ReplaySubject, type Subject } from 'rxjs';
+import type { ConfigType } from '../../config';
+import { SiemRuleMigrationsService } from './rules/siem_rule_migrations_service';
+import type {
+  SiemMigrationsClient,
+  SiemMigrationsSetupParams,
+  SiemMigrationsGetClientParams,
+} from './types';
+
+export class SiemMigrationsService {
+  private pluginStop$: Subject<void>;
+  private rules: SiemRuleMigrationsService;
+
+  constructor(private config: ConfigType, logger: Logger, kibanaVersion: string) {
+    this.pluginStop$ = new ReplaySubject(1);
+    this.rules = new SiemRuleMigrationsService(logger, kibanaVersion);
+  }
+
+  setup(params: SiemMigrationsSetupParams) {
+    if (this.config.experimentalFeatures.siemMigrationsEnabled) {
+      this.rules.setup({ ...params, pluginStop$: this.pluginStop$ });
+    }
+  }
+
+  createClient(params: SiemMigrationsGetClientParams): SiemMigrationsClient {
+    return {
+      rules: this.rules.getClient(params),
+    };
+  }
+
+  stop() {
+    this.pluginStop$.next();
+    this.pluginStop$.complete();
+  }
+}

x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IClusterClient } from '@kbn/core/server';
+import type { SiemRuleMigrationsClient, SiemRuleMigrationsGetClientParams } from './rules/types';
+
+export interface SiemMigrationsSetupParams {
+  esClusterClient: IClusterClient;
+  tasksTimeoutMs?: number;
+}
+
+export type SiemMigrationsGetClientParams = SiemRuleMigrationsGetClientParams;
+
+export interface SiemMigrationsClient {
+  rules: SiemRuleMigrationsClient;
+}

x-pack/plugins/security_solution/server/plugin.ts

@@ -123,6 +123,7 @@ import { getAssistantTools } from './assistant/tools';
 import { turnOffAgentPolicyFeatures } from './endpoint/migrations/turn_off_agent_policy_features';
 import { getCriblPackagePolicyPostCreateOrUpdateCallback } from './security_integrations';
 import { scheduleEntityAnalyticsMigration } from './lib/entity_analytics/migrations';
+import { SiemMigrationsService } from './lib/siem_migrations/siem_migrations_service';
 
 export type { SetupPlugins, StartPlugins, PluginSetup, PluginStart } from './plugin_contract';
 
@@ -135,6 +136,7 @@ export class Plugin implements ISecuritySolutionPlugin {
 
   private readonly ruleMonitoringService: IRuleMonitoringService;
   private readonly endpointAppContextService = new EndpointAppContextService();
+  private readonly siemMigrationsService: SiemMigrationsService;
   private readonly telemetryReceiver: ITelemetryReceiver;
   private readonly telemetryEventsSender: ITelemetryEventsSender;
   private readonly asyncTelemetryEventsSender: IAsyncTelemetryEventsSender;
@@ -160,6 +162,11 @@ export class Plugin implements ISecuritySolutionPlugin {
       this.logger,
       this.config.experimentalFeatures
     );
+    this.siemMigrationsService = new SiemMigrationsService(
+      this.config,
+      this.logger,
+      this.pluginContext.env.packageInfo.version
+    );
 
     this.ruleMonitoringService = createRuleMonitoringService(this.config, this.logger);
     this.telemetryEventsSender = new TelemetryEventsSender(this.logger);
@@ -236,6 +243,7 @@ export class Plugin implements ISecuritySolutionPlugin {
       plugins,
       endpointAppContextService: this.endpointAppContextService,
       ruleMonitoringService: this.ruleMonitoringService,
+      siemMigrationsService: this.siemMigrationsService,
       kibanaVersion: pluginContext.env.packageInfo.version,
       kibanaBranch: pluginContext.env.packageInfo.branch,
       buildFlavor: pluginContext.env.packageInfo.buildFlavor,
@@ -427,7 +435,7 @@ export class Plugin implements ISecuritySolutionPlugin {
 
     core
       .getStartServices()
-      .then(async ([_, depsStart]) => {
+      .then(async ([coreStart, depsStart]) => {
         appClientFactory.setup({
           getSpaceId: depsStart.spaces?.spacesService?.getSpaceId,
           config,
@@ -477,6 +485,8 @@ export class Plugin implements ISecuritySolutionPlugin {
          * Register a config for the security guide
          */
         plugins.guidedOnboarding?.registerGuideConfig(siemGuideId, getSiemGuideConfig());
+
+        this.siemMigrationsService.setup({ esClusterClient: coreStart.elasticsearch.client });
       })
       .catch(() => {}); // it shouldn't reject, but just in case
 
@@ -715,6 +725,7 @@ export class Plugin implements ISecuritySolutionPlugin {
     this.endpointAppContextService.stop();
     this.policyWatcher?.stop();
     this.completeExternalResponseActionsTask.stop().catch(() => {});
+    this.siemMigrationsService.stop();
     licenseService.stop();
   }
 }

x-pack/plugins/security_solution/server/request_context_factory.ts

@@ -32,6 +32,7 @@ import { AssetCriticalityDataClient } from './lib/entity_analytics/asset_critica
 import { createDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client';
 import { buildMlAuthz } from './lib/machine_learning/authz';
 import { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client';
+import type { SiemMigrationsService } from './lib/siem_migrations/siem_migrations_service';
 
 export interface IRequestContextFactory {
   create(
@@ -47,6 +48,7 @@ interface ConstructorOptions {
   plugins: SecuritySolutionPluginSetupDependencies;
   endpointAppContextService: EndpointAppContextService;
   ruleMonitoringService: IRuleMonitoringService;
+  siemMigrationsService: SiemMigrationsService;
   kibanaVersion: string;
   kibanaBranch: string;
   buildFlavor: BuildFlavor;
@@ -64,7 +66,14 @@ export class RequestContextFactory implements IRequestContextFactory {
     request: KibanaRequest
   ): Promise<SecuritySolutionApiRequestHandlerContext> {
     const { options, appClientFactory } = this;
-    const { config, core, plugins, endpointAppContextService, ruleMonitoringService } = options;
+    const {
+      config,
+      core,
+      plugins,
+      endpointAppContextService,
+      ruleMonitoringService,
+      siemMigrationsService,
+    } = options;
 
     const { lists, ruleRegistry, security } = plugins;
 
@@ -157,6 +166,10 @@ export class RequestContextFactory implements IRequestContextFactory {
         })
       ),
 
+      getSiemMigrationsClient: memoize(() =>
+        siemMigrationsService.createClient({ request, spaceId: getSpaceId() })
+      ),
+
       getExceptionListClient: () => {
         if (!lists) {
           return null;

x-pack/plugins/security_solution/server/routes/index.ts

@@ -61,6 +61,7 @@ import { suggestUserProfilesRoute } from '../lib/detection_engine/routes/users/s
 import { registerTimelineRoutes } from '../lib/timeline/routes';
 import { getFleetManagedIndexTemplatesRoute } from '../lib/security_integrations/cribl/routes';
 import { registerEntityAnalyticsRoutes } from '../lib/entity_analytics/register_entity_analytics_routes';
+import { registerSiemMigrationsRoutes } from '../lib/siem_migrations/routes';
 
 export const initRoutes = (
   router: SecuritySolutionPluginRouter,
@@ -138,13 +139,17 @@ export const initRoutes = (
   // Dashboards
   registerDashboardsRoutes(router, logger);
   registerTagsRoutes(router, logger);
+
   const { previewTelemetryUrlEnabled } = config.experimentalFeatures;
+
   if (previewTelemetryUrlEnabled) {
     // telemetry preview endpoint for e2e integration tests only at the moment.
     telemetryDetectionRulesPreviewRoute(router, logger, previewTelemetryReceiver, telemetrySender);
   }
 
   registerEntityAnalyticsRoutes({ router, config, getStartServices, logger });
+  registerSiemMigrationsRoutes(router, config, logger);
+
   // Security Integrations
   getFleetManagedIndexTemplatesRoute(router);
 };

x-pack/plugins/security_solution/server/types.ts

@@ -35,6 +35,7 @@ import type { RiskScoreDataClient } from './lib/entity_analytics/risk_score/risk
 import type { AssetCriticalityDataClient } from './lib/entity_analytics/asset_criticality';
 import type { IDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface';
 import type { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client';
+import type { SiemMigrationsClient } from './lib/siem_migrations/types';
 export { AppClient };
 
 export interface SecuritySolutionApiRequestHandlerContext {
@@ -57,6 +58,7 @@ export interface SecuritySolutionApiRequestHandlerContext {
   getRiskScoreDataClient: () => RiskScoreDataClient;
   getAssetCriticalityDataClient: () => AssetCriticalityDataClient;
   getEntityStoreDataClient: () => EntityStoreDataClient;
+  getSiemMigrationsClient: () => SiemMigrationsClient;
 }
 
 export type SecuritySolutionRequestHandlerContext = CustomRequestHandlerContext<{

x-pack/plugins/security_solution/tsconfig.json

@@ -230,5 +230,6 @@
     "@kbn/core-security-server-mocks",
     "@kbn/serverless",
     "@kbn/core-user-profile-browser",
+    "@kbn/data-stream-adapter",
   ]
 }

x-pack/test/api_integration/services/security_solution_api.gen.ts

@@ -32,6 +32,7 @@ import { CopyTimelineRequestBodyInput } from '@kbn/security-solution-plugin/comm
 import { CreateAlertsMigrationRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/signals_migration/create_signals_migration/create_signals_migration.gen';
 import { CreateAssetCriticalityRecordRequestBodyInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/asset_criticality/create_asset_criticality.gen';
 import { CreateRuleRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/crud/create_rule/create_rule_route.gen';
+import { CreateRuleMigrationRequestBodyInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen';
 import { CreateTimelinesRequestBodyInput } from '@kbn/security-solution-plugin/common/api/timeline/create_timelines/create_timelines_route.gen';
 import {
   CreateUpdateProtectionUpdatesNoteRequestParamsInput,
@@ -340,6 +341,17 @@ If a record already exists for the specified entity, that record is overwritten
         .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
         .send(props.body as object);
     },
+    /**
+     * Creates a new SIEM rules migration using the original vendor rules provided
+     */
+    createRuleMigration(props: CreateRuleMigrationProps, kibanaSpace: string = 'default') {
+      return supertest
+        .post(routeWithNamespace('/internal/siem_migrations/rules', kibanaSpace))
+        .set('kbn-xsrf', 'true')
+        .set(ELASTIC_HTTP_VERSION_HEADER, '1')
+        .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
+        .send(props.body as object);
+    },
     /**
      * Create a new Timeline or Timeline template.
      */
@@ -919,6 +931,16 @@ finalize it.
         .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
         .query(props.query);
     },
+    /**
+     * Retrieves the rule migrations stored in the system
+     */
+    getRuleMigration(kibanaSpace: string = 'default') {
+      return supertest
+        .get(routeWithNamespace('/internal/siem_migrations/rules', kibanaSpace))
+        .set('kbn-xsrf', 'true')
+        .set(ELASTIC_HTTP_VERSION_HEADER, '1')
+        .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana');
+    },
     /**
      * Get the details of an existing saved Timeline or Timeline template.
      */
@@ -1382,6 +1404,9 @@ export interface CreateAssetCriticalityRecordProps {
 export interface CreateRuleProps {
   body: CreateRuleRequestBodyInput;
 }
+export interface CreateRuleMigrationProps {
+  body: CreateRuleMigrationRequestBodyInput;
+}
 export interface CreateTimelinesProps {
   body: CreateTimelinesRequestBodyInput;
 }