[8.x] [SecuritySolution][SIEM Migrations] Rule migrations storage (#197032) (#197603)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[SecuritySolution][SIEM Migrations] Rule migrations storage
(#197032)](https://github.com/elastic/kibana/pull/197032)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Sergi
Massaneda","email":"sergi.massaneda@elastic.co"},"sourceCommit":{"committedDate":"2024-10-24T09:56:08Z","message":"[SecuritySolution][SIEM
Migrations] Rule migrations storage (#197032)\n\n##
Summary\r\n\r\nissue:
https://github.com/elastic/security-team/issues/10654?reload=1\r\n\r\nImplements
the persistence layer for the rule migrations from other\r\nvendors, as
part of the SIEM Rule migrations effort.\r\n\r\n### Changes\r\n\r\n-
Schemas created for `SiemRuleMigration` document entity, along
with\r\n`ElasticRule` and `OriginalRule`.\r\n\r\n- New API
`/internal/siem_migrations/rules` was created:\r\n- `POST` -> Receives
an array of (original) rules and stores them with\r\n`status: pending`
to be processed. Responds with the `migration_id` that\r\nwill be used
to start the migration background task (implementation\r\ndetails here:
https://github.com/elastic/security-team/issues/10850).\r\n - `GET` ->
(to be implemented later)\r\n\r\n- New `SiemMigrationsService` added to
the `securitySolution` route\r\ncontext, to encapsulate all operations
related to SIEM migrations (We\r\nstart with _rule_ migrations, but
there are more \"kinds\" of SIEM\r\nmigrations in the pipeline:
_dashboards_, _saved queries_...). It\r\ncontains:\r\n\r\n-
`SiemRuleMigrationsService` to encapsulate all operations related
to\r\nSIEM rule migrations.\r\n- `RuleMigrationsDataStream` class to
manage the\r\n`.kibana.siem-rule-migrations-<spaceId>` data stream
operations using\r\n`DataStreamSpacesAdapter`.\r\n- It exposes a client
with abstracted operations that are exposed to the\r\nAPI routes:\r\n-
`create`: indexes an array of _SiemRuleMigration_ documents to
the\r\ndata stream\r\n- `search`: searches _SiemRuleMigration_ documents
by specific terms.\r\n\r\n> [!NOTE] \r\n> Without
`siemMigrationsEnabled` experimental flag the new API route\r\nwon't be
registered, and the `SiemRuleMigrationsService` _setup_ won't\r\nbe
called, so no index/component template will be installed to
ES.\r\n\r\n### Testing locally\r\n\r\nEnable the
flag\r\n```\r\nxpack.securitySolution.enableExperimental:
['siemMigrationsEnabled']\r\n```\r\n\r\n<details>\r\n <summary>Example
curl request</summary>\r\n\r\n```\r\ncurl --location
'http://elastic:changeme@localhost:5601/internal/siem_migrations/rules'
\\\r\n--header 'kbn-xsrf;' \\\r\n--header 'x-elastic-internal-origin:
security-solution' \\\r\n--header 'elastic-api-version: 1'
\\\r\n--header 'Content-Type: application/json' \\\r\n--data '[\r\n
{\r\n \"id\": \"f8c325ea-506e-4105-8ccf-da1492e90115\",\r\n \"vendor\":
\"splunk\",\r\n \"title\": \"Linux Auditd Add User Account Type\",\r\n
\"description\": \"The following analytic detects the suspicious add
user account type. This behavior is critical for a SOC to monitor
because it may indicate attempts to gain unauthorized access or maintain
control over a system. Such actions could be signs of malicious
activity. If confirmed, this could lead to serious consequences,
including a compromised system, unauthorized access to sensitive data,
or even a wider breach affecting the entire network. Detecting and
responding to these signs early is essential to prevent potential
security incidents.\",\r\n \"query\": \"sourcetype=\\\"linux:audit\\\"
type=ADD_USER \\n| rename hostname as dest \\n| stats count min(_time)
as firstTime max(_time) as lastTime by exe pid dest res UID type \\n|
`security_content_ctime(firstTime)` \\n|
`security_content_ctime(lastTime)`\\n| search *\",\r\n
\"query_language\":\"spl\",\r\n \"mitre_attack_ids\": [\r\n
\"T1136\"\r\n ]\r\n },\r\n {\r\n \"id\":
\"7b87c556-0ca4-47e0-b84c-6cd62a0a3e90\",\r\n \"vendor\":
\"splunk\",\r\n \"title\": \"Linux Auditd Change File Owner To
Root\",\r\n \"description\": \"The following analytic detects the use of
the '\\''chown'\\'' command to change a file owner to '\\''root'\\'' on
a Linux system. It leverages Linux Auditd telemetry, specifically
monitoring command-line executions and process details. This activity is
significant as it may indicate an attempt to escalate privileges by
adversaries, malware, or red teamers. If confirmed malicious, this
action could allow an attacker to gain root-level access, leading to
full control over the compromised host and potential persistence within
the environment.\",\r\n \"query\": \"`linux_auditd`
`linux_auditd_normalized_proctitle_process`\\r\\n| rename host as dest
\\r\\n| where LIKE (process_exec, \\\"%chown %root%\\\") \\r\\n| stats
count min(_time) as firstTime max(_time) as lastTime by process_exec
proctitle normalized_proctitle_delimiter dest \\r\\n|
`security_content_ctime(firstTime)` \\r\\n|
`security_content_ctime(lastTime)`\\r\\n|
`linux_auditd_change_file_owner_to_root_filter`\",\r\n
\"query_language\": \"spl\",\r\n \"mitre_attack_ids\": [\r\n
\"T1222\"\r\n ]\r\n }\r\n]'\r\n```\r\n</details>\r\n\r\nThe newly
created documents can be retrieved using Kibana
DevTools\r\nconsole:\r\n```\r\nGET
.kibana.siem-rule-migrations-default/_search\r\n```\r\n\r\n###
Screenshots\r\n\r\n\r\n![postman_screenshot](https://github.com/user-attachments/assets/9d3852d2-48ef-4955-b621-fdba6b249c65)\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"d7109d67810cedfce1ad2bf9e8fd826b20aee06b","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Threat
Hunting","Team: SecuritySolution","backport:prev-minor","8.18
candidate","v8.17.0"],"number":197032,"url":"https://github.com/elastic/kibana/pull/197032","mergeCommit":{"message":"[SecuritySolution][SIEM
Migrations] Rule migrations storage (#197032)\n\n##
Summary\r\n\r\nissue:
https://github.com/elastic/security-team/issues/10654?reload=1\r\n\r\nImplements
the persistence layer for the rule migrations from other\r\nvendors, as
part of the SIEM Rule migrations effort.\r\n\r\n### Changes\r\n\r\n-
Schemas created for `SiemRuleMigration` document entity, along
with\r\n`ElasticRule` and `OriginalRule`.\r\n\r\n- New API
`/internal/siem_migrations/rules` was created:\r\n- `POST` -> Receives
an array of (original) rules and stores them with\r\n`status: pending`
to be processed. Responds with the `migration_id` that\r\nwill be used
to start the migration background task (implementation\r\ndetails here:
https://github.com/elastic/security-team/issues/10850).\r\n - `GET` ->
(to be implemented later)\r\n\r\n- New `SiemMigrationsService` added to
the `securitySolution` route\r\ncontext, to encapsulate all operations
related to SIEM migrations (We\r\nstart with _rule_ migrations, but
there are more \"kinds\" of SIEM\r\nmigrations in the pipeline:
_dashboards_, _saved queries_...). It\r\ncontains:\r\n\r\n-
`SiemRuleMigrationsService` to encapsulate all operations related
to\r\nSIEM rule migrations.\r\n- `RuleMigrationsDataStream` class to
manage the\r\n`.kibana.siem-rule-migrations-<spaceId>` data stream
operations using\r\n`DataStreamSpacesAdapter`.\r\n- It exposes a client
with abstracted operations that are exposed to the\r\nAPI routes:\r\n-
`create`: indexes an array of _SiemRuleMigration_ documents to
the\r\ndata stream\r\n- `search`: searches _SiemRuleMigration_ documents
by specific terms.\r\n\r\n> [!NOTE] \r\n> Without
`siemMigrationsEnabled` experimental flag the new API route\r\nwon't be
registered, and the `SiemRuleMigrationsService` _setup_ won't\r\nbe
called, so no index/component template will be installed to
ES.\r\n\r\n### Testing locally\r\n\r\nEnable the
flag\r\n```\r\nxpack.securitySolution.enableExperimental:
['siemMigrationsEnabled']\r\n```\r\n\r\n<details>\r\n <summary>Example
curl request</summary>\r\n\r\n```\r\ncurl --location
'http://elastic:changeme@localhost:5601/internal/siem_migrations/rules'
\\\r\n--header 'kbn-xsrf;' \\\r\n--header 'x-elastic-internal-origin:
security-solution' \\\r\n--header 'elastic-api-version: 1'
\\\r\n--header 'Content-Type: application/json' \\\r\n--data '[\r\n
{\r\n \"id\": \"f8c325ea-506e-4105-8ccf-da1492e90115\",\r\n \"vendor\":
\"splunk\",\r\n \"title\": \"Linux Auditd Add User Account Type\",\r\n
\"description\": \"The following analytic detects the suspicious add
user account type. This behavior is critical for a SOC to monitor
because it may indicate attempts to gain unauthorized access or maintain
control over a system. Such actions could be signs of malicious
activity. If confirmed, this could lead to serious consequences,
including a compromised system, unauthorized access to sensitive data,
or even a wider breach affecting the entire network. Detecting and
responding to these signs early is essential to prevent potential
security incidents.\",\r\n \"query\": \"sourcetype=\\\"linux:audit\\\"
type=ADD_USER \\n| rename hostname as dest \\n| stats count min(_time)
as firstTime max(_time) as lastTime by exe pid dest res UID type \\n|
`security_content_ctime(firstTime)` \\n|
`security_content_ctime(lastTime)`\\n| search *\",\r\n
\"query_language\":\"spl\",\r\n \"mitre_attack_ids\": [\r\n
\"T1136\"\r\n ]\r\n },\r\n {\r\n \"id\":
\"7b87c556-0ca4-47e0-b84c-6cd62a0a3e90\",\r\n \"vendor\":
\"splunk\",\r\n \"title\": \"Linux Auditd Change File Owner To
Root\",\r\n \"description\": \"The following analytic detects the use of
the '\\''chown'\\'' command to change a file owner to '\\''root'\\'' on
a Linux system. It leverages Linux Auditd telemetry, specifically
monitoring command-line executions and process details. This activity is
significant as it may indicate an attempt to escalate privileges by
adversaries, malware, or red teamers. If confirmed malicious, this
action could allow an attacker to gain root-level access, leading to
full control over the compromised host and potential persistence within
the environment.\",\r\n \"query\": \"`linux_auditd`
`linux_auditd_normalized_proctitle_process`\\r\\n| rename host as dest
\\r\\n| where LIKE (process_exec, \\\"%chown %root%\\\") \\r\\n| stats
count min(_time) as firstTime max(_time) as lastTime by process_exec
proctitle normalized_proctitle_delimiter dest \\r\\n|
`security_content_ctime(firstTime)` \\r\\n|
`security_content_ctime(lastTime)`\\r\\n|
`linux_auditd_change_file_owner_to_root_filter`\",\r\n
\"query_language\": \"spl\",\r\n \"mitre_attack_ids\": [\r\n
\"T1222\"\r\n ]\r\n }\r\n]'\r\n```\r\n</details>\r\n\r\nThe newly
created documents can be retrieved using Kibana
DevTools\r\nconsole:\r\n```\r\nGET
.kibana.siem-rule-migrations-default/_search\r\n```\r\n\r\n###
Screenshots\r\n\r\n\r\n![postman_screenshot](https://github.com/user-attachments/assets/9d3852d2-48ef-4955-b621-fdba6b249c65)\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"d7109d67810cedfce1ad2bf9e8fd826b20aee06b"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/197032","number":197032,"mergeCommit":{"message":"[SecuritySolution][SIEM
Migrations] Rule migrations storage (#197032)\n\n##
Summary\r\n\r\nissue:
https://github.com/elastic/security-team/issues/10654?reload=1\r\n\r\nImplements
the persistence layer for the rule migrations from other\r\nvendors, as
part of the SIEM Rule migrations effort.\r\n\r\n### Changes\r\n\r\n-
Schemas created for `SiemRuleMigration` document entity, along
with\r\n`ElasticRule` and `OriginalRule`.\r\n\r\n- New API
`/internal/siem_migrations/rules` was created:\r\n- `POST` -> Receives
an array of (original) rules and stores them with\r\n`status: pending`
to be processed. Responds with the `migration_id` that\r\nwill be used
to start the migration background task (implementation\r\ndetails here:
https://github.com/elastic/security-team/issues/10850).\r\n - `GET` ->
(to be implemented later)\r\n\r\n- New `SiemMigrationsService` added to
the `securitySolution` route\r\ncontext, to encapsulate all operations
related to SIEM migrations (We\r\nstart with _rule_ migrations, but
there are more \"kinds\" of SIEM\r\nmigrations in the pipeline:
_dashboards_, _saved queries_...). It\r\ncontains:\r\n\r\n-
`SiemRuleMigrationsService` to encapsulate all operations related
to\r\nSIEM rule migrations.\r\n- `RuleMigrationsDataStream` class to
manage the\r\n`.kibana.siem-rule-migrations-<spaceId>` data stream
operations using\r\n`DataStreamSpacesAdapter`.\r\n- It exposes a client
with abstracted operations that are exposed to the\r\nAPI routes:\r\n-
`create`: indexes an array of _SiemRuleMigration_ documents to
the\r\ndata stream\r\n- `search`: searches _SiemRuleMigration_ documents
by specific terms.\r\n\r\n> [!NOTE] \r\n> Without
`siemMigrationsEnabled` experimental flag the new API route\r\nwon't be
registered, and the `SiemRuleMigrationsService` _setup_ won't\r\nbe
called, so no index/component template will be installed to
ES.\r\n\r\n### Testing locally\r\n\r\nEnable the
flag\r\n```\r\nxpack.securitySolution.enableExperimental:
['siemMigrationsEnabled']\r\n```\r\n\r\n<details>\r\n <summary>Example
curl request</summary>\r\n\r\n```\r\ncurl --location
'http://elastic:changeme@localhost:5601/internal/siem_migrations/rules'
\\\r\n--header 'kbn-xsrf;' \\\r\n--header 'x-elastic-internal-origin:
security-solution' \\\r\n--header 'elastic-api-version: 1'
\\\r\n--header 'Content-Type: application/json' \\\r\n--data '[\r\n
{\r\n \"id\": \"f8c325ea-506e-4105-8ccf-da1492e90115\",\r\n \"vendor\":
\"splunk\",\r\n \"title\": \"Linux Auditd Add User Account Type\",\r\n
\"description\": \"The following analytic detects the suspicious add
user account type. This behavior is critical for a SOC to monitor
because it may indicate attempts to gain unauthorized access or maintain
control over a system. Such actions could be signs of malicious
activity. If confirmed, this could lead to serious consequences,
including a compromised system, unauthorized access to sensitive data,
or even a wider breach affecting the entire network. Detecting and
responding to these signs early is essential to prevent potential
security incidents.\",\r\n \"query\": \"sourcetype=\\\"linux:audit\\\"
type=ADD_USER \\n| rename hostname as dest \\n| stats count min(_time)
as firstTime max(_time) as lastTime by exe pid dest res UID type \\n|
`security_content_ctime(firstTime)` \\n|
`security_content_ctime(lastTime)`\\n| search *\",\r\n
\"query_language\":\"spl\",\r\n \"mitre_attack_ids\": [\r\n
\"T1136\"\r\n ]\r\n },\r\n {\r\n \"id\":
\"7b87c556-0ca4-47e0-b84c-6cd62a0a3e90\",\r\n \"vendor\":
\"splunk\",\r\n \"title\": \"Linux Auditd Change File Owner To
Root\",\r\n \"description\": \"The following analytic detects the use of
the '\\''chown'\\'' command to change a file owner to '\\''root'\\'' on
a Linux system. It leverages Linux Auditd telemetry, specifically
monitoring command-line executions and process details. This activity is
significant as it may indicate an attempt to escalate privileges by
adversaries, malware, or red teamers. If confirmed malicious, this
action could allow an attacker to gain root-level access, leading to
full control over the compromised host and potential persistence within
the environment.\",\r\n \"query\": \"`linux_auditd`
`linux_auditd_normalized_proctitle_process`\\r\\n| rename host as dest
\\r\\n| where LIKE (process_exec, \\\"%chown %root%\\\") \\r\\n| stats
count min(_time) as firstTime max(_time) as lastTime by process_exec
proctitle normalized_proctitle_delimiter dest \\r\\n|
`security_content_ctime(firstTime)` \\r\\n|
`security_content_ctime(lastTime)`\\r\\n|
`linux_auditd_change_file_owner_to_root_filter`\",\r\n
\"query_language\": \"spl\",\r\n \"mitre_attack_ids\": [\r\n
\"T1222\"\r\n ]\r\n }\r\n]'\r\n```\r\n</details>\r\n\r\nThe newly
created documents can be retrieved using Kibana
DevTools\r\nconsole:\r\n```\r\nGET
.kibana.siem-rule-migrations-default/_search\r\n```\r\n\r\n###
Screenshots\r\n\r\n\r\n![postman_screenshot](https://github.com/user-attachments/assets/9d3852d2-48ef-4955-b621-fdba6b249c65)\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"d7109d67810cedfce1ad2bf9e8fd826b20aee06b"}},{"branch":"8.x","label":"v8.17.0","labelRegex":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

.github/CODEOWNERS

@@ -1594,7 +1594,12 @@ x-pack/test/security_solution_api_integration/test_suites/sources @elastic/secur
 /x-pack/test/security_solution_playwright @elastic/security-engineering-productivity
 /x-pack/plugins/security_solution/scripts/run_cypress @MadameSheema @patrykkopycinski @maximpn @banderror
 
-## Security Solution sub teams - Threat Hunting Investigations
+## Security Solution sub teams - Threat Hunting
+
+/x-pack/plugins/security_solution/server/lib/siem_migrations @elastic/security-threat-hunting
+/x-pack/plugins/security_solution/common/siem_migrations @elastic/security-threat-hunting
+
+## Security Solution Threat Hunting areas - Threat Hunting Investigations
 
 /x-pack/plugins/security_solution/common/api/timeline @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/common/search_strategy/timeline @elastic/security-threat-hunting-investigations
@@ -1624,7 +1629,7 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 
 /x-pack/plugins/security_solution/server/lib/timeline @elastic/security-threat-hunting-investigations
 
-## Security Solution sub teams - Threat Hunting Explore
+## Security Solution Threat Hunting areas - Threat Hunting Explore
 /x-pack/plugins/security_solution/common/api/tags @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram @elastic/security-threat-hunting-explore

packages/kbn-data-stream-adapter/src/field_maps/types.ts

@@ -38,8 +38,9 @@ export interface EcsMetadata {
   properties?: Record<string, { type: string }>;
 }
 
-export interface FieldMap {
-  [key: string]: {
+export type FieldMap<T extends string = string> = Record<
+  T,
+  {
     type: string;
     required: boolean;
     array?: boolean;
@@ -53,5 +54,17 @@ export interface FieldMap {
     scaling_factor?: number;
     dynamic?: boolean | 'strict';
     properties?: Record<string, { type: string }>;
-  };
-}
+  }
+>;
+
+// This utility type flattens all the keys of a schema object and its nested objects as a union type.
+// Its purpose is to ensure that the FieldMap keys are always in sync with the schema object.
+// It assumes all optional fields of the schema are required in the field map, they can always be omitted from the resulting type.
+export type SchemaFieldMapKeys<
+  T extends Record<string, unknown>,
+  Key = keyof T
+> = Key extends string
+  ? NonNullable<T[Key]> extends Record<string, unknown>
+    ? `${Key}` | `${Key}.${SchemaFieldMapKeys<NonNullable<T[Key]>>}`
+    : `${Key}`
+  : never;

x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts

@@ -361,6 +361,11 @@ import type {
   ResolveTimelineRequestQueryInput,
   ResolveTimelineResponse,
 } from './timeline/resolve_timeline/resolve_timeline_route.gen';
+import type {
+  CreateRuleMigrationRequestBodyInput,
+  CreateRuleMigrationResponse,
+  GetRuleMigrationResponse,
+} from '../siem_migrations/model/api/rules/rules_migration.gen';
 
 export interface ClientOptions {
   kbnClient: KbnClient;
@@ -647,6 +652,22 @@ Migrations are initiated per index. While the process is neither destructive nor
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  /**
+   * Creates a new SIEM rules migration using the original vendor rules provided
+   */
+  async createRuleMigration(props: CreateRuleMigrationProps) {
+    this.log.info(`${new Date().toISOString()} Calling API CreateRuleMigration`);
+    return this.kbnClient
+      .request<CreateRuleMigrationResponse>({
+        path: '/internal/siem_migrations/rules',
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '1',
+        },
+        method: 'POST',
+        body: props.body,
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   async createTimelines(props: CreateTimelinesProps) {
     this.log.info(`${new Date().toISOString()} Calling API CreateTimelines`);
     return this.kbnClient
@@ -1373,6 +1394,21 @@ finalize it.
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  /**
+   * Retrieves the rule migrations stored in the system
+   */
+  async getRuleMigration() {
+    this.log.info(`${new Date().toISOString()} Calling API GetRuleMigration`);
+    return this.kbnClient
+      .request<GetRuleMigrationResponse>({
+        path: '/internal/siem_migrations/rules',
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '1',
+        },
+        method: 'GET',
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   async getTimeline(props: GetTimelineProps) {
     this.log.info(`${new Date().toISOString()} Calling API GetTimeline`);
     return this.kbnClient
@@ -1992,6 +2028,9 @@ export interface CreateAssetCriticalityRecordProps {
 export interface CreateRuleProps {
   body: CreateRuleRequestBodyInput;
 }
+export interface CreateRuleMigrationProps {
+  body: CreateRuleMigrationRequestBodyInput;
+}
 export interface CreateTimelinesProps {
   body: CreateTimelinesRequestBodyInput;
 }

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -235,6 +235,11 @@ export const allowedExperimentalValues = Object.freeze({
    * can be disabled if necessary in a given environment.
    */
   entityStoreDisabled: false,
+
+  /**
+   * Enables the siem migrations feature
+   */
+  siemMigrationsEnabled: false,
 });
 
 type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;

x-pack/plugins/security_solution/common/siem_migrations/constants.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const SIEM_MIGRATIONS_PATH = '/internal/siem_migrations' as const;
+export const SIEM_RULE_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/rules` as const;
+
+export enum SiemMigrationsStatus {
+  PENDING = 'pending',
+  PROCESSING = 'processing',
+  FINISHED = 'finished',
+  ERROR = 'error',
+}

x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Common SIEM Migrations Attributes
+ *   version: not applicable
+ */
+
+import { z } from '@kbn/zod';
+
+/**
+ * The GenAI connector id to use.
+ */
+export type ConnectorId = z.infer<typeof ConnectorId>;
+export const ConnectorId = z.string();
+
+/**
+ * The LangSmith options object.
+ */
+export type LangSmithOptions = z.infer<typeof LangSmithOptions>;
+export const LangSmithOptions = z.object({
+  /**
+   * The project name.
+   */
+  project_name: z.string(),
+  /**
+   * The API key to use for tracing.
+   */
+  api_key: z.string(),
+});

x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml

@@ -0,0 +1,24 @@
+openapi: 3.0.3
+info:
+  title: Common SIEM Migrations Attributes
+  version: 'not applicable'
+paths: {}
+components:
+  x-codegen-enabled: true
+  schemas:
+    ConnectorId:
+      type: string
+      description: The GenAI connector id to use.
+    LangSmithOptions:
+      type: object
+      description: The LangSmith options object.
+      required:
+        - project_name
+        - api_key
+      properties:
+        project_name:
+          type: string
+          description: The project name.
+        api_key:
+          type: string
+          description: The API key to use for tracing.

x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: SIEM Rules Migration API endpoint
+ *   version: 1
+ */
+
+import { z } from '@kbn/zod';
+
+import { OriginalRule, RuleMigration } from '../../rule_migration.gen';
+
+export type CreateRuleMigrationRequestBody = z.infer<typeof CreateRuleMigrationRequestBody>;
+export const CreateRuleMigrationRequestBody = z.array(OriginalRule);
+export type CreateRuleMigrationRequestBodyInput = z.input<typeof CreateRuleMigrationRequestBody>;
+
+export type CreateRuleMigrationResponse = z.infer<typeof CreateRuleMigrationResponse>;
+export const CreateRuleMigrationResponse = z.object({
+  /**
+   * The migration id created.
+   */
+  migration_id: z.string(),
+});
+
+export type GetRuleMigrationResponse = z.infer<typeof GetRuleMigrationResponse>;
+export const GetRuleMigrationResponse = z.array(RuleMigration);

x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml

@@ -0,0 +1,52 @@
+openapi: 3.0.3
+info:
+  title: SIEM Rules Migration API endpoint
+  version: '1'
+paths:
+  /internal/siem_migrations/rules:
+    post:
+      summary: Creates a new rule migration
+      operationId: CreateRuleMigration
+      x-codegen-enabled: true
+      description: Creates a new SIEM rules migration using the original vendor rules provided
+      tags:
+        - SIEM Migrations
+        - Rule Migrations
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                $ref: '../../rule_migration.schema.yaml#/components/schemas/OriginalRule'
+      responses:
+        200:
+          description: Indicates migration have been created correctly.
+          content:
+            application/json:
+              schema:
+                type: object
+                required:
+                  - migration_id
+                properties:
+                  migration_id:
+                    type: string
+                    description: The migration id created.
+    get:
+      summary: Retrieves rule migrations
+      operationId: GetRuleMigration
+      x-codegen-enabled: true
+      description: Retrieves the rule migrations stored in the system
+      tags:
+        - SIEM Migrations
+        - Rule Migrations
+      responses:
+        200:
+          description: Indicates rule migrations have been retrieved correctly.
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigration'

x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Common Splunk Rules Attributes
+ *   version: not applicable
+ */
+
+import { z } from '@kbn/zod';
+
+/**
+ * The original rule to migrate.
+ */
+export type OriginalRule = z.infer<typeof OriginalRule>;
+export const OriginalRule = z.object({
+  /**
+   * The original rule id.
+   */
+  id: z.string(),
+  /**
+   * The original rule vendor identifier.
+   */
+  vendor: z.literal('splunk'),
+  /**
+   * The original rule name.
+   */
+  title: z.string(),
+  /**
+   * The original rule description.
+   */
+  description: z.string(),
+  /**
+   * The original rule query.
+   */
+  query: z.string(),
+  /**
+   * The original rule query language.
+   */
+  query_language: z.string(),
+  /**
+   * The original rule Mitre Attack technique IDs.
+   */
+  mitre_attack_ids: z.array(z.string()).optional(),
+});
+
+/**
+ * The migrated elastic rule.
+ */
+export type ElasticRule = z.infer<typeof ElasticRule>;
+export const ElasticRule = z.object({
+  /**
+   * The migrated rule title.
+   */
+  title: z.string(),
+  /**
+   * The migrated rule description.
+   */
+  description: z.string().optional(),
+  /**
+   * The migrated rule severity.
+   */
+  severity: z.string().optional(),
+  /**
+   * The translated elastic query.
+   */
+  query: z.string(),
+  /**
+   * The translated elastic query language.
+   */
+  query_language: z.literal('esql').default('esql'),
+  /**
+   * The Elastic prebuilt rule id matched.
+   */
+  prebuilt_rule_id: z.string().optional(),
+  /**
+   * The Elastic rule id installed as a result.
+   */
+  id: z.string().optional(),
+});
+
+/**
+ * The rule migration document object.
+ */
+export type RuleMigration = z.infer<typeof RuleMigration>;
+export const RuleMigration = z.object({
+  /**
+   * The moment of creation
+   */
+  '@timestamp': z.string(),
+  /**
+   * The migration id.
+   */
+  migration_id: z.string(),
+  original_rule: OriginalRule,
+  elastic_rule: ElasticRule.optional(),
+  /**
+   * The translation state.
+   */
+  translation_state: z.enum(['complete', 'partial', 'untranslatable']).optional(),
+  /**
+   * The status of the rule migration.
+   */
+  status: z.enum(['pending', 'processing', 'finished', 'error']).default('pending'),
+  /**
+   * The comments for the migration including a summary from the LLM in markdown.
+   */
+  comments: z.array(z.string()).optional(),
+  /**
+   * The moment of the last update
+   */
+  updated_at: z.string().optional(),
+  /**
+   * The user who last updated the migration
+   */
+  updated_by: z.string().optional(),
+});

x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml

@@ -0,0 +1,124 @@
+openapi: 3.0.3
+info:
+  title: Common Splunk Rules Attributes
+  version: 'not applicable'
+paths: {}
+components:
+  x-codegen-enabled: true
+  schemas:
+    OriginalRule:
+      type: object
+      description: The original rule to migrate.
+      required:
+        - id
+        - vendor
+        - title
+        - description
+        - query
+        - query_language
+      properties:
+        id:
+          type: string
+          description: The original rule id.
+        vendor:
+          type: string
+          description: The original rule vendor identifier.
+          enum:
+            - splunk
+        title:
+          type: string
+          description: The original rule name.
+        description:
+          type: string
+          description: The original rule description.
+        query:
+          type: string
+          description: The original rule query.
+        query_language:
+          type: string
+          description: The original rule query language.
+        mitre_attack_ids:
+          type: array
+          items:
+            type: string
+          description: The original rule Mitre Attack technique IDs.
+
+    ElasticRule:
+      type: object
+      description: The migrated elastic rule.
+      required:
+        - title
+        - query
+        - query_language
+      properties:
+        title:
+          type: string
+          description: The migrated rule title.
+        description:
+          type: string
+          description: The migrated rule description.
+        severity:
+          type: string
+          description: The migrated rule severity.
+        query:
+          type: string
+          description: The translated elastic query.
+        query_language:
+          type: string
+          description: The translated elastic query language.
+          enum:
+            - esql
+          default: esql
+        prebuilt_rule_id:
+          type: string
+          description: The Elastic prebuilt rule id matched.
+        id:
+          type: string
+          description: The Elastic rule id installed as a result.
+
+    RuleMigration:
+      type: object
+      description: The rule migration document object.
+      required:
+        - '@timestamp'
+        - migration_id
+        - original_rule
+        - status
+      properties:
+        "@timestamp":
+          type: string
+          description: The moment of creation
+        migration_id:
+          type: string
+          description: The migration id.
+        original_rule:
+          $ref: '#/components/schemas/OriginalRule'
+        elastic_rule:
+          $ref: '#/components/schemas/ElasticRule'
+        translation_state:
+          type: string
+          description: The translation state.
+          enum:
+            - complete
+            - partial
+            - untranslatable
+        status:
+          type: string
+          description: The status of the rule migration.
+          enum: # should match SiemMigrationsStatus enum at ../constants.ts
+            - pending
+            - processing
+            - finished
+            - error
+          default: pending
+        comments:
+          type: array
+          description: The comments for the migration including a summary from the LLM in markdown.
+          items:
+            type: string
+        updated_at:
+          type: string
+          description: The moment of the last update
+        updated_by:
+          type: string
+          description: The user who last updated the migration

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts

@@ -42,6 +42,7 @@ import { auditLoggerMock } from '@kbn/security-plugin/server/audit/mocks';
 import { detectionRulesClientMock } from '../../rule_management/logic/detection_rules_client/__mocks__/detection_rules_client';
 import { packageServiceMock } from '@kbn/fleet-plugin/server/services/epm/package_service.mock';
 import type { EndpointInternalFleetServicesInterface } from '../../../../endpoint/services/fleet';
+import { siemMigrationsServiceMock } from '../../../siem_migrations/__mocks__/mocks';
 
 export const createMockClients = () => {
   const core = coreMock.createRequestHandlerContext();
@@ -78,6 +79,7 @@ export const createMockClients = () => {
     internalFleetServices: {
       packages: packageServiceMock.createClient(),
     },
+    siemMigrationsClient: siemMigrationsServiceMock.createClient(),
   };
 };
 
@@ -163,6 +165,7 @@ const createSecuritySolutionRequestContextMock = (
     getAssetCriticalityDataClient: jest.fn(() => clients.assetCriticalityDataClient),
     getAuditLogger: jest.fn(() => mockAuditLogger),
     getEntityStoreDataClient: jest.fn(() => clients.entityStoreDataClient),
+    getSiemMigrationsClient: jest.fn(() => clients.siemMigrationsClient),
   };
 };
 

x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createRuleMigrationClient } from '../rules/__mocks__/mocks';
+
+const createClient = () => ({ rules: createRuleMigrationClient() });
+
+export const mockSetup = jest.fn().mockResolvedValue(undefined);
+export const mockCreateClient = jest.fn().mockReturnValue(createClient());
+export const mockStop = jest.fn();
+
+export const siemMigrationsServiceMock = {
+  create: () =>
+    jest.fn().mockImplementation(() => ({
+      setup: mockSetup,
+      createClient: mockCreateClient,
+      stop: mockStop,
+    })),
+  createClient: () => createClient(),
+};

x-pack/plugins/security_solution/server/lib/siem_migrations/routes.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import { registerSiemRuleMigrationsRoutes } from './rules/api';
+import type { SecuritySolutionPluginRouter } from '../../types';
+import type { ConfigType } from '../../config';
+
+export const registerSiemMigrationsRoutes = (
+  router: SecuritySolutionPluginRouter,
+  config: ConfigType,
+  logger: Logger
+) => {
+  if (config.experimentalFeatures.siemMigrationsEnabled) {
+    registerSiemRuleMigrationsRoutes(router, logger);
+  }
+};

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SiemRuleMigrationsClient } from '../types';
+
+export const createRuleMigrationClient = (): SiemRuleMigrationsClient => ({
+  create: jest.fn().mockResolvedValue({ success: true }),
+  search: jest.fn().mockResolvedValue([]),
+});
+
+export const mockSetup = jest.fn();
+export const mockGetClient = jest.fn().mockReturnValue(createRuleMigrationClient());
+
+export const MockSiemRuleMigrationsService = jest.fn().mockImplementation(() => ({
+  setup: mockSetup,
+  getClient: mockGetClient,
+}));

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_service.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MockSiemRuleMigrationsService } from './mocks';
+export const SiemRuleMigrationsService = MockSiemRuleMigrationsService;

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IKibanaResponse, Logger } from '@kbn/core/server';
+import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
+import { v4 as uuidV4 } from 'uuid';
+import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen';
+import type { CreateRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen';
+import { CreateRuleMigrationRequestBody } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen';
+import {
+  SIEM_RULE_MIGRATIONS_PATH,
+  SiemMigrationsStatus,
+} from '../../../../../common/siem_migrations/constants';
+import type { SecuritySolutionPluginRouter } from '../../../../types';
+
+export const registerSiemRuleMigrationsCreateRoute = (
+  router: SecuritySolutionPluginRouter,
+  logger: Logger
+) => {
+  router.versioned
+    .post({
+      path: SIEM_RULE_MIGRATIONS_PATH,
+      access: 'internal',
+      options: { tags: ['access:securitySolution'] },
+    })
+    .addVersion(
+      {
+        version: '1',
+        validate: {
+          request: { body: buildRouteValidationWithZod(CreateRuleMigrationRequestBody) },
+        },
+      },
+      async (context, req, res): Promise<IKibanaResponse<CreateRuleMigrationResponse>> => {
+        const originalRules = req.body;
+        try {
+          const ctx = await context.resolve(['core', 'actions', 'securitySolution']);
+
+          const siemMigrationClient = ctx.securitySolution.getSiemMigrationsClient();
+
+          const migrationId = uuidV4();
+          const timestamp = new Date().toISOString();
+
+          const ruleMigrations = originalRules.map<RuleMigration>((originalRule) => ({
+            '@timestamp': timestamp,
+            migration_id: migrationId,
+            original_rule: originalRule,
+            status: SiemMigrationsStatus.PENDING,
+          }));
+          await siemMigrationClient.rules.create(ruleMigrations);
+
+          return res.ok({ body: { migration_id: migrationId } });
+        } catch (err) {
+          logger.error(err);
+          return res.badRequest({
+            body: err.message,
+          });
+        }
+      }
+    );
+};

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import type { SecuritySolutionPluginRouter } from '../../../../types';
+import { registerSiemRuleMigrationsCreateRoute } from './create';
+
+export const registerSiemRuleMigrationsRoutes = (
+  router: SecuritySolutionPluginRouter,
+  logger: Logger
+) => {
+  registerSiemRuleMigrationsCreateRoute(router, logger);
+};

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const mockIndexName = 'mocked_data_stream_name';
+export const mockInstall = jest.fn().mockResolvedValue(undefined);
+export const mockInstallSpace = jest.fn().mockResolvedValue(mockIndexName);
+
+export const MockRuleMigrationsDataStream = jest.fn().mockImplementation(() => ({
+  install: mockInstall,
+  installSpace: mockInstallSpace,
+}));

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/rule_migrations_data_stream.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MockRuleMigrationsDataStream } from './mocks';
+export const RuleMigrationsDataStream = MockRuleMigrationsDataStream;

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { RuleMigrationsDataStream } from './rule_migrations_data_stream';
+import { Subject } from 'rxjs';
+import type { InstallParams } from '@kbn/data-stream-adapter';
+import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter';
+import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
+import { loggerMock } from '@kbn/logging-mocks';
+
+jest.mock('@kbn/data-stream-adapter');
+
+const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest.MockedClass<
+  typeof DataStreamSpacesAdapter
+>;
+
+const esClient = elasticsearchServiceMock.createStart().client.asInternalUser;
+
+describe('SiemRuleMigrationsDataStream', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('constructor', () => {
+    it('should create DataStreamSpacesAdapter', () => {
+      new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      expect(MockedDataStreamSpacesAdapter).toHaveBeenCalledTimes(1);
+    });
+
+    it('should create component templates', () => {
+      new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      expect(dataStreamSpacesAdapter.setComponentTemplate).toHaveBeenCalledWith(
+        expect.objectContaining({ name: '.kibana.siem-rule-migrations' })
+      );
+    });
+
+    it('should create index templates', () => {
+      new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      expect(dataStreamSpacesAdapter.setIndexTemplate).toHaveBeenCalledWith(
+        expect.objectContaining({ name: '.kibana.siem-rule-migrations' })
+      );
+    });
+  });
+
+  describe('install', () => {
+    it('should install data stream', async () => {
+      const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const params: InstallParams = {
+        esClient,
+        logger: loggerMock.create(),
+        pluginStop$: new Subject(),
+      };
+      await dataStream.install(params);
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(params);
+    });
+
+    it('should log error', async () => {
+      const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const params: InstallParams = {
+        esClient,
+        logger: loggerMock.create(),
+        pluginStop$: new Subject(),
+      };
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      const error = new Error('test-error');
+      (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error);
+
+      await dataStream.install(params);
+      expect(params.logger.error).toHaveBeenCalledWith(expect.any(String), error);
+    });
+  });
+
+  describe('installSpace', () => {
+    it('should install space data stream', async () => {
+      const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const params: InstallParams = {
+        esClient,
+        logger: loggerMock.create(),
+        pluginStop$: new Subject(),
+      };
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      (dataStreamSpacesAdapter.install as jest.Mock).mockResolvedValueOnce(undefined);
+
+      await dataStream.install(params);
+      await dataStream.installSpace('space1');
+
+      expect(dataStreamSpacesAdapter.getInstalledSpaceName).toHaveBeenCalledWith('space1');
+      expect(dataStreamSpacesAdapter.installSpace).toHaveBeenCalledWith('space1');
+    });
+
+    it('should not install space data stream if install not executed', async () => {
+      const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      await expect(dataStream.installSpace('space1')).rejects.toThrowError();
+    });
+
+    it('should throw error if main install had error', async () => {
+      const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' });
+      const params: InstallParams = {
+        esClient,
+        logger: loggerMock.create(),
+        pluginStop$: new Subject(),
+      };
+      const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
+      const error = new Error('test-error');
+      (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error);
+      await dataStream.install(params);
+
+      await expect(dataStream.installSpace('space1')).rejects.toThrowError(error);
+    });
+  });
+});

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DataStreamSpacesAdapter, type InstallParams } from '@kbn/data-stream-adapter';
+import { ruleMigrationsFieldMap } from './rule_migrations_field_map';
+
+const TOTAL_FIELDS_LIMIT = 2500;
+
+const DATA_STREAM_NAME = '.kibana.siem-rule-migrations';
+const ECS_COMPONENT_TEMPLATE_NAME = 'ecs';
+
+export class RuleMigrationsDataStream {
+  private readonly dataStream: DataStreamSpacesAdapter;
+  private installPromise?: Promise<void>;
+
+  constructor({ kibanaVersion }: { kibanaVersion: string }) {
+    this.dataStream = new DataStreamSpacesAdapter(DATA_STREAM_NAME, {
+      kibanaVersion,
+      totalFieldsLimit: TOTAL_FIELDS_LIMIT,
+    });
+    this.dataStream.setComponentTemplate({
+      name: DATA_STREAM_NAME,
+      fieldMap: ruleMigrationsFieldMap,
+    });
+
+    this.dataStream.setIndexTemplate({
+      name: DATA_STREAM_NAME,
+      componentTemplateRefs: [DATA_STREAM_NAME, ECS_COMPONENT_TEMPLATE_NAME],
+    });
+  }
+
+  async install(params: InstallParams) {
+    try {
+      this.installPromise = this.dataStream.install(params);
+      await this.installPromise;
+    } catch (err) {
+      params.logger.error(`Error installing siem rule migrations data stream. ${err.message}`, err);
+    }
+  }
+
+  async installSpace(spaceId: string): Promise<string> {
+    if (!this.installPromise) {
+      throw new Error('Siem rule migrations data stream not installed');
+    }
+    // wait for install to complete, may reject if install failed, routes should handle this
+    await this.installPromise;
+    let dataStreamName = await this.dataStream.getInstalledSpaceName(spaceId);
+    if (!dataStreamName) {
+      dataStreamName = await this.dataStream.installSpace(spaceId);
+    }
+    return dataStreamName;
+  }
+}

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { FieldMap, SchemaFieldMapKeys } from '@kbn/data-stream-adapter';
+import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen';
+
+export const ruleMigrationsFieldMap: FieldMap<SchemaFieldMapKeys<RuleMigration>> = {
+  '@timestamp': { type: 'date', required: false },
+  migration_id: { type: 'keyword', required: true },
+  status: { type: 'keyword', required: true },
+  original_rule: { type: 'nested', required: true },
+  'original_rule.vendor': { type: 'keyword', required: true },
+  'original_rule.id': { type: 'keyword', required: true },
+  'original_rule.title': { type: 'keyword', required: true },
+  'original_rule.description': { type: 'keyword', required: false },
+  'original_rule.query': { type: 'text', required: true },
+  'original_rule.query_language': { type: 'keyword', required: true },
+  'original_rule.mitre_attack_ids': { type: 'keyword', array: true, required: false },
+  elastic_rule: { type: 'nested', required: false },
+  'elastic_rule.title': { type: 'keyword', required: true },
+  'elastic_rule.query': { type: 'text', required: true },
+  'elastic_rule.query_language': { type: 'keyword', required: true },
+  'elastic_rule.description': { type: 'keyword', required: false },
+  'elastic_rule.severity': { type: 'keyword', required: false },
+  'elastic_rule.prebuilt_rule_id': { type: 'keyword', required: false },
+  'elastic_rule.id': { type: 'keyword', required: false },
+  translation_state: { type: 'keyword', required: false },
+  comments: { type: 'text', array: true, required: false },
+  updated_at: { type: 'date', required: false },
+  updated_by: { type: 'keyword', required: false },
+};

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  loggingSystemMock,
+  elasticsearchServiceMock,
+  httpServerMock,
+} from '@kbn/core/server/mocks';
+import { SiemRuleMigrationsService } from './siem_rule_migrations_service';
+import { Subject } from 'rxjs';
+import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen';
+import {
+  MockRuleMigrationsDataStream,
+  mockInstall,
+  mockInstallSpace,
+  mockIndexName,
+} from './data_stream/__mocks__/mocks';
+import type { KibanaRequest } from '@kbn/core/server';
+
+jest.mock('./data_stream/rule_migrations_data_stream');
+
+describe('SiemRuleMigrationsService', () => {
+  let ruleMigrationsService: SiemRuleMigrationsService;
+  const kibanaVersion = '8.16.0';
+
+  const esClusterClient = elasticsearchServiceMock.createClusterClient();
+  const logger = loggingSystemMock.createLogger();
+  const pluginStop$ = new Subject<void>();
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    ruleMigrationsService = new SiemRuleMigrationsService(logger, kibanaVersion);
+  });
+
+  it('should instantiate the rule migrations data stream adapter', () => {
+    expect(MockRuleMigrationsDataStream).toHaveBeenCalledWith({ kibanaVersion });
+  });
+
+  describe('when setup is called', () => {
+    it('should set esClusterClient and call dataStreamAdapter.install', () => {
+      ruleMigrationsService.setup({ esClusterClient, pluginStop$ });
+
+      expect(mockInstall).toHaveBeenCalledWith({
+        esClient: esClusterClient.asInternalUser,
+        logger,
+        pluginStop$,
+      });
+    });
+  });
+
+  describe('when getClient is called', () => {
+    let request: KibanaRequest;
+    beforeEach(() => {
+      request = httpServerMock.createKibanaRequest();
+    });
+
+    describe('without setup', () => {
+      it('should throw an error', () => {
+        expect(() => {
+          ruleMigrationsService.getClient({ spaceId: 'default', request });
+        }).toThrowError('ES client not available, please call setup first');
+      });
+    });
+
+    describe('with setup', () => {
+      beforeEach(() => {
+        ruleMigrationsService.setup({ esClusterClient, pluginStop$ });
+      });
+
+      it('should call installSpace', () => {
+        ruleMigrationsService.getClient({ spaceId: 'default', request });
+
+        expect(mockInstallSpace).toHaveBeenCalledWith('default');
+      });
+
+      it('should return a client with create and search methods after setup', () => {
+        const client = ruleMigrationsService.getClient({ spaceId: 'default', request });
+
+        expect(client).toHaveProperty('create');
+        expect(client).toHaveProperty('search');
+      });
+
+      it('should call ES bulk create API with the correct parameters with create is called', async () => {
+        const client = ruleMigrationsService.getClient({ spaceId: 'default', request });
+
+        const ruleMigrations = [{ migration_id: 'dummy_migration_id' } as RuleMigration];
+        await client.create(ruleMigrations);
+
+        expect(esClusterClient.asScoped().asCurrentUser.bulk).toHaveBeenCalledWith(
+          expect.objectContaining({
+            body: [{ create: { _index: mockIndexName } }, { migration_id: 'dummy_migration_id' }],
+            refresh: 'wait_for',
+          })
+        );
+      });
+
+      it('should call ES search API with the correct parameters with search is called', async () => {
+        const client = ruleMigrationsService.getClient({ spaceId: 'default', request });
+
+        const term = { migration_id: 'dummy_migration_id' };
+        await client.search(term);
+
+        expect(esClusterClient.asScoped().asCurrentUser.search).toHaveBeenCalledWith(
+          expect.objectContaining({
+            index: mockIndexName,
+            body: { query: { term } },
+          })
+        );
+      });
+    });
+  });
+});

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IClusterClient, Logger } from '@kbn/core/server';
+import { RuleMigrationsDataStream } from './data_stream/rule_migrations_data_stream';
+import type {
+  SiemRuleMigrationsClient,
+  SiemRulesMigrationsSetupParams,
+  SiemRuleMigrationsGetClientParams,
+} from './types';
+
+export class SiemRuleMigrationsService {
+  private dataStreamAdapter: RuleMigrationsDataStream;
+  private esClusterClient?: IClusterClient;
+
+  constructor(private logger: Logger, kibanaVersion: string) {
+    this.dataStreamAdapter = new RuleMigrationsDataStream({ kibanaVersion });
+  }
+
+  setup({ esClusterClient, ...params }: SiemRulesMigrationsSetupParams) {
+    this.esClusterClient = esClusterClient;
+    const esClient = esClusterClient.asInternalUser;
+    this.dataStreamAdapter.install({ ...params, esClient, logger: this.logger }).catch((err) => {
+      this.logger.error(`Error installing data stream for rule migrations: ${err.message}`);
+      throw err;
+    });
+  }
+
+  getClient({ spaceId, request }: SiemRuleMigrationsGetClientParams): SiemRuleMigrationsClient {
+    if (!this.esClusterClient) {
+      throw new Error('ES client not available, please call setup first');
+    }
+    // Installs the data stream for the specific space. it will only install if it hasn't been installed yet.
+    // The adapter stores the data stream name promise, it will return it directly when the data stream is known to be installed.
+    const dataStreamNamePromise = this.dataStreamAdapter.installSpace(spaceId);
+
+    const esClient = this.esClusterClient.asScoped(request).asCurrentUser;
+    return {
+      create: async (ruleMigrations) => {
+        const _index = await dataStreamNamePromise;
+        return esClient.bulk({
+          refresh: 'wait_for',
+          body: ruleMigrations.flatMap((ruleMigration) => [{ create: { _index } }, ruleMigration]),
+        });
+      },
+      search: async (term) => {
+        const index = await dataStreamNamePromise;
+        return esClient.search({ index, body: { query: { term } } });
+      },
+    };
+  }
+}

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { BulkResponse, SearchResponse } from '@elastic/elasticsearch/lib/api/types';
+import type { IClusterClient, KibanaRequest } from '@kbn/core/server';
+import type { Subject } from 'rxjs';
+import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen';
+
+export interface SiemRulesMigrationsSetupParams {
+  esClusterClient: IClusterClient;
+  pluginStop$: Subject<void>;
+  tasksTimeoutMs?: number;
+}
+
+export interface SiemRuleMigrationsGetClientParams {
+  request: KibanaRequest;
+  spaceId: string;
+}
+
+export interface RuleMigrationSearchParams {
+  migration_id?: string;
+}
+export interface SiemRuleMigrationsClient {
+  create: (body: RuleMigration[]) => Promise<BulkResponse>;
+  search: (params: RuleMigrationSearchParams) => Promise<SearchResponse>;
+}

x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  loggingSystemMock,
+  elasticsearchServiceMock,
+  httpServerMock,
+} from '@kbn/core/server/mocks';
+import { SiemMigrationsService } from './siem_migrations_service';
+import { MockSiemRuleMigrationsService, mockSetup, mockGetClient } from './rules/__mocks__/mocks';
+import type { ConfigType } from '../../config';
+
+jest.mock('./rules/siem_rule_migrations_service');
+
+const mockReplaySubject$ = { next: jest.fn(), complete: jest.fn() };
+jest.mock('rxjs', () => ({
+  ...jest.requireActual('rxjs'),
+  ReplaySubject: jest.fn().mockImplementation(() => mockReplaySubject$),
+}));
+
+describe('SiemMigrationsService', () => {
+  let siemMigrationsService: SiemMigrationsService;
+  const kibanaVersion = '8.16.0';
+
+  const esClusterClient = elasticsearchServiceMock.createClusterClient();
+  const logger = loggingSystemMock.createLogger();
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('with siemMigrationsEnabled flag', () => {
+    beforeEach(() => {
+      siemMigrationsService = new SiemMigrationsService(
+        { experimentalFeatures: { siemMigrationsEnabled: true } } as ConfigType,
+        logger,
+        kibanaVersion
+      );
+    });
+
+    it('should instantiate the rule migrations service', async () => {
+      expect(MockSiemRuleMigrationsService).toHaveBeenCalledWith(logger, kibanaVersion);
+    });
+
+    describe('when setup is called', () => {
+      it('should call siemRuleMigrationsService setup', async () => {
+        siemMigrationsService.setup({ esClusterClient, tasksTimeoutMs: 100 });
+
+        expect(mockSetup).toHaveBeenCalledWith({
+          esClusterClient,
+          tasksTimeoutMs: 100,
+          pluginStop$: mockReplaySubject$,
+        });
+      });
+    });
+
+    describe('when createClient is called', () => {
+      it('should create rules client', async () => {
+        const request = httpServerMock.createKibanaRequest();
+        siemMigrationsService.createClient({ spaceId: 'default', request });
+        expect(mockGetClient).toHaveBeenCalledWith({ spaceId: 'default', request });
+      });
+    });
+
+    describe('when stop is called', () => {
+      it('should trigger the pluginStop subject', async () => {
+        siemMigrationsService.stop();
+        expect(mockReplaySubject$.next).toHaveBeenCalled();
+        expect(mockReplaySubject$.complete).toHaveBeenCalled();
+      });
+    });
+  });
+
+  describe('without siemMigrationsEnabled flag', () => {
+    beforeEach(() => {
+      siemMigrationsService = new SiemMigrationsService(
+        { experimentalFeatures: { siemMigrationsEnabled: false } } as ConfigType,
+        logger,
+        kibanaVersion
+      );
+    });
+
+    it('should instantiate the rule migrations service', async () => {
+      expect(MockSiemRuleMigrationsService).toHaveBeenCalledWith(logger, kibanaVersion);
+    });
+
+    describe('when setup is called', () => {
+      it('should not call siemRuleMigrationsService setup', async () => {
+        siemMigrationsService.setup({ esClusterClient, tasksTimeoutMs: 100 });
+        expect(mockSetup).not.toHaveBeenCalled();
+      });
+    });
+  });
+});

x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import { ReplaySubject, type Subject } from 'rxjs';
+import type { ConfigType } from '../../config';
+import { SiemRuleMigrationsService } from './rules/siem_rule_migrations_service';
+import type {
+  SiemMigrationsClient,
+  SiemMigrationsSetupParams,
+  SiemMigrationsGetClientParams,
+} from './types';
+
+export class SiemMigrationsService {
+  private pluginStop$: Subject<void>;
+  private rules: SiemRuleMigrationsService;
+
+  constructor(private config: ConfigType, logger: Logger, kibanaVersion: string) {
+    this.pluginStop$ = new ReplaySubject(1);
+    this.rules = new SiemRuleMigrationsService(logger, kibanaVersion);
+  }
+
+  setup(params: SiemMigrationsSetupParams) {
+    if (this.config.experimentalFeatures.siemMigrationsEnabled) {
+      this.rules.setup({ ...params, pluginStop$: this.pluginStop$ });
+    }
+  }
+
+  createClient(params: SiemMigrationsGetClientParams): SiemMigrationsClient {
+    return {
+      rules: this.rules.getClient(params),
+    };
+  }
+
+  stop() {
+    this.pluginStop$.next();
+    this.pluginStop$.complete();
+  }
+}

x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IClusterClient } from '@kbn/core/server';
+import type { SiemRuleMigrationsClient, SiemRuleMigrationsGetClientParams } from './rules/types';
+
+export interface SiemMigrationsSetupParams {
+  esClusterClient: IClusterClient;
+  tasksTimeoutMs?: number;
+}
+
+export type SiemMigrationsGetClientParams = SiemRuleMigrationsGetClientParams;
+
+export interface SiemMigrationsClient {
+  rules: SiemRuleMigrationsClient;
+}

x-pack/plugins/security_solution/server/plugin.ts

@@ -123,6 +123,7 @@ import { getAssistantTools } from './assistant/tools';
 import { turnOffAgentPolicyFeatures } from './endpoint/migrations/turn_off_agent_policy_features';
 import { getCriblPackagePolicyPostCreateOrUpdateCallback } from './security_integrations';
 import { scheduleEntityAnalyticsMigration } from './lib/entity_analytics/migrations';
+import { SiemMigrationsService } from './lib/siem_migrations/siem_migrations_service';
 
 export type { SetupPlugins, StartPlugins, PluginSetup, PluginStart } from './plugin_contract';
 
@@ -135,6 +136,7 @@ export class Plugin implements ISecuritySolutionPlugin {
 
   private readonly ruleMonitoringService: IRuleMonitoringService;
   private readonly endpointAppContextService = new EndpointAppContextService();
+  private readonly siemMigrationsService: SiemMigrationsService;
   private readonly telemetryReceiver: ITelemetryReceiver;
   private readonly telemetryEventsSender: ITelemetryEventsSender;
   private readonly asyncTelemetryEventsSender: IAsyncTelemetryEventsSender;
@@ -160,6 +162,11 @@ export class Plugin implements ISecuritySolutionPlugin {
       this.logger,
       this.config.experimentalFeatures
     );
+    this.siemMigrationsService = new SiemMigrationsService(
+      this.config,
+      this.logger,
+      this.pluginContext.env.packageInfo.version
+    );
 
     this.ruleMonitoringService = createRuleMonitoringService(this.config, this.logger);
     this.telemetryEventsSender = new TelemetryEventsSender(this.logger);
@@ -236,6 +243,7 @@ export class Plugin implements ISecuritySolutionPlugin {
       plugins,
       endpointAppContextService: this.endpointAppContextService,
       ruleMonitoringService: this.ruleMonitoringService,
+      siemMigrationsService: this.siemMigrationsService,
       kibanaVersion: pluginContext.env.packageInfo.version,
       kibanaBranch: pluginContext.env.packageInfo.branch,
       buildFlavor: pluginContext.env.packageInfo.buildFlavor,
@@ -427,7 +435,7 @@ export class Plugin implements ISecuritySolutionPlugin {
 
     core
       .getStartServices()
-      .then(async ([_, depsStart]) => {
+      .then(async ([coreStart, depsStart]) => {
         appClientFactory.setup({
           getSpaceId: depsStart.spaces?.spacesService?.getSpaceId,
           config,
@@ -477,6 +485,8 @@ export class Plugin implements ISecuritySolutionPlugin {
          * Register a config for the security guide
          */
         plugins.guidedOnboarding?.registerGuideConfig(siemGuideId, getSiemGuideConfig());
+
+        this.siemMigrationsService.setup({ esClusterClient: coreStart.elasticsearch.client });
       })
       .catch(() => {}); // it shouldn't reject, but just in case
 
@@ -715,6 +725,7 @@ export class Plugin implements ISecuritySolutionPlugin {
     this.endpointAppContextService.stop();
     this.policyWatcher?.stop();
     this.completeExternalResponseActionsTask.stop().catch(() => {});
+    this.siemMigrationsService.stop();
     licenseService.stop();
   }
 }

x-pack/plugins/security_solution/server/request_context_factory.ts

@@ -32,6 +32,7 @@ import { AssetCriticalityDataClient } from './lib/entity_analytics/asset_critica
 import { createDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client';
 import { buildMlAuthz } from './lib/machine_learning/authz';
 import { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client';
+import type { SiemMigrationsService } from './lib/siem_migrations/siem_migrations_service';
 
 export interface IRequestContextFactory {
   create(
@@ -47,6 +48,7 @@ interface ConstructorOptions {
   plugins: SecuritySolutionPluginSetupDependencies;
   endpointAppContextService: EndpointAppContextService;
   ruleMonitoringService: IRuleMonitoringService;
+  siemMigrationsService: SiemMigrationsService;
   kibanaVersion: string;
   kibanaBranch: string;
   buildFlavor: BuildFlavor;
@@ -64,7 +66,14 @@ export class RequestContextFactory implements IRequestContextFactory {
     request: KibanaRequest
   ): Promise<SecuritySolutionApiRequestHandlerContext> {
     const { options, appClientFactory } = this;
-    const { config, core, plugins, endpointAppContextService, ruleMonitoringService } = options;
+    const {
+      config,
+      core,
+      plugins,
+      endpointAppContextService,
+      ruleMonitoringService,
+      siemMigrationsService,
+    } = options;
 
     const { lists, ruleRegistry, security } = plugins;
 
@@ -157,6 +166,10 @@ export class RequestContextFactory implements IRequestContextFactory {
         })
       ),
 
+      getSiemMigrationsClient: memoize(() =>
+        siemMigrationsService.createClient({ request, spaceId: getSpaceId() })
+      ),
+
       getExceptionListClient: () => {
         if (!lists) {
           return null;

x-pack/plugins/security_solution/server/routes/index.ts

@@ -61,6 +61,7 @@ import { suggestUserProfilesRoute } from '../lib/detection_engine/routes/users/s
 import { registerTimelineRoutes } from '../lib/timeline/routes';
 import { getFleetManagedIndexTemplatesRoute } from '../lib/security_integrations/cribl/routes';
 import { registerEntityAnalyticsRoutes } from '../lib/entity_analytics/register_entity_analytics_routes';
+import { registerSiemMigrationsRoutes } from '../lib/siem_migrations/routes';
 
 export const initRoutes = (
   router: SecuritySolutionPluginRouter,
@@ -138,13 +139,17 @@ export const initRoutes = (
   // Dashboards
   registerDashboardsRoutes(router, logger);
   registerTagsRoutes(router, logger);
+
   const { previewTelemetryUrlEnabled } = config.experimentalFeatures;
+
   if (previewTelemetryUrlEnabled) {
     // telemetry preview endpoint for e2e integration tests only at the moment.
     telemetryDetectionRulesPreviewRoute(router, logger, previewTelemetryReceiver, telemetrySender);
   }
 
   registerEntityAnalyticsRoutes({ router, config, getStartServices, logger });
+  registerSiemMigrationsRoutes(router, config, logger);
+
   // Security Integrations
   getFleetManagedIndexTemplatesRoute(router);
 };

x-pack/plugins/security_solution/server/types.ts

@@ -35,6 +35,7 @@ import type { RiskScoreDataClient } from './lib/entity_analytics/risk_score/risk
 import type { AssetCriticalityDataClient } from './lib/entity_analytics/asset_criticality';
 import type { IDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface';
 import type { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client';
+import type { SiemMigrationsClient } from './lib/siem_migrations/types';
 export { AppClient };
 
 export interface SecuritySolutionApiRequestHandlerContext {
@@ -57,6 +58,7 @@ export interface SecuritySolutionApiRequestHandlerContext {
   getRiskScoreDataClient: () => RiskScoreDataClient;
   getAssetCriticalityDataClient: () => AssetCriticalityDataClient;
   getEntityStoreDataClient: () => EntityStoreDataClient;
+  getSiemMigrationsClient: () => SiemMigrationsClient;
 }
 
 export type SecuritySolutionRequestHandlerContext = CustomRequestHandlerContext<{

x-pack/plugins/security_solution/tsconfig.json

@@ -231,5 +231,6 @@
     "@kbn/core-security-server-mocks",
     "@kbn/serverless",
     "@kbn/core-user-profile-browser",
+    "@kbn/data-stream-adapter",
   ]
 }

x-pack/test/api_integration/services/security_solution_api.gen.ts

@@ -32,6 +32,7 @@ import { CopyTimelineRequestBodyInput } from '@kbn/security-solution-plugin/comm
 import { CreateAlertsMigrationRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/signals_migration/create_signals_migration/create_signals_migration.gen';
 import { CreateAssetCriticalityRecordRequestBodyInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/asset_criticality/create_asset_criticality.gen';
 import { CreateRuleRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/crud/create_rule/create_rule_route.gen';
+import { CreateRuleMigrationRequestBodyInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen';
 import { CreateTimelinesRequestBodyInput } from '@kbn/security-solution-plugin/common/api/timeline/create_timelines/create_timelines_route.gen';
 import {
   CreateUpdateProtectionUpdatesNoteRequestParamsInput,
@@ -332,6 +333,17 @@ Migrations are initiated per index. While the process is neither destructive nor
         .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
         .send(props.body as object);
     },
+    /**
+     * Creates a new SIEM rules migration using the original vendor rules provided
+     */
+    createRuleMigration(props: CreateRuleMigrationProps, kibanaSpace: string = 'default') {
+      return supertest
+        .post(routeWithNamespace('/internal/siem_migrations/rules', kibanaSpace))
+        .set('kbn-xsrf', 'true')
+        .set(ELASTIC_HTTP_VERSION_HEADER, '1')
+        .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
+        .send(props.body as object);
+    },
     createTimelines(props: CreateTimelinesProps, kibanaSpace: string = 'default') {
       return supertest
         .post(routeWithNamespace('/api/timeline', kibanaSpace))
@@ -896,6 +908,16 @@ finalize it.
         .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
         .query(props.query);
     },
+    /**
+     * Retrieves the rule migrations stored in the system
+     */
+    getRuleMigration(kibanaSpace: string = 'default') {
+      return supertest
+        .get(routeWithNamespace('/internal/siem_migrations/rules', kibanaSpace))
+        .set('kbn-xsrf', 'true')
+        .set(ELASTIC_HTTP_VERSION_HEADER, '1')
+        .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana');
+    },
     getTimeline(props: GetTimelineProps, kibanaSpace: string = 'default') {
       return supertest
         .get(routeWithNamespace('/api/timeline', kibanaSpace))
@@ -1335,6 +1357,9 @@ export interface CreateAssetCriticalityRecordProps {
 export interface CreateRuleProps {
   body: CreateRuleRequestBodyInput;
 }
+export interface CreateRuleMigrationProps {
+  body: CreateRuleMigrationRequestBodyInput;
+}
 export interface CreateTimelinesProps {
   body: CreateTimelinesRequestBodyInput;
 }