[Rule Migration] Add telemetry events to translation graphs (#209352)

## Summary

This PR adds telemetry events to SIEM migration backend using the event
based telemetry already existing in security solutions.

Here is a list of events:

```typescript
export const SIEM_MIGRATIONS_MIGRATION_SUCCESS: EventTypeOpts<{
  model: string;
  migrationId: string;
  duration: number;
  completed: number;
  failed: number;
  total: number;
}

export const SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS: EventTypeOpts<{
  model: string;
  migrationId: string;
  duration: number;
  translationResult: string;
  prebuiltMatch: boolean;
}

export const SIEM_MIGRATIONS_PREBUILT_RULES_MATCH: EventTypeOpts<{
  model: string;
  migrationId: string;
  preFilterRuleNames: string[];
  preFilterRuleCount: number;
  postFilterRuleName: string;
  postFilterRuleCount: number;
}

export const SIEM_MIGRATIONS_INTEGRATIONS_MATCH: EventTypeOpts<{
  model: string;
  migrationId: string;
  preFilterIntegrationNames: string[];
  preFilterIntegrationCount: number;
  postFilterIntegrationName: string;
  postFilterIntegrationCount: number;
}

export const SIEM_MIGRATIONS_MIGRATION_FAILURE: EventTypeOpts<{
  model: string;
  error: string;
  migrationId: string;
  duration: number;
  completed: number;
  failed: number;
  total: number;
}

export const SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE: EventTypeOpts<{
  model: string;
  error: string;
  migrationId: string;
}
```

x-pack/solutions/security/plugins/security_solution/scripts/siem_migration/draw_graphs_script.ts

@@ -17,6 +17,7 @@ import fs from 'fs/promises';
 import path from 'path';
 import { getRuleMigrationAgent } from '../../server/lib/siem_migrations/rules/task/agent';
 import type { RuleMigrationsRetriever } from '../../server/lib/siem_migrations/rules/task/retrievers';
+import type { SiemMigrationTelemetryClient } from '../../server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client';
 
 interface Drawable {
   drawMermaidPng: () => Promise<Blob>;
@@ -36,12 +37,14 @@ const createLlmInstance = () => {
 
 async function getAgentGraph(logger: Logger): Promise<Drawable> {
   const model = createLlmInstance();
+  const telemetryClient = {} as SiemMigrationTelemetryClient;
   const graph = getRuleMigrationAgent({
     model,
     inferenceClient,
     ruleMigrationsRetriever,
     connectorId,
     logger,
+    telemetryClient,
   });
   return graph.getGraphAsync({ xray: true });
 }

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.test.ts

@@ -13,13 +13,14 @@ import type {
 import { loggerMock } from '@kbn/logging-mocks';
 import { FakeLLM } from '@langchain/core/utils/testing';
 import type { RuleMigrationsRetriever } from '../retrievers';
+import type { SiemMigrationTelemetryClient } from '../rule_migrations_telemetry_client';
 import { getRuleMigrationAgent } from './graph';
 
 describe('getRuleMigrationAgent', () => {
   const model = new FakeLLM({
     response: JSON.stringify({}, null, 2),
   }) as unknown as ActionsClientChatOpenAI | ActionsClientSimpleChatModel;
-
+  const telemetryClient = {} as SiemMigrationTelemetryClient;
   const inferenceClient = {} as InferenceClient;
   const connectorId = 'draw_graphs';
   const ruleMigrationsRetriever = {} as RuleMigrationsRetriever;
@@ -33,6 +34,7 @@ describe('getRuleMigrationAgent', () => {
         ruleMigrationsRetriever,
         connectorId,
         logger,
+        telemetryClient,
       });
     } catch (error) {
       throw Error(`getRuleMigrationAgent threw an error: ${error}`);

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts

@@ -8,28 +8,29 @@
 import { END, START, StateGraph } from '@langchain/langgraph';
 import { getCreateSemanticQueryNode } from './nodes/create_semantic_query';
 import { getMatchPrebuiltRuleNode } from './nodes/match_prebuilt_rule';
-
 import { migrateRuleState } from './state';
 import { getTranslateRuleGraph } from './sub_graphs/translate_rule';
 import type { MigrateRuleGraphParams, MigrateRuleState } from './types';
-
 export function getRuleMigrationAgent({
   model,
   inferenceClient,
   ruleMigrationsRetriever,
   connectorId,
   logger,
+  telemetryClient,
 }: MigrateRuleGraphParams) {
   const matchPrebuiltRuleNode = getMatchPrebuiltRuleNode({
     model,
     logger,
     ruleMigrationsRetriever,
+    telemetryClient,
   });
   const translationSubGraph = getTranslateRuleGraph({
     model,
     inferenceClient,
     ruleMigrationsRetriever,
     connectorId,
+    telemetryClient,
     logger,
   });
   const createSemanticQueryNode = getCreateSemanticQueryNode({ model });

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts

@@ -13,14 +13,16 @@ import {
   RuleTranslationResult,
 } from '../../../../../../../../common/siem_migrations/constants';
 import type { RuleMigrationsRetriever } from '../../../retrievers';
+import type { SiemMigrationTelemetryClient } from '../../../rule_migrations_telemetry_client';
 import type { ChatModel } from '../../../util/actions_client_chat';
+import { cleanMarkdown, generateAssistantComment } from '../../../util/comments';
 import type { GraphNode } from '../../types';
 import { MATCH_PREBUILT_RULE_PROMPT } from './prompts';
-import { cleanMarkdown, generateAssistantComment } from '../../../util/comments';
 
 interface GetMatchPrebuiltRuleNodeParams {
   model: ChatModel;
   logger: Logger;
+  telemetryClient: SiemMigrationTelemetryClient;
   ruleMigrationsRetriever: RuleMigrationsRetriever;
 }
 
@@ -32,6 +34,7 @@ interface GetMatchedRuleResponse {
 export const getMatchPrebuiltRuleNode = ({
   model,
   ruleMigrationsRetriever,
+  telemetryClient,
   logger,
 }: GetMatchPrebuiltRuleNodeParams): GraphNode => {
   return async (state) => {
@@ -42,6 +45,8 @@ export const getMatchPrebuiltRuleNode = ({
       techniqueIds.join(',')
     );
     if (prebuiltRules.length === 0) {
+      telemetryClient.reportPrebuiltRulesMatch({ preFilterRules: [] });
+
       return {
         comments: [
           generateAssistantComment(
@@ -80,6 +85,10 @@ export const getMatchPrebuiltRuleNode = ({
 
     if (response.match) {
       const matchedRule = prebuiltRules.find((r) => r.name === response.match);
+      telemetryClient.reportPrebuiltRulesMatch({
+        preFilterRules: prebuiltRules,
+        postFilterRule: matchedRule,
+      });
       if (matchedRule) {
         return {
           comments,

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts

@@ -9,12 +9,12 @@ import { END, START, StateGraph } from '@langchain/langgraph';
 import { isEmpty } from 'lodash/fp';
 import { RuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants';
 import { getEcsMappingNode } from './nodes/ecs_mapping';
-import { translationResultNode } from './nodes/translation_result';
 import { getFixQueryErrorsNode } from './nodes/fix_query_errors';
+import { getInlineQueryNode } from './nodes/inline_query';
 import { getRetrieveIntegrationsNode } from './nodes/retrieve_integrations';
 import { getTranslateRuleNode } from './nodes/translate_rule';
+import { getTranslationResultNode } from './nodes/translation_result';
 import { getValidationNode } from './nodes/validation';
-import { getInlineQueryNode } from './nodes/inline_query';
 import { translateRuleState } from './state';
 import type { TranslateRuleGraphParams, TranslateRuleState } from './types';
 
@@ -27,16 +27,22 @@ export function getTranslateRuleGraph({
   connectorId,
   ruleMigrationsRetriever,
   logger,
+  telemetryClient,
 }: TranslateRuleGraphParams) {
   const translateRuleNode = getTranslateRuleNode({
     inferenceClient,
     connectorId,
     logger,
   });
+  const translationResultNode = getTranslationResultNode();
   const inlineQueryNode = getInlineQueryNode({ model, ruleMigrationsRetriever });
   const validationNode = getValidationNode({ logger });
   const fixQueryErrorsNode = getFixQueryErrorsNode({ inferenceClient, connectorId, logger });
-  const retrieveIntegrationsNode = getRetrieveIntegrationsNode({ model, ruleMigrationsRetriever });
+  const retrieveIntegrationsNode = getRetrieveIntegrationsNode({
+    model,
+    ruleMigrationsRetriever,
+    telemetryClient,
+  });
   const ecsMappingNode = getEcsMappingNode({ inferenceClient, connectorId, logger });
 
   const translateRuleGraph = new StateGraph(translateRuleState)

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts

@@ -7,13 +7,15 @@
 
 import { JsonOutputParser } from '@langchain/core/output_parsers';
 import type { RuleMigrationsRetriever } from '../../../../../retrievers';
+import type { SiemMigrationTelemetryClient } from '../../../../../rule_migrations_telemetry_client';
 import type { ChatModel } from '../../../../../util/actions_client_chat';
+import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments';
 import type { GraphNode } from '../../types';
 import { MATCH_INTEGRATION_PROMPT } from './prompts';
-import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments';
 
 interface GetRetrieveIntegrationsNodeParams {
   model: ChatModel;
+  telemetryClient: SiemMigrationTelemetryClient;
   ruleMigrationsRetriever: RuleMigrationsRetriever;
 }
 
@@ -25,12 +27,16 @@ interface GetMatchedIntegrationResponse {
 export const getRetrieveIntegrationsNode = ({
   model,
   ruleMigrationsRetriever,
+  telemetryClient,
 }: GetRetrieveIntegrationsNodeParams): GraphNode => {
   return async (state) => {
     const query = state.semantic_query;
 
     const integrations = await ruleMigrationsRetriever.integrations.getIntegrations(query);
     if (integrations.length === 0) {
+      telemetryClient.reportIntegrationsMatch({
+        preFilterIntegrations: [],
+      });
       return {
         comments: [
           generateAssistantComment(
@@ -67,6 +73,10 @@ export const getRetrieveIntegrationsNode = ({
 
     if (response.match) {
       const matchedIntegration = integrations.find((r) => r.title === response.match);
+      telemetryClient.reportIntegrationsMatch({
+        preFilterIntegrations: integrations,
+        postFilterIntegration: matchedIntegration,
+      });
       if (matchedIntegration) {
         return { integration: matchedIntegration, comments };
       }

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts

@@ -7,10 +7,10 @@
 
 import type { Logger } from '@kbn/core/server';
 import type { InferenceClient } from '@kbn/inference-plugin/server';
+import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments';
 import { getEsqlKnowledgeBase } from '../../../../../util/esql_knowledge_base_caller';
 import type { GraphNode } from '../../types';
 import { ESQL_SYNTAX_TRANSLATION_PROMPT } from './prompts';
-import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments';
 
 interface GetTranslateRuleNodeParams {
   inferenceClient: InferenceClient;

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/index.ts

@@ -4,4 +4,4 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-export { translationResultNode } from './translation_result';
+export { getTranslationResultNode } from './translation_result';

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts

@@ -12,36 +12,38 @@ import {
 } from '../../../../../../../../../../common/siem_migrations/constants';
 import type { GraphNode } from '../../types';
 
-export const translationResultNode: GraphNode = async (state) => {
-  // Set defaults
-  const elasticRule = {
-    title: state.original_rule.title,
-    description: state.original_rule.description || state.original_rule.title,
-    severity: DEFAULT_TRANSLATION_SEVERITY,
-    risk_score: DEFAULT_TRANSLATION_RISK_SCORE,
-    ...state.elastic_rule,
-  };
+export const getTranslationResultNode = (): GraphNode => {
+  return async (state) => {
+    // Set defaults
+    const elasticRule = {
+      title: state.original_rule.title,
+      description: state.original_rule.description || state.original_rule.title,
+      severity: DEFAULT_TRANSLATION_SEVERITY,
+      risk_score: DEFAULT_TRANSLATION_RISK_SCORE,
+      ...state.elastic_rule,
+    };
 
-  const query = elasticRule.query;
-  let translationResult;
+    const query = elasticRule.query;
+    let translationResult;
 
-  if (!query) {
-    translationResult = RuleTranslationResult.UNTRANSLATABLE;
-  } else {
-    if (query.startsWith('FROM logs-*')) {
-      elasticRule.query = query.replace('FROM logs-*', 'FROM [indexPattern]');
-      translationResult = RuleTranslationResult.PARTIAL;
-    } else if (state.validation_errors?.esql_errors) {
-      translationResult = RuleTranslationResult.PARTIAL;
-    } else if (query.match(/\[(macro|lookup):.*?\]/)) {
-      translationResult = RuleTranslationResult.PARTIAL;
+    if (!query) {
+      translationResult = RuleTranslationResult.UNTRANSLATABLE;
     } else {
-      translationResult = RuleTranslationResult.FULL;
+      if (query.startsWith('FROM logs-*')) {
+        elasticRule.query = query.replace('FROM logs-*', 'FROM [indexPattern]');
+        translationResult = RuleTranslationResult.PARTIAL;
+      } else if (state.validation_errors?.esql_errors) {
+        translationResult = RuleTranslationResult.PARTIAL;
+      } else if (query.match(/\[(macro|lookup):.*?\]/)) {
+        translationResult = RuleTranslationResult.PARTIAL;
+      } else {
+        translationResult = RuleTranslationResult.FULL;
+      }
     }
-  }
 
-  return {
-    elastic_rule: elasticRule,
-    translation_result: translationResult,
+    return {
+      elastic_rule: elasticRule,
+      translation_result: translationResult,
+    };
   };
 };

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/types.ts

@@ -8,6 +8,7 @@
 import type { Logger } from '@kbn/core/server';
 import type { InferenceClient } from '@kbn/inference-plugin/server';
 import type { RuleMigrationsRetriever } from '../../../retrievers';
+import type { SiemMigrationTelemetryClient } from '../../../rule_migrations_telemetry_client';
 import type { ChatModel } from '../../../util/actions_client_chat';
 import type { translateRuleState } from './state';
 
@@ -19,6 +20,7 @@ export interface TranslateRuleGraphParams {
   inferenceClient: InferenceClient;
   connectorId: string;
   ruleMigrationsRetriever: RuleMigrationsRetriever;
+  telemetryClient: SiemMigrationTelemetryClient;
   logger: Logger;
 }
 

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts

@@ -8,6 +8,7 @@
 import type { Logger } from '@kbn/core/server';
 import type { InferenceClient } from '@kbn/inference-plugin/server';
 import type { RuleMigrationsRetriever } from '../retrievers';
+import type { SiemMigrationTelemetryClient } from '../rule_migrations_telemetry_client';
 import type { ChatModel } from '../util/actions_client_chat';
 import type { migrateRuleState } from './state';
 
@@ -20,4 +21,5 @@ export interface MigrateRuleGraphParams {
   connectorId: string;
   ruleMigrationsRetriever: RuleMigrationsRetriever;
   logger: Logger;
+  telemetryClient: SiemMigrationTelemetryClient;
 }

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts

@@ -20,13 +20,16 @@ import type { SiemRuleMigrationsClientDependencies } from '../types';
 import { getRuleMigrationAgent } from './agent';
 import type { MigrateRuleState } from './agent/types';
 import { RuleMigrationsRetriever } from './retrievers';
+import { SiemMigrationTelemetryClient } from './rule_migrations_telemetry_client';
 import type {
   MigrationAgent,
   RuleMigrationTaskCreateAgentParams,
+  RuleMigrationTaskRunParams,
   RuleMigrationTaskStartParams,
   RuleMigrationTaskStartResult,
   RuleMigrationTaskStopResult,
 } from './types';
+import type { ChatModel } from './util/actions_client_chat';
 import { ActionsClientChat } from './util/actions_client_chat';
 import { generateAssistantComment } from './util/comments';
 
@@ -46,7 +49,7 @@ export class RuleMigrationsTaskClient {
 
   /** Starts a rule migration task */
   async start(params: RuleMigrationTaskStartParams): Promise<RuleMigrationTaskStartResult> {
-    const { migrationId } = params;
+    const { migrationId, connectorId } = params;
     if (this.migrationsRunning.has(migrationId)) {
       return { exists: true, started: false };
     }
@@ -65,24 +68,25 @@ export class RuleMigrationsTaskClient {
     if (rules.pending === 0) {
       return { exists: true, started: false };
     }
+    const abortController = new AbortController();
+    const model = await this.createModel(connectorId, abortController);
 
     // run the migration without awaiting it to execute it in the background
-    this.run(params).catch((error) => {
+    this.run({ ...params, model, abortController }).catch((error) => {
       this.logger.error(`Error executing migration ID:${migrationId}`, error);
     });
 
     return { exists: true, started: true };
   }
 
-  private async run(params: RuleMigrationTaskStartParams): Promise<void> {
-    const { migrationId, invocationConfig } = params;
+  private async run(params: RuleMigrationTaskRunParams): Promise<void> {
+    const { migrationId, invocationConfig, abortController, model } = params;
     if (this.migrationsRunning.has(migrationId)) {
       // This should never happen, but just in case
       throw new Error(`Task already running for migration ID:${migrationId} `);
     }
     this.logger.info(`Starting migration ID:${migrationId}`);
 
-    const abortController = new AbortController();
     this.migrationsRunning.set(migrationId, { user: this.currentUser.username, abortController });
 
     const abortPromise = abortSignalToPromise(abortController.signal);
@@ -93,15 +97,22 @@ export class RuleMigrationsTaskClient {
       await withAbortRace(new Promise((resolve) => setTimeout(resolve, seconds * 1000)));
     };
 
+    const stats = { completed: 0, failed: 0 };
+    const telemetryClient = new SiemMigrationTelemetryClient(
+      this.dependencies.telemetry,
+      migrationId,
+      model.model
+    );
+    const endSiemMigration = telemetryClient.startSiemMigration();
     try {
       this.logger.debug(`Creating agent for migration ID:${migrationId}`);
-      const agent = await withAbortRace(this.createAgent({ ...params, abortController }));
+
+      const agent = await withAbortRace(this.createAgent({ ...params, model, telemetryClient }));
 
       const config: RunnableConfig = {
         ...invocationConfig,
         // signal: abortController.signal, // not working properly https://github.com/langchain-ai/langgraphjs/issues/319
       };
-
       let isDone: boolean = false;
       do {
         const ruleMigrations = await this.data.rules.takePending(migrationId, ITERATION_BATCH_SIZE);
@@ -116,34 +127,36 @@ export class RuleMigrationsTaskClient {
               await this.data.rules.saveCompleted(ruleMigration);
               return; // skip already installed rules
             }
+            const endRuleTranslation = telemetryClient.startRuleTranslation();
             try {
-              const start = Date.now();
-
-              const invocationData = { original_rule: ruleMigration.original_rule };
+              const invocationData = {
+                original_rule: ruleMigration.original_rule,
+              };
 
               // using withAbortRace is a workaround for the issue with the langGraph signal not working properly
               const migrationResult = await withAbortRace<MigrateRuleState>(
                 agent.invoke(invocationData, config)
               );
 
-              const duration = (Date.now() - start) / 1000;
               this.logger.debug(
-                `Migration of rule "${ruleMigration.original_rule.title}" finished in ${duration}s`
+                `Migration of rule "${ruleMigration.original_rule.title}" finished`
               );
-
+              endRuleTranslation({ migrationResult });
               await this.data.rules.saveCompleted({
                 ...ruleMigration,
                 elastic_rule: migrationResult.elastic_rule,
                 translation_result: migrationResult.translation_result,
                 comments: migrationResult.comments,
               });
+              stats.completed++;
             } catch (error) {
+              stats.failed++;
               if (error instanceof AbortError) {
                 throw error;
               }
+              endRuleTranslation({ error });
               this.logger.error(
-                `Error migrating rule "${ruleMigration.original_rule.title}"`,
-                error
+                `Error migrating rule "${ruleMigration.original_rule.title} with error: ${error.message}"`
               );
               await this.data.rules.saveError({
                 ...ruleMigration,
@@ -163,6 +176,8 @@ export class RuleMigrationsTaskClient {
       } while (!isDone);
 
       this.logger.info(`Finished migration ID:${migrationId}`);
+
+      endSiemMigration({ stats });
     } catch (error) {
       await this.data.rules.releaseProcessing(migrationId);
 
@@ -170,6 +185,7 @@ export class RuleMigrationsTaskClient {
         this.logger.info(`Abort signal received, stopping migration ID:${migrationId}`);
         return;
       } else {
+        endSiemMigration({ error, stats });
         this.logger.error(`Error processing migration ID:${migrationId} ${error}`);
       }
     } finally {
@@ -179,17 +195,12 @@ export class RuleMigrationsTaskClient {
   }
 
   private async createAgent({
-    migrationId,
     connectorId,
-    abortController,
+    migrationId,
+    model,
+    telemetryClient,
   }: RuleMigrationTaskCreateAgentParams): Promise<MigrationAgent> {
-    const { inferenceClient, actionsClient, rulesClient, savedObjectsClient } = this.dependencies;
-
-    const actionsClientChat = new ActionsClientChat(connectorId, actionsClient, this.logger);
-    const model = await actionsClientChat.createModel({
-      signal: abortController.signal,
-      temperature: 0.05,
-    });
+    const { inferenceClient, rulesClient, savedObjectsClient } = this.dependencies;
 
     const ruleMigrationsRetriever = new RuleMigrationsRetriever(migrationId, {
       data: this.data,
@@ -204,6 +215,7 @@ export class RuleMigrationsTaskClient {
       model,
       inferenceClient,
       ruleMigrationsRetriever,
+      telemetryClient,
       logger: this.logger,
     });
   }
@@ -274,4 +286,17 @@ export class RuleMigrationsTaskClient {
       return { exists: true, stopped: false };
     }
   }
+
+  private async createModel(
+    connectorId: string,
+    abortController: AbortController
+  ): Promise<ChatModel> {
+    const { actionsClient } = this.dependencies;
+    const actionsClientChat = new ActionsClientChat(connectorId, actionsClient, this.logger);
+    const model = await actionsClientChat.createModel({
+      signal: abortController.signal,
+      temperature: 0.05,
+    });
+    return model;
+  }
 }

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client.ts

@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { AnalyticsServiceSetup } from '@kbn/core/public';
+import {
+  SIEM_MIGRATIONS_INTEGRATIONS_MATCH,
+  SIEM_MIGRATIONS_MIGRATION_FAILURE,
+  SIEM_MIGRATIONS_MIGRATION_SUCCESS,
+  SIEM_MIGRATIONS_PREBUILT_RULES_MATCH,
+  SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE,
+  SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS,
+} from '../../../telemetry/event_based/events';
+import type { RuleMigrationIntegration, RuleSemanticSearchResult } from '../types';
+import type { MigrateRuleState } from './agent/types';
+
+interface IntegrationMatchEvent {
+  preFilterIntegrations: RuleMigrationIntegration[];
+  postFilterIntegration?: RuleMigrationIntegration;
+}
+
+interface PrebuiltRuleMatchEvent {
+  preFilterRules: RuleSemanticSearchResult[];
+  postFilterRule?: RuleSemanticSearchResult;
+}
+
+interface RuleTranslationEvent {
+  error?: string;
+  migrationResult?: MigrateRuleState;
+}
+
+interface SiemMigrationEvent {
+  error?: string;
+  stats: {
+    failed: number;
+    completed: number;
+  };
+}
+
+export class SiemMigrationTelemetryClient {
+  constructor(
+    private readonly telemetry: AnalyticsServiceSetup,
+    private readonly migrationId: string,
+    private readonly modelName: string | undefined
+  ) {
+    this.modelName = modelName || '';
+  }
+
+  public reportIntegrationsMatch({
+    preFilterIntegrations,
+    postFilterIntegration,
+  }: IntegrationMatchEvent): void {
+    this.telemetry.reportEvent(SIEM_MIGRATIONS_INTEGRATIONS_MATCH.eventType, {
+      model: this.modelName,
+      migrationId: this.migrationId,
+      preFilterIntegrationNames: preFilterIntegrations.map((integration) => integration.id) || [],
+      preFilterIntegrationCount: preFilterIntegrations.length,
+      postFilterIntegrationName: postFilterIntegration ? postFilterIntegration.id : '',
+      postFilterIntegrationCount: postFilterIntegration ? 1 : 0,
+    });
+  }
+  public reportPrebuiltRulesMatch({
+    preFilterRules,
+    postFilterRule,
+  }: PrebuiltRuleMatchEvent): void {
+    this.telemetry.reportEvent(SIEM_MIGRATIONS_PREBUILT_RULES_MATCH.eventType, {
+      model: this.modelName,
+      migrationId: this.migrationId,
+      preFilterRuleNames: preFilterRules.map((rule) => rule.rule_id) || [],
+      preFilterRuleCount: preFilterRules.length,
+      postFilterRuleNames: postFilterRule ? postFilterRule.rule_id : '',
+      postFilterRuleCount: postFilterRule ? 1 : 0,
+    });
+  }
+  public startRuleTranslation(): (
+    args: Pick<RuleTranslationEvent, 'error' | 'migrationResult'>
+  ) => void {
+    const startTime = Date.now();
+
+    return ({ error, migrationResult }) => {
+      const duration = Date.now() - startTime;
+
+      if (error) {
+        this.telemetry.reportEvent(SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE.eventType, {
+          migrationId: this.migrationId,
+          error,
+          model: this.modelName,
+        });
+        return;
+      }
+
+      this.telemetry.reportEvent(SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS.eventType, {
+        migrationId: this.migrationId,
+        translationResult: migrationResult?.translation_result,
+        duration,
+        model: this.modelName,
+        prebuiltMatch: migrationResult?.elastic_rule?.prebuilt_rule_id ? true : false,
+      });
+    };
+  }
+  public startSiemMigration(): (args: Pick<SiemMigrationEvent, 'error' | 'stats'>) => void {
+    const startTime = Date.now();
+
+    return ({ error, stats }) => {
+      const duration = Date.now() - startTime;
+      const total = stats?.completed + stats?.failed;
+
+      if (error) {
+        this.telemetry.reportEvent(SIEM_MIGRATIONS_MIGRATION_FAILURE.eventType, {
+          migrationId: this.migrationId,
+          model: this.modelName || '',
+          completed: stats?.completed,
+          failed: stats?.failed,
+          total,
+          duration,
+          error,
+        });
+        return;
+      }
+
+      this.telemetry.reportEvent(SIEM_MIGRATIONS_MIGRATION_SUCCESS.eventType, {
+        migrationId: this.migrationId,
+        model: this.modelName || '',
+        completed: stats?.completed,
+        failed: stats?.failed,
+        total,
+        duration,
+      });
+    };
+  }
+}

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts

@@ -10,6 +10,8 @@ import type { RunnableConfig } from '@langchain/core/runnables';
 import type { RuleMigrationsDataClient } from '../data/rule_migrations_data_client';
 import type { SiemRuleMigrationsClientDependencies } from '../types';
 import type { getRuleMigrationAgent } from './agent';
+import type { SiemMigrationTelemetryClient } from './rule_migrations_telemetry_client';
+import type { ChatModel } from './util/actions_client_chat';
 
 export type MigrationAgent = ReturnType<typeof getRuleMigrationAgent>;
 
@@ -25,10 +27,16 @@ export interface RuleMigrationTaskStartParams {
   invocationConfig: RunnableConfig;
 }
 
-export interface RuleMigrationTaskCreateAgentParams extends RuleMigrationTaskStartParams {
+export interface RuleMigrationTaskRunParams extends RuleMigrationTaskStartParams {
+  model: ChatModel;
   abortController: AbortController;
 }
 
+export interface RuleMigrationTaskCreateAgentParams extends RuleMigrationTaskStartParams {
+  telemetryClient: SiemMigrationTelemetryClient;
+  model: ChatModel;
+}
+
 export interface RuleMigrationTaskStartResult {
   started: boolean;
   exists: boolean;

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts

@@ -5,14 +5,15 @@
  * 2.0.
  */
 
+import type { ActionsClient } from '@kbn/actions-plugin/server';
+import type { RulesClient } from '@kbn/alerting-plugin/server';
+import type { AnalyticsServiceSetup } from '@kbn/core/public';
 import type { SavedObjectsClientContract } from '@kbn/core/server';
 import type { PackageService } from '@kbn/fleet-plugin/server';
-import type { RulesClient } from '@kbn/alerting-plugin/server';
-import type { ActionsClient } from '@kbn/actions-plugin/server';
 import type { InferenceClient } from '@kbn/inference-plugin/server';
 import type {
-  UpdateRuleMigrationData,
   RuleMigrationTranslationResult,
+  UpdateRuleMigrationData,
 } from '../../../../common/siem_migrations/model/rule_migration.gen';
 import {
   type RuleMigration,
@@ -31,6 +32,7 @@ export interface SiemRuleMigrationsClientDependencies {
   actionsClient: ActionsClient;
   savedObjectsClient: SavedObjectsClientContract;
   packageService?: PackageService;
+  telemetry: AnalyticsServiceSetup;
 }
 
 export interface RuleMigrationIntegration {

x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/event_based/events.ts

@@ -5,12 +5,12 @@
  * 2.0.
  */
 import type { EventTypeOpts } from '@kbn/core/server';
+import type { BulkUpsertAssetCriticalityRecordsResponse } from '../../../../common/api/entity_analytics';
 import type {
   ResponseActionAgentType,
   ResponseActionStatus,
   ResponseActionsApiCommandNames,
 } from '../../../../common/endpoint/service/response_actions/constants';
-import type { BulkUpsertAssetCriticalityRecordsResponse } from '../../../../common/api/entity_analytics';
 import type { DataStreams, IlmPolicies, IlmsStats, IndicesStats } from '../indices.metadata.types';
 
 export const RISK_SCORE_EXECUTION_SUCCESS_EVENT: EventTypeOpts<{
@@ -701,6 +701,285 @@ export const ENDPOINT_RESPONSE_ACTION_STATUS_CHANGE_EVENT: EventTypeOpts<{
   },
 };
 
+export const SIEM_MIGRATIONS_MIGRATION_SUCCESS: EventTypeOpts<{
+  model: string;
+  migrationId: string;
+  duration: number;
+  completed: number;
+  failed: number;
+  total: number;
+}> = {
+  eventType: 'siem_migrations_migration_success',
+  schema: {
+    model: {
+      type: 'keyword',
+      _meta: {
+        description: 'The LLM model that was used',
+      },
+    },
+    migrationId: {
+      type: 'keyword',
+      _meta: {
+        description: 'Unique identifier for the migration',
+      },
+    },
+    duration: {
+      type: 'long',
+      _meta: {
+        description: 'Duration of the migration in milliseconds',
+      },
+    },
+    completed: {
+      type: 'long',
+      _meta: {
+        description: 'Number of rules successfully migrated',
+      },
+    },
+    failed: {
+      type: 'long',
+      _meta: {
+        description: 'Number of rules that failed to migrate',
+      },
+    },
+    total: {
+      type: 'long',
+      _meta: {
+        description: 'Total number of rules to migrate',
+      },
+    },
+  },
+};
+
+export const SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS: EventTypeOpts<{
+  model: string;
+  migrationId: string;
+  duration: number;
+  translationResult: string;
+  prebuiltMatch: boolean;
+}> = {
+  eventType: 'siem_migrations_rule_translation_success',
+  schema: {
+    translationResult: {
+      type: 'keyword',
+      _meta: {
+        description: 'Describes if the translation was full or partial',
+      },
+    },
+    model: {
+      type: 'keyword',
+      _meta: {
+        description: 'The LLM model that was used',
+      },
+    },
+    migrationId: {
+      type: 'keyword',
+      _meta: {
+        description: 'Unique identifier for the migration',
+      },
+    },
+    duration: {
+      type: 'long',
+      _meta: {
+        description: 'Duration of the migration in milliseconds',
+      },
+    },
+    prebuiltMatch: {
+      type: 'boolean',
+      _meta: {
+        description: 'Whether a prebuilt rule was matched',
+      },
+    },
+  },
+};
+
+export const SIEM_MIGRATIONS_PREBUILT_RULES_MATCH: EventTypeOpts<{
+  model: string;
+  migrationId: string;
+  preFilterRuleNames: string[];
+  preFilterRuleCount: number;
+  postFilterRuleName: string;
+  postFilterRuleCount: number;
+}> = {
+  eventType: 'siem_migrations_prebuilt_rules_match',
+  schema: {
+    model: {
+      type: 'keyword',
+      _meta: {
+        description: 'The LLM model that was used',
+      },
+    },
+    migrationId: {
+      type: 'keyword',
+      _meta: {
+        description: 'Unique identifier for the migration',
+      },
+    },
+    preFilterRuleNames: {
+      type: 'array',
+      items: {
+        type: 'keyword',
+        _meta: {
+          description: 'List of matched rules from Semantic search before LLM filtering',
+        },
+      },
+    },
+    preFilterRuleCount: {
+      type: 'long',
+      _meta: {
+        description: 'Count of rules matched before LLM filtering',
+      },
+    },
+    postFilterRuleName: {
+      type: 'keyword',
+      _meta: {
+        description: 'List of matched rules from Semantic search after LLM filtering',
+      },
+    },
+    postFilterRuleCount: {
+      type: 'long',
+      _meta: {
+        description: 'Count of rules matched before LLM filtering',
+      },
+    },
+  },
+};
+
+export const SIEM_MIGRATIONS_INTEGRATIONS_MATCH: EventTypeOpts<{
+  model: string;
+  migrationId: string;
+  preFilterIntegrationNames: string[];
+  preFilterIntegrationCount: number;
+  postFilterIntegrationName: string;
+  postFilterIntegrationCount: number;
+}> = {
+  eventType: 'siem_migrations_integration_match',
+  schema: {
+    model: {
+      type: 'keyword',
+      _meta: {
+        description: 'The LLM model that was used',
+      },
+    },
+    migrationId: {
+      type: 'keyword',
+      _meta: {
+        description: 'Unique identifier for the migration',
+      },
+    },
+    preFilterIntegrationNames: {
+      type: 'array',
+      items: {
+        type: 'keyword',
+        _meta: {
+          description: 'List of matched integrations from Semantic search before LLM filtering',
+        },
+      },
+    },
+    preFilterIntegrationCount: {
+      type: 'long',
+      _meta: {
+        description: 'Count of integrations matched before LLM filtering',
+      },
+    },
+    postFilterIntegrationName: {
+      type: 'keyword',
+      _meta: {
+        description: 'List of matched integrations from Semantic search after LLM filtering',
+      },
+    },
+    postFilterIntegrationCount: {
+      type: 'long',
+      _meta: {
+        description: 'Count of integrations matched before LLM filtering',
+      },
+    },
+  },
+};
+
+export const SIEM_MIGRATIONS_MIGRATION_FAILURE: EventTypeOpts<{
+  model: string;
+  error: string;
+  migrationId: string;
+  duration: number;
+  completed: number;
+  failed: number;
+  total: number;
+}> = {
+  eventType: 'siem_migrations_migration_failure',
+  schema: {
+    error: {
+      type: 'keyword',
+      _meta: {
+        description: 'Error message for the migration failure',
+      },
+    },
+    model: {
+      type: 'keyword',
+      _meta: {
+        description: 'The LLM model that was used',
+      },
+    },
+    migrationId: {
+      type: 'keyword',
+      _meta: {
+        description: 'Unique identifier for the migration',
+      },
+    },
+    duration: {
+      type: 'long',
+      _meta: {
+        description: 'Duration of the migration in milliseconds',
+      },
+    },
+    completed: {
+      type: 'long',
+      _meta: {
+        description: 'Number of rules successfully migrated',
+      },
+    },
+    failed: {
+      type: 'long',
+      _meta: {
+        description: 'Number of rules that failed to migrate',
+      },
+    },
+    total: {
+      type: 'long',
+      _meta: {
+        description: 'Total number of rules to migrate',
+      },
+    },
+  },
+};
+
+export const SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE: EventTypeOpts<{
+  model: string;
+  error: string;
+  migrationId: string;
+}> = {
+  eventType: 'siem_migrations_rule_translation_failure',
+  schema: {
+    error: {
+      type: 'keyword',
+      _meta: {
+        description: 'Error message for the translation failure',
+      },
+    },
+    model: {
+      type: 'keyword',
+      _meta: {
+        description: 'The LLM model that was used',
+      },
+    },
+    migrationId: {
+      type: 'keyword',
+      _meta: {
+        description: 'Unique identifier for the migration',
+      },
+    },
+  },
+};
+
 export const events = [
   RISK_SCORE_EXECUTION_SUCCESS_EVENT,
   RISK_SCORE_EXECUTION_ERROR_EVENT,
@@ -719,4 +998,10 @@ export const events = [
   TELEMETRY_ILM_POLICY_EVENT,
   TELEMETRY_ILM_STATS_EVENT,
   TELEMETRY_INDEX_STATS_EVENT,
+  SIEM_MIGRATIONS_MIGRATION_SUCCESS,
+  SIEM_MIGRATIONS_MIGRATION_FAILURE,
+  SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS,
+  SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE,
+  SIEM_MIGRATIONS_PREBUILT_RULES_MATCH,
+  SIEM_MIGRATIONS_INTEGRATIONS_MATCH,
 ];

x-pack/solutions/security/plugins/security_solution/server/request_context_factory.ts

@@ -7,15 +7,27 @@
 
 import { memoize } from 'lodash';
 
-import type { Logger, KibanaRequest, RequestHandlerContext } from '@kbn/core/server';
+import type { KibanaRequest, Logger, RequestHandlerContext } from '@kbn/core/server';
 
 import type { BuildFlavor } from '@kbn/config';
 import { EntityDiscoveryApiKeyType } from '@kbn/entityManager-plugin/server/saved_objects';
-import { getApiKeyManager } from './lib/entity_analytics/entity_store/auth/api_key';
 import { DEFAULT_SPACE_ID } from '../common/constants';
+import type { Immutable } from '../common/endpoint/types';
+import type { EndpointAuthz } from '../common/endpoint/types/authz';
 import { AppClientFactory } from './client';
 import type { ConfigType } from './config';
+import type { EndpointAppContextService } from './endpoint/endpoint_app_context_services';
+import { AssetInventoryDataClient } from './lib/asset_inventory/asset_inventory_data_client';
+import { createDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client';
 import type { IRuleMonitoringService } from './lib/detection_engine/rule_monitoring';
+import { AssetCriticalityDataClient } from './lib/entity_analytics/asset_criticality';
+import { getApiKeyManager } from './lib/entity_analytics/entity_store/auth/api_key';
+import { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client';
+import { RiskEngineDataClient } from './lib/entity_analytics/risk_engine/risk_engine_data_client';
+import { RiskScoreDataClient } from './lib/entity_analytics/risk_score/risk_score_data_client';
+import { buildMlAuthz } from './lib/machine_learning/authz';
+import type { ProductFeaturesService } from './lib/product_features_service';
+import type { SiemMigrationsService } from './lib/siem_migrations/siem_migrations_service';
 import { buildFrameworkRequest } from './lib/timeline/utils/common';
 import type {
   SecuritySolutionPluginCoreSetupDependencies,
@@ -25,18 +37,6 @@ import type {
   SecuritySolutionApiRequestHandlerContext,
   SecuritySolutionRequestHandlerContext,
 } from './types';
-import type { Immutable } from '../common/endpoint/types';
-import type { EndpointAuthz } from '../common/endpoint/types/authz';
-import type { EndpointAppContextService } from './endpoint/endpoint_app_context_services';
-import { RiskEngineDataClient } from './lib/entity_analytics/risk_engine/risk_engine_data_client';
-import { RiskScoreDataClient } from './lib/entity_analytics/risk_score/risk_score_data_client';
-import { AssetCriticalityDataClient } from './lib/entity_analytics/asset_criticality';
-import { createDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client';
-import { buildMlAuthz } from './lib/machine_learning/authz';
-import { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client';
-import type { SiemMigrationsService } from './lib/siem_migrations/siem_migrations_service';
-import { AssetInventoryDataClient } from './lib/asset_inventory/asset_inventory_data_client';
-import type { ProductFeaturesService } from './lib/product_features_service';
 
 export interface IRequestContextFactory {
   create(
@@ -190,6 +190,7 @@ export class RequestContextFactory implements IRequestContextFactory {
             actionsClient,
             savedObjectsClient: coreContext.savedObjects.client,
             packageService: startPlugins.fleet?.packageService,
+            telemetry: core.analytics,
           },
         })
       ),