github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[SIEM Migrations] Add missing fields to rule migrations results (#206833)

Browse source 
## Summary

Include all data from the migration process in the translated rule
documents, so we are able to display the correct information in the
table, allowing us also to sort and filter by these fields.

The fields added are: 
- `integration_ids` -> new field mapped in the index (from
`integration_id`), the field is set when we match a prebuilt rule too.
- `risk_score` -> new field mapped in the index, the field is set when
we match a prebuilt rule and set the default value otherwise.
- `severity` -> the field is set when we match a prebuilt rule too.
Defaults moved from the UI to the LLM graph result.

Next steps:

- Take the `risk_score` from the original rule for the custom translated
rules
- Infer `severity` from the original rule risk_score (and maybe other
parameters) for the custom translated rules

Other changes

- The RuleMigrationSevice has been refactored to take all dependencies
(clients, services) from the API context factory. This change makes all
dependencies always available within the Rule migration service so we
don't need to pass them by parameters in each single operation.

- The Prebuilt rule retriever now stores all the prebuilt rules data in
memory during the migration, so we can return all the prebuilt rule
information when we execute semantic searches. This was necessary to set
`rule_id`, `integration_ids`, `severity`, and `risk_score` fields
correctly.

## Screenshots


![screenshot](https://github.com/user-attachments/assets/ee85879e-9d37-498c-9803-0fd3850c3cc5)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Sergi Massaneda 2025-01-17 12:12:01 +01:00 • committed by GitHub
parent a04274723e
commit 7f1e24e343
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 168 additions and 147 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/solutions/security/plugins/security_solution/common/siem_migrations/constants.ts
View file

@ -60,8 +60,6 @@ export const DEFAULT_TRANSLATION_RISK_SCORE = 21;
export const DEFAULT_TRANSLATION_SEVERITY: Severity = 'low';
export const DEFAULT_TRANSLATION_FIELDS = {
risk_score: DEFAULT_TRANSLATION_RISK_SCORE,
severity: DEFAULT_TRANSLATION_SEVERITY,
from: 'now-360s',
to: 'now',
interval: '5m',

8
x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts
View file

@ -90,6 +90,10 @@ export const ElasticRule = z.object({
* The migrated rule severity.
*/
severity: z.string().optional(),
/**
* The migrated rule risk_score value, integer between 0 and 100.
*/
risk_score: z.number().optional(),
/**
* The translated elastic query.
*/
@ -103,9 +107,9 @@ export const ElasticRule = z.object({
*/
prebuilt_rule_id: NonEmptyString.optional(),
/**
* The Elastic integration ID found to be most relevant to the splunk rule.
* The IDs of the Elastic integrations suggested to be installed for this rule.
*/
integration_id: z.string().optional(),
integration_ids: z.array(z.string()).optional(),
/**
* The Elastic rule id installed as a result.
*/

11
x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml
View file

@ -72,6 +72,9 @@ components:
severity:
type: string
description: The migrated rule severity.
risk_score:
type: number
description: The migrated rule risk_score value, integer between 0 and 100.
query:
type: string
description: The translated elastic query.
@ -83,9 +86,11 @@ components:
prebuilt_rule_id:
description: The Elastic prebuilt rule id matched.
$ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString'
integration_id:
type: string
description: The Elastic integration ID found to be most relevant to the splunk rule.
integration_ids:
type: array
description: The IDs of the Elastic integrations suggested to be installed for this rule.
items:
type: string
id:
description: The Elastic rule id installed as a result.
$ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString'

20
x-pack/solutions/security/plugins/security_solution/common/siem_migrations/rules/utils.ts
View file

@ -6,14 +6,24 @@
*/
import type { Severity } from '../../api/detection_engine';
import { DEFAULT_TRANSLATION_FIELDS, DEFAULT_TRANSLATION_SEVERITY } from '../constants';
import { DEFAULT_TRANSLATION_FIELDS } from '../constants';
import type { ElasticRule, ElasticRulePartial } from '../model/rule_migration.gen';
export type MigrationPrebuiltRule = ElasticRulePartial &
Required<Pick<ElasticRulePartial, 'title' | 'description' | 'prebuilt_rule_id'>>;
Required<
Pick<
ElasticRulePartial,
'title' | 'description' | 'prebuilt_rule_id' | 'severity' | 'risk_score'
>
>;
export type MigrationCustomRule = ElasticRulePartial &
Required<Pick<ElasticRulePartial, 'title' | 'description' | 'query' | 'query_language'>>;
Required<
Pick<
ElasticRulePartial,
'title' | 'description' | 'query' | 'query_language' | 'severity' | 'risk_score'
>
>;
export const isMigrationPrebuiltRule = (rule?: ElasticRule): rule is MigrationPrebuiltRule =>
!!(rule?.title && rule?.description && rule?.prebuilt_rule_id);
@ -33,8 +43,8 @@ export const convertMigrationCustomRuleToSecurityRulePayload = (
name: rule.title,
description: rule.description,
enabled,
severity: rule.severity as Severity,
risk_score: rule.risk_score,
...DEFAULT_TRANSLATION_FIELDS,
severity: (rule.severity as Severity) ?? DEFAULT_TRANSLATION_SEVERITY,
};
};

24
x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx
View file

@ -262,24 +262,18 @@ export const MigrationRulesTable: React.FC<MigrationRulesTableProps> = React.mem
if (!isLoading && ruleMigrations.length) {
const ruleMigration = ruleMigrations.find((item) => item.id === ruleId);
let matchedPrebuiltRule: RuleResponse | undefined;
const relatedIntegrations: RelatedIntegration[] = [];
let relatedIntegrations: RelatedIntegration[] = [];
if (ruleMigration) {
// Find matched prebuilt rule if any and prioritize its installed version
const matchedPrebuiltRuleVersion = ruleMigration.elastic_rule?.prebuilt_rule_id
? prebuiltRules[ruleMigration.elastic_rule.prebuilt_rule_id]
: undefined;
matchedPrebuiltRule =
matchedPrebuiltRuleVersion?.current ?? matchedPrebuiltRuleVersion?.target;
const prebuiltRuleId = ruleMigration.elastic_rule?.prebuilt_rule_id;
const prebuiltRuleVersions = prebuiltRuleId ? prebuiltRules[prebuiltRuleId] : undefined;
matchedPrebuiltRule = prebuiltRuleVersions?.current ?? prebuiltRuleVersions?.target;
if (integrations) {
if (matchedPrebuiltRule?.related_integrations) {
relatedIntegrations.push(...matchedPrebuiltRule.related_integrations);
} else if (ruleMigration.elastic_rule?.integration_id) {
const integration = integrations[ruleMigration.elastic_rule.integration_id];
if (integration) {
relatedIntegrations.push(integration);
}
}
const integrationIds = ruleMigration.elastic_rule?.integration_ids;
if (integrations && integrationIds) {
relatedIntegrations = integrationIds
.map((integrationId) => integrations[integrationId])
.filter((integration) => integration != null);
}
}
return { ruleMigration, matchedPrebuiltRule, relatedIntegrations, isIntegrationsLoading };

2
x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/integrations.tsx
View file

@ -21,7 +21,7 @@ export const createIntegrationsColumn = ({
) => { relatedIntegrations?: RelatedIntegration[]; isIntegrationsLoading?: boolean } | undefined;
}): TableColumn => {
return {
field: 'elastic_rule.integration_id',
field: 'elastic_rule.integration_ids',
name: i18n.COLUMN_INTEGRATIONS,
render: (_, rule: RuleMigration) => {
const migrationRuleData = getMigrationRuleData(rule.id);

13
x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/risk_score.tsx
View file

@ -8,22 +8,17 @@
import React from 'react';
import { EuiText } from '@elastic/eui';
import { type RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen';
import {
DEFAULT_TRANSLATION_RISK_SCORE,
SiemMigrationStatus,
} from '../../../../../common/siem_migrations/constants';
import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants';
import * as i18n from './translations';
import { COLUMN_EMPTY_VALUE, type TableColumn } from './constants';
export const createRiskScoreColumn = (): TableColumn => {
return {
field: 'risk_score',
field: 'elastic_rule.risk_score',
name: i18n.COLUMN_RISK_SCORE,
render: (_, rule: RuleMigration) => (
render: (riskScore, rule: RuleMigration) => (
<EuiText data-test-subj="riskScore" size="s">
{rule.status === SiemMigrationStatus.FAILED
? COLUMN_EMPTY_VALUE
: DEFAULT_TRANSLATION_RISK_SCORE}
{rule.status === SiemMigrationStatus.FAILED ? COLUMN_EMPTY_VALUE : riskScore}
</EuiText>
),
sortable: true,

7
x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/rules_table_columns/severity.tsx
View file

@ -8,10 +8,7 @@
import React from 'react';
import type { Severity } from '@kbn/securitysolution-io-ts-alerting-types';
import { type RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen';
import {
DEFAULT_TRANSLATION_SEVERITY,
SiemMigrationStatus,
} from '../../../../../common/siem_migrations/constants';
import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants';
import { SeverityBadge } from '../../../../common/components/severity_badge';
import { COLUMN_EMPTY_VALUE, type TableColumn } from './constants';
import * as i18n from './translations';
@ -20,7 +17,7 @@ export const createSeverityColumn = (): TableColumn => {
return {
field: 'elastic_rule.severity',
name: i18n.COLUMN_SEVERITY,
render: (value: Severity = DEFAULT_TRANSLATION_SEVERITY, rule: RuleMigration) =>
render: (value: Severity, rule: RuleMigration) =>
rule.status === SiemMigrationStatus.FAILED ? (
<>{COLUMN_EMPTY_VALUE}</>
) : (

8
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts
View file

@ -52,10 +52,6 @@ export const registerSiemRuleMigrationsStartRoute = (
const ctx = await context.resolve(['core', 'actions', 'alerting', 'securitySolution']);
const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient();
const inferenceClient = ctx.securitySolution.getInferenceClient();
const actionsClient = ctx.actions.getActionsClient();
const soClient = ctx.core.savedObjects.client;
const rulesClient = await ctx.alerting.getRulesClient();
if (retry) {
const { updated } = await ruleMigrationsClient.task.updateToRetry(
@ -78,10 +74,6 @@ export const registerSiemRuleMigrationsStartRoute = (
migrationId,
connectorId,
invocationConfig,
inferenceClient,
actionsClient,
soClient,
rulesClient,
});
if (!exists) {

5
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts
View file

@ -18,7 +18,7 @@ import type {
Logger,
} from '@kbn/core/server';
import assert from 'assert';
import type { Stored } from '../types';
import type { Stored, SiemRuleMigrationsClientDependencies } from '../types';
import type { IndexNameProvider } from './rule_migrations_data_client';
const DEFAULT_PIT_KEEP_ALIVE: Duration = '30s' as const;
@ -30,7 +30,8 @@ export class RuleMigrationsDataBaseClient {
protected getIndexName: IndexNameProvider,
protected currentUser: AuthenticatedUser,
protected esScopedClient: IScopedClusterClient,
protected logger: Logger
protected logger: Logger,
protected dependencies: SiemRuleMigrationsClientDependencies
) {
this.esClient = esScopedClient.asInternalUser;
}

15
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts
View file

@ -6,12 +6,12 @@
*/
import type { AuthenticatedUser, IScopedClusterClient, Logger } from '@kbn/core/server';
import type { PackageService } from '@kbn/fleet-plugin/server';
import { RuleMigrationsDataIntegrationsClient } from './rule_migrations_data_integrations_client';
import { RuleMigrationsDataPrebuiltRulesClient } from './rule_migrations_data_prebuilt_rules_client';
import { RuleMigrationsDataResourcesClient } from './rule_migrations_data_resources_client';
import { RuleMigrationsDataRulesClient } from './rule_migrations_data_rules_client';
import { RuleMigrationsDataLookupsClient } from './rule_migrations_data_lookups_client';
import type { SiemRuleMigrationsClientDependencies } from '../types';
import type { AdapterId } from './rule_migrations_data_service';
export type IndexNameProvider = () => Promise<string>;
@ -29,32 +29,35 @@ export class RuleMigrationsDataClient {
currentUser: AuthenticatedUser,
esScopedClient: IScopedClusterClient,
logger: Logger,
packageService?: PackageService
dependencies: SiemRuleMigrationsClientDependencies
) {
this.rules = new RuleMigrationsDataRulesClient(
indexNameProviders.rules,
currentUser,
esScopedClient,
logger
logger,
dependencies
);
this.resources = new RuleMigrationsDataResourcesClient(
indexNameProviders.resources,
currentUser,
esScopedClient,
logger
logger,
dependencies
);
this.integrations = new RuleMigrationsDataIntegrationsClient(
indexNameProviders.integrations,
currentUser,
esScopedClient,
logger,
packageService
dependencies
);
this.prebuiltRules = new RuleMigrationsDataPrebuiltRulesClient(
indexNameProviders.prebuiltrules,
currentUser,
esScopedClient,
logger
logger,
dependencies
);
this.lookups = new RuleMigrationsDataLookupsClient(currentUser, esScopedClient, logger);
}

15
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_integrations_client.ts
View file

@ -5,15 +5,12 @@
* 2.0.
*/
import type { PackageService } from '@kbn/fleet-plugin/server';
import type { AuthenticatedUser, IScopedClusterClient, Logger } from '@kbn/core/server';
import type { PackageList } from '@kbn/fleet-plugin/common';
import type { RuleMigrationIntegration } from '../types';
import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client';
/* This will be removed once the package registry changes is performed */
import integrationsFile from './integrations_temp.json';
import type { IndexNameProvider } from './rule_migrations_data_client';
/* The minimum score required for a integration to be considered correct, might need to change this later */
const MIN_SCORE = 40 as const;
@ -26,18 +23,8 @@ const INTEGRATIONS = integrationsFile as RuleMigrationIntegration[];
* The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed.
*/
export class RuleMigrationsDataIntegrationsClient extends RuleMigrationsDataBaseClient {
constructor(
getIndexName: IndexNameProvider,
currentUser: AuthenticatedUser,
esScopedClient: IScopedClusterClient,
logger: Logger,
private packageService?: PackageService
) {
super(getIndexName, currentUser, esScopedClient, logger);
}
async getIntegrationPackages(): Promise<PackageList | undefined> {
return this.packageService?.asInternalUser.getPackages();
return this.dependencies.packageService?.asInternalUser.getPackages();
}
/** Indexes an array of integrations to be used with ELSER semantic search queries */

35
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_prebuilt_rules_client.ts
View file

@ -5,19 +5,15 @@
* 2.0.
*/
import type { RulesClient } from '@kbn/alerting-plugin/server';
import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
import type { RuleVersions } from '../../../detection_engine/prebuilt_rules/logic/diff/calculate_rule_diff';
import { createPrebuiltRuleAssetsClient } from '../../../detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client';
import { createPrebuiltRuleObjectsClient } from '../../../detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client';
import { fetchRuleVersionsTriad } from '../../../detection_engine/prebuilt_rules/logic/rule_versions/fetch_rule_versions_triad';
import type { RuleMigrationPrebuiltRule } from '../types';
import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client';
interface RetrievePrebuiltRulesParams {
soClient: SavedObjectsClientContract;
rulesClient: RulesClient;
}
export type { RuleVersions };
export type PrebuildRuleVersionsMap = Map<string, RuleVersions>;
/* The minimum score required for a integration to be considered correct, might need to change this later */
const MIN_SCORE = 40 as const;
/* The number of integrations the RAG will return, sorted by score */
@ -29,17 +25,16 @@ const RETURNED_RULES = 5 as const;
const BULK_MAX_SIZE = 500 as const;
export class RuleMigrationsDataPrebuiltRulesClient extends RuleMigrationsDataBaseClient {
async getRuleVersionsMap(): Promise<PrebuildRuleVersionsMap> {
const ruleAssetsClient = createPrebuiltRuleAssetsClient(this.dependencies.savedObjectsClient);
const ruleObjectsClient = createPrebuiltRuleObjectsClient(this.dependencies.rulesClient);
return fetchRuleVersionsTriad({ ruleAssetsClient, ruleObjectsClient });
}
/** Indexes an array of integrations to be used with ELSER semantic search queries */
async create({ soClient, rulesClient }: RetrievePrebuiltRulesParams): Promise<void> {
const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient);
const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient);
const ruleVersionsMap = await fetchRuleVersionsTriad({
ruleAssetsClient,
ruleObjectsClient,
});
async populate(ruleVersionsMap: PrebuildRuleVersionsMap): Promise<void> {
const filteredRules: RuleMigrationPrebuiltRule[] = [];
ruleVersionsMap.forEach((ruleVersions) => {
const rule = ruleVersions.target || ruleVersions.current;
if (rule) {
@ -50,7 +45,6 @@ export class RuleMigrationsDataPrebuiltRulesClient extends RuleMigrationsDataBas
filteredRules.push({
rule_id: rule.rule_id,
name: rule.name,
installed_rule_id: ruleVersions.current?.id,
description: rule.description,
elser_embedding: `${rule.name} - ${rule.description}`,
...(mitreAttackIds?.length && { mitre_attack_ids: mitreAttackIds }),
@ -87,10 +81,7 @@ export class RuleMigrationsDataPrebuiltRulesClient extends RuleMigrationsDataBas
}
/** Based on a LLM generated semantic string, returns the 5 best results with a score above 40 */
async retrieveRules(
semanticString: string,
techniqueIds: string
): Promise<RuleMigrationPrebuiltRule[]> {
async search(semanticString: string, techniqueIds: string): Promise<RuleMigrationPrebuiltRule[]> {
const index = await this.getIndexName();
const query = {
bool: {
@ -126,7 +117,7 @@ export class RuleMigrationsDataPrebuiltRulesClient extends RuleMigrationsDataBas
size: RETURNED_RULES,
min_score: MIN_SCORE,
})
.then(this.processResponseHits.bind(this))
.then((response) => this.processResponseHits(response))
.catch((error) => {
this.logger.error(`Error querying prebuilt rule details for ELSER: ${error.message}`);
throw error;

3
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts
View file

@ -12,6 +12,7 @@ import type { InstallParams } from '@kbn/index-adapter';
import { IndexPatternAdapter, IndexAdapter } from '@kbn/index-adapter';
import { loggerMock } from '@kbn/logging-mocks';
import { Subject } from 'rxjs';
import type { SiemRuleMigrationsClientDependencies } from '../types';
import type { IndexNameProviders } from './rule_migrations_data_client';
import { INDEX_PATTERN, RuleMigrationsDataService } from './rule_migrations_data_service';
@ -30,6 +31,7 @@ const MockedIndexPatternAdapter = IndexPatternAdapter as unknown as jest.MockedC
>;
const MockedIndexAdapter = IndexAdapter as unknown as jest.MockedClass<typeof IndexAdapter>;
const dependencies = {} as SiemRuleMigrationsClientDependencies;
const esClient = elasticsearchServiceMock.createStart().client.asInternalUser;
describe('SiemRuleMigrationsDataService', () => {
@ -106,6 +108,7 @@ describe('SiemRuleMigrationsDataService', () => {
spaceId: 'space1',
currentUser,
esScopedClient: elasticsearchServiceMock.createStart().client.asScoped(),
dependencies,
};
it('should install space index pattern', async () => {

13
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts
View file

@ -11,9 +11,9 @@ import {
type FieldMap,
type InstallParams,
} from '@kbn/index-adapter';
import type { PackageService } from '@kbn/fleet-plugin/server';
import type { IndexNameProvider, IndexNameProviders } from './rule_migrations_data_client';
import { RuleMigrationsDataClient } from './rule_migrations_data_client';
import type { SiemRuleMigrationsClientDependencies } from '../types';
import {
integrationsFieldMap,
prebuiltRulesFieldMap,
@ -37,7 +37,7 @@ interface CreateClientParams {
spaceId: string;
currentUser: AuthenticatedUser;
esScopedClient: IScopedClusterClient;
packageService?: PackageService;
dependencies: SiemRuleMigrationsClientDependencies;
}
interface CreateAdapterParams {
adapterId: AdapterId;
@ -103,12 +103,7 @@ export class RuleMigrationsDataService {
]);
}
public createClient({
spaceId,
currentUser,
esScopedClient,
packageService,
}: CreateClientParams) {
public createClient({ spaceId, currentUser, esScopedClient, dependencies }: CreateClientParams) {
const indexNameProviders: IndexNameProviders = {
rules: this.createIndexNameProvider(this.adapters.rules, spaceId),
resources: this.createIndexNameProvider(this.adapters.resources, spaceId),
@ -121,7 +116,7 @@ export class RuleMigrationsDataService {
currentUser,
esScopedClient,
this.logger,
packageService
dependencies
);
}

4
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts
View file

@ -28,10 +28,11 @@ export const ruleMigrationsFieldMap: FieldMap<SchemaFieldMapKeys<Omit<RuleMigrat
'original_rule.annotations.mitre_attack': { type: 'keyword', array: true, required: false },
elastic_rule: { type: 'nested', required: false },
'elastic_rule.title': { type: 'text', required: true, fields: { keyword: { type: 'keyword' } } },
'elastic_rule.integration_id': { type: 'keyword', required: false },
'elastic_rule.integration_ids': { type: 'keyword', required: false, array: true },
'elastic_rule.query': { type: 'text', required: true },
'elastic_rule.query_language': { type: 'keyword', required: true },
'elastic_rule.description': { type: 'text', required: false },
'elastic_rule.risk_score': { type: 'short', required: false },
'elastic_rule.severity': { type: 'keyword', required: false },
'elastic_rule.prebuilt_rule_id': { type: 'keyword', required: false },
'elastic_rule.id': { type: 'keyword', required: false },
@ -69,6 +70,5 @@ export const prebuiltRulesFieldMap: FieldMap<SchemaFieldMapKeys<RuleMigrationPre
description: { type: 'text', required: true },
elser_embedding: { type: 'semantic_text', required: true },
rule_id: { type: 'keyword', required: true },
installed_rule_id: { type: 'keyword', required: true },
mitre_attack_ids: { type: 'keyword', array: true, required: false },
};

6
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts
View file

@ -22,10 +22,13 @@ import {
} from './data/__mocks__/mocks';
import { mockCreateClient as mockTaskCreateClient, mockStopAll } from './task/__mocks__/mocks';
import { waitFor } from '@testing-library/dom';
import type { SiemRuleMigrationsClientDependencies } from './types';
jest.mock('./data/rule_migrations_data_service');
jest.mock('./task/rule_migrations_task_service');
const dependencies = {} as SiemRuleMigrationsClientDependencies;
describe('SiemRuleMigrationsService', () => {
let ruleMigrationsService: SiemRuleMigrationsService;
const kibanaVersion = '8.16.0';
@ -74,6 +77,7 @@ describe('SiemRuleMigrationsService', () => {
spaceId: 'default',
currentUser,
request: httpServerMock.createKibanaRequest(),
dependencies,
};
});
@ -96,6 +100,7 @@ describe('SiemRuleMigrationsService', () => {
spaceId: createClientParams.spaceId,
currentUser: createClientParams.currentUser,
esScopedClient: esClusterClient.asScoped(),
dependencies,
});
});
@ -104,6 +109,7 @@ describe('SiemRuleMigrationsService', () => {
expect(mockTaskCreateClient).toHaveBeenCalledWith({
currentUser: createClientParams.currentUser,
dataClient: mockDataCreateClient(),
dependencies,
});
});

14
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts
View file

@ -14,11 +14,11 @@ import type {
KibanaRequest,
Logger,
} from '@kbn/core/server';
import type { PackageService } from '@kbn/fleet-plugin/server';
import { RuleMigrationsDataService } from './data/rule_migrations_data_service';
import type { RuleMigrationsDataClient } from './data/rule_migrations_data_client';
import type { RuleMigrationsTaskClient } from './task/rule_migrations_task_client';
import { RuleMigrationsTaskService } from './task/rule_migrations_task_service';
import type { SiemRuleMigrationsClientDependencies } from './types';
export interface SiemRulesMigrationsSetupParams {
esClusterClient: IClusterClient;
@ -30,7 +30,7 @@ export interface SiemRuleMigrationsCreateClientParams {
request: KibanaRequest;
currentUser: AuthenticatedUser | null;
spaceId: string;
packageService?: PackageService;
dependencies: SiemRuleMigrationsClientDependencies;
}
export interface SiemRuleMigrationsClient {
@ -60,10 +60,10 @@ export class SiemRuleMigrationsService {
}
createClient({
spaceId,
currentUser,
packageService,
request,
currentUser,
spaceId,
dependencies,
}: SiemRuleMigrationsCreateClientParams): SiemRuleMigrationsClient {
assert(currentUser, 'Current user must be authenticated');
assert(this.esClusterClient, 'ES client not available, please call setup first');
@ -73,9 +73,9 @@ export class SiemRuleMigrationsService {
spaceId,
currentUser,
esScopedClient,
packageService,
dependencies,
});
const taskClient = this.taskService.createClient({ currentUser, dataClient });
const taskClient = this.taskService.createClient({ currentUser, dataClient, dependencies });
return { data: dataClient, task: taskClient };
}

13
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts
View file

@ -7,7 +7,11 @@
import type { Logger } from '@kbn/core/server';
import { JsonOutputParser } from '@langchain/core/output_parsers';
import { RuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants';
import {
DEFAULT_TRANSLATION_RISK_SCORE,
DEFAULT_TRANSLATION_SEVERITY,
RuleTranslationResult,
} from '../../../../../../../../common/siem_migrations/constants';
import type { RuleMigrationsRetriever } from '../../../retrievers';
import type { ChatModel } from '../../../util/actions_client_chat';
import type { GraphNode } from '../../types';
@ -33,7 +37,7 @@ export const getMatchPrebuiltRuleNode = ({
return async (state) => {
const query = state.semantic_query;
const techniqueIds = state.original_rule.annotations?.mitre_attack || [];
const prebuiltRules = await ruleMigrationsRetriever.prebuiltRules.getRules(
const prebuiltRules = await ruleMigrationsRetriever.prebuiltRules.search(
query,
techniqueIds.join(',')
);
@ -74,8 +78,11 @@ export const getMatchPrebuiltRuleNode = ({
elastic_rule: {
title: matchedRule.name,
description: matchedRule.description,
id: matchedRule.installed_rule_id,
prebuilt_rule_id: matchedRule.rule_id,
id: matchedRule.current?.id,
integration_ids: matchedRule.target?.related_integrations?.map((i) => i.package),
severity: matchedRule.target?.severity ?? DEFAULT_TRANSLATION_SEVERITY,
risk_score: matchedRule.target?.risk_score ?? DEFAULT_TRANSLATION_RISK_SCORE,
},
translation_result: RuleTranslationResult.FULL,
};

2
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts
View file

@ -49,7 +49,7 @@ export const getTranslateRuleNode = ({
response,
comments: [cleanMarkdown(translationSummary)],
elastic_rule: {
integration_id: integrationId,
integration_ids: [integrationId],
query: esqlQuery,
query_language: 'esql',
},

9
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts
View file

@ -5,7 +5,11 @@
* 2.0.
*/
import { RuleTranslationResult } from '../../../../../../../../../../common/siem_migrations/constants';
import {
DEFAULT_TRANSLATION_RISK_SCORE,
DEFAULT_TRANSLATION_SEVERITY,
RuleTranslationResult,
} from '../../../../../../../../../../common/siem_migrations/constants';
import type { GraphNode } from '../../types';
export const translationResultNode: GraphNode = async (state) => {
@ -13,7 +17,8 @@ export const translationResultNode: GraphNode = async (state) => {
const elasticRule = {
title: state.original_rule.title,
description: state.original_rule.description || state.original_rule.title,
severity: 'low',
severity: DEFAULT_TRANSLATION_SEVERITY,
risk_score: DEFAULT_TRANSLATION_RISK_SCORE,
...state.elastic_rule,
};

30
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/retrievers/prebuilt_rules_retriever.ts
View file

@ -5,27 +5,33 @@
* 2.0.
*/
import type { RuleMigrationPrebuiltRule } from '../../types';
import type { PrebuildRuleVersionsMap } from '../../data/rule_migrations_data_prebuilt_rules_client';
import type { RuleSemanticSearchResult } from '../../types';
import type { RuleMigrationsRetrieverClients } from './rule_migrations_retriever';
export class PrebuiltRulesRetriever {
private rulesMap?: PrebuildRuleVersionsMap;
constructor(private readonly clients: RuleMigrationsRetrieverClients) {}
// TODO:
// 1. Implement the `initialize` method to retrieve prebuilt rules and keep them in memory.
// 2. Improve the `retrieveRules` method to return the real prebuilt rules instead of the ELSER index doc.
public async populateIndex() {
return this.clients.data.prebuiltRules.create({
rulesClient: this.clients.rules,
soClient: this.clients.savedObjects,
});
if (!this.rulesMap) {
this.rulesMap = await this.clients.data.prebuiltRules.getRuleVersionsMap();
}
return this.clients.data.prebuiltRules.populate(this.rulesMap);
}
public async getRules(
public async search(
semanticString: string,
techniqueIds: string
): Promise<RuleMigrationPrebuiltRule[]> {
return this.clients.data.prebuiltRules.retrieveRules(semanticString, techniqueIds);
): Promise<RuleSemanticSearchResult[]> {
if (!this.rulesMap) {
this.rulesMap = await this.clients.data.prebuiltRules.getRuleVersionsMap();
}
const results = await this.clients.data.prebuiltRules.search(semanticString, techniqueIds);
return results.map((rule) => {
const versions = this.rulesMap?.get(rule.rule_id) ?? {};
return { ...rule, ...versions };
});
}
}

12
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts
View file

@ -18,6 +18,7 @@ import type {
RuleMigrationDataStats,
RuleMigrationFilters,
} from '../data/rule_migrations_data_rules_client';
import type { SiemRuleMigrationsClientDependencies } from '../types';
import { getRuleMigrationAgent } from './agent';
import type { MigrateRuleState } from './agent/types';
import { RuleMigrationsRetriever } from './retrievers';
@ -40,7 +41,8 @@ export class RuleMigrationsTaskClient {
private migrationsRunning: MigrationsRunning,
private logger: Logger,
private data: RuleMigrationsDataClient,
private currentUser: AuthenticatedUser
private currentUser: AuthenticatedUser,
private dependencies: SiemRuleMigrationsClientDependencies
) {}
/** Starts a rule migration task */
@ -180,12 +182,10 @@ export class RuleMigrationsTaskClient {
private async createAgent({
migrationId,
connectorId,
inferenceClient,
actionsClient,
rulesClient,
soClient,
abortController,
}: RuleMigrationTaskCreateAgentParams): Promise<MigrationAgent> {
const { inferenceClient, actionsClient, rulesClient, savedObjectsClient } = this.dependencies;
const actionsClientChat = new ActionsClientChat(connectorId, actionsClient, this.logger);
const model = await actionsClientChat.createModel({
signal: abortController.signal,
@ -195,7 +195,7 @@ export class RuleMigrationsTaskClient {
const ruleMigrationsRetriever = new RuleMigrationsRetriever(migrationId, {
data: this.data,
rules: rulesClient,
savedObjects: soClient,
savedObjects: savedObjectsClient,
});
await ruleMigrationsRetriever.initialize();

4
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_service.ts
View file

@ -21,12 +21,14 @@ export class RuleMigrationsTaskService {
public createClient({
currentUser,
dataClient,
dependencies,
}: RuleMigrationTaskCreateClientParams): RuleMigrationsTaskClient {
return new RuleMigrationsTaskClient(
this.migrationsRunning,
this.logger,
dataClient,
currentUser
currentUser,
dependencies
);
}

11
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts
View file

@ -5,12 +5,10 @@
* 2.0.
*/
import type { AuthenticatedUser, SavedObjectsClientContract } from '@kbn/core/server';
import type { AuthenticatedUser } from '@kbn/core/server';
import type { RunnableConfig } from '@langchain/core/runnables';
import type { InferenceClient } from '@kbn/inference-plugin/server';
import type { ActionsClient } from '@kbn/actions-plugin/server';
import type { RulesClient } from '@kbn/alerting-plugin/server';
import type { RuleMigrationsDataClient } from '../data/rule_migrations_data_client';
import type { SiemRuleMigrationsClientDependencies } from '../types';
import type { getRuleMigrationAgent } from './agent';
export type MigrationAgent = ReturnType<typeof getRuleMigrationAgent>;
@ -18,16 +16,13 @@ export type MigrationAgent = ReturnType<typeof getRuleMigrationAgent>;
export interface RuleMigrationTaskCreateClientParams {
currentUser: AuthenticatedUser;
dataClient: RuleMigrationsDataClient;
dependencies: SiemRuleMigrationsClientDependencies;
}
export interface RuleMigrationTaskStartParams {
migrationId: string;
connectorId: string;
invocationConfig: RunnableConfig;
inferenceClient: InferenceClient;
actionsClient: ActionsClient;
rulesClient: RulesClient;
soClient: SavedObjectsClientContract;
}
export interface RuleMigrationTaskCreateAgentParams extends RuleMigrationTaskStartParams {

17
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/types.ts
View file

@ -5,6 +5,11 @@
* 2.0.
*/
import type { SavedObjectsClientContract } from '@kbn/core/server';
import type { PackageService } from '@kbn/fleet-plugin/server';
import type { RulesClient } from '@kbn/alerting-plugin/server';
import type { ActionsClient } from '@kbn/actions-plugin/server';
import type { InferenceClient } from '@kbn/inference-plugin/server';
import type {
UpdateRuleMigrationData,
RuleMigrationTranslationResult,
@ -13,12 +18,21 @@ import {
type RuleMigration,
type RuleMigrationResource,
} from '../../../../common/siem_migrations/model/rule_migration.gen';
import type { RuleVersions } from './data/rule_migrations_data_prebuilt_rules_client';
export type Stored<T extends object> = T & { id: string };
export type StoredRuleMigration = Stored<RuleMigration>;
export type StoredRuleMigrationResource = Stored<RuleMigrationResource>;
export interface SiemRuleMigrationsClientDependencies {
inferenceClient: InferenceClient;
rulesClient: RulesClient;
actionsClient: ActionsClient;
savedObjectsClient: SavedObjectsClientContract;
packageService?: PackageService;
}
export interface RuleMigrationIntegration {
id: string;
title: string;
@ -29,13 +43,14 @@ export interface RuleMigrationIntegration {
export interface RuleMigrationPrebuiltRule {
rule_id: string;
installed_rule_id?: string;
name: string;
description: string;
elser_embedding: string;
mitre_attack_ids?: string[];
}
export type RuleSemanticSearchResult = RuleMigrationPrebuiltRule & RuleVersions;
export type InternalUpdateRuleMigrationData = UpdateRuleMigrationData & {
translation_result?: RuleMigrationTranslationResult;
};

4
x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts
View file

@ -18,6 +18,7 @@ import {
mockStop,
} from './rules/__mocks__/mocks';
import type { ConfigType } from '../../config';
import type { SiemRuleMigrationsClientDependencies } from './rules/types';
jest.mock('./rules/siem_rule_migrations_service');
@ -27,6 +28,8 @@ jest.mock('rxjs', () => ({
ReplaySubject: jest.fn().mockImplementation(() => mockReplaySubject$),
}));
const dependencies = {} as SiemRuleMigrationsClientDependencies;
describe('SiemMigrationsService', () => {
let siemMigrationsService: SiemMigrationsService;
const kibanaVersion = '8.16.0';
@ -70,6 +73,7 @@ describe('SiemMigrationsService', () => {
spaceId: 'default',
request: httpServerMock.createKibanaRequest(),
currentUser,
dependencies,
};
siemMigrationsService.createRulesClient(createRulesClientParams);
expect(mockCreateClient).toHaveBeenCalledWith(createRulesClientParams);

8
x-pack/solutions/security/plugins/security_solution/server/request_context_factory.ts
View file

@ -176,7 +176,13 @@ export class RequestContextFactory implements IRequestContextFactory {
request,
currentUser: coreContext.security.authc.getCurrentUser(),
spaceId: getSpaceId(),
packageService: startPlugins.fleet?.packageService,
dependencies: {
inferenceClient: startPlugins.inference.getClient({ request }),
rulesClient,
actionsClient,
savedObjectsClient: coreContext.savedObjects.client,
packageService: startPlugins.fleet?.packageService,
},
})
),