[Siem Migrations] Adds separate migration index to store migration metadata (#216164)

## Summary

Fixes: https://github.com/elastic/security-team/issues/12233


This PR simply adds a new `migration` index to store the migration data
with `migration_id` as the only property for now.

The APIs remain unchanged.

Below are the mapping for new index
`.kibana-siem-rule-migrations-migrations-default` based on the pattern
`..kibana-siem-rule-migrations-<indexAdapterId>-<spaceName>`

```
{
  ".kibana-siem-rule-migrations-migrations-default": {
    "mappings": {
      "dynamic": "false",
      "_meta": {
        "namespace": "default",
        "kibana": {
          "version": "9.1.0"
        },
        "managed": true
      },
      "properties": {
        "created_at": {
          "type": "date"
        },
        "created_by": {
          "type": "keyword"
        },
        "id": {
          "type": "keyword"
        }
      }
    }
  }
}
```


Below is how a sample document looks like:

```json
      {
        "_index": ".kibana-siem-rule-migrations-migrations-default",
        "_id": "C7oi15UBS6DCfB3qd4_l",
        "_score": 1,
        "_source": {
      
          "created_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0",
          "created_at": "2025-03-27T10:25:15.232Z"
        }
      }
```

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Jatin Kathuria 2025-04-04 11:25:12 +02:00 committed by GitHub
parent 12ebf35673
commit 5846a5821c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 264 additions and 44 deletions

View file

@ -7,7 +7,6 @@
import type { IKibanaResponse, Logger } from '@kbn/core/server';
import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
import { v4 as uuidV4 } from 'uuid';
import { SIEM_RULE_MIGRATION_CREATE_PATH } from '../../../../../common/siem_migrations/constants';
import {
CreateRuleMigrationRequestBody,
@ -44,8 +43,8 @@ export const registerSiemRuleMigrationsCreateRoute = (
withLicense(
async (context, req, res): Promise<IKibanaResponse<CreateRuleMigrationResponse>> => {
const originalRules = req.body;
const migrationId = req.params.migration_id ?? uuidV4();
const siemMigrationAuditLogger = new SiemMigrationAuditLogger(context.securitySolution);
const providedMigrationId = req.params?.migration_id;
try {
const [firstOriginalRule] = originalRules;
if (!firstOriginalRule) {
@ -53,8 +52,17 @@ export const registerSiemRuleMigrationsCreateRoute = (
}
const ctx = await context.resolve(['securitySolution']);
const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient();
await siemMigrationAuditLogger.logCreateMigration({ migrationId: providedMigrationId });
await siemMigrationAuditLogger.logCreateMigration({ migrationId });
let migrationId: string;
if (!providedMigrationId) {
/** if new migration */
migrationId = await ruleMigrationsClient.data.migrations.create();
} else {
/** if updating existing migration */
migrationId = providedMigrationId;
}
const ruleMigrations = originalRules.map<CreateRuleMigrationInput>((originalRule) => ({
migration_id: migrationId,
@ -76,7 +84,10 @@ export const registerSiemRuleMigrationsCreateRoute = (
return res.ok({ body: { migration_id: migrationId } });
} catch (error) {
logger.error(error);
await siemMigrationAuditLogger.logCreateMigration({ migrationId, error });
await siemMigrationAuditLogger.logCreateMigration({
migrationId: providedMigrationId,
error,
});
return res.badRequest({ body: error.message });
}
}

View file

@ -106,7 +106,7 @@ export class SiemMigrationAuditLogger {
}
}
public async logCreateMigration(params: { migrationId: string; error?: Error }): Promise<void> {
public async logCreateMigration(params: { migrationId?: string; error?: Error }): Promise<void> {
const { migrationId, error } = params;
const message = `User created a new SIEM migration with [id=${migrationId}]`;
return this.log({

View file

@ -7,6 +7,7 @@
import type { RuleMigrationsDataIntegrationsClient } from '../rule_migrations_data_integrations_client';
import type { RuleMigrationsDataLookupsClient } from '../rule_migrations_data_lookups_client';
import type { RuleMigrationsDataMigrationClient } from '../rule_migrations_data_migration_client';
import type { RuleMigrationsDataPrebuiltRulesClient } from '../rule_migrations_data_prebuilt_rules_client';
import type { RuleMigrationsDataResourcesClient } from '../rule_migrations_data_resources_client';
import type { RuleMigrationsDataRulesClient } from '../rule_migrations_data_rules_client';
@ -57,6 +58,10 @@ export const mockRuleMigrationsDataLookupsClient = {
create: jest.fn().mockResolvedValue(undefined),
indexData: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<RuleMigrationsDataLookupsClient>;
export const mockRuleMigrationsDataMigrationsClient = {
create: jest.fn().mockResolvedValue(undefined),
get: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<RuleMigrationsDataMigrationClient>;
// Rule migrations data client
export const createRuleMigrationsDataClientMock = () => ({
@ -65,6 +70,7 @@ export const createRuleMigrationsDataClientMock = () => ({
integrations: mockRuleMigrationsDataIntegrationsClient,
prebuiltRules: mockRuleMigrationsDataPrebuiltRulesClient,
lookups: mockRuleMigrationsDataLookupsClient,
migrations: mockRuleMigrationsDataMigrationsClient,
});
export const MockRuleMigrationsDataClient = jest

View file

@ -18,8 +18,7 @@ import type {
Logger,
} from '@kbn/core/server';
import assert from 'assert';
import type { Stored, SiemRuleMigrationsClientDependencies } from '../types';
import type { IndexNameProvider } from './rule_migrations_data_client';
import type { IndexNameProvider, SiemRuleMigrationsClientDependencies, Stored } from '../types';
const DEFAULT_PIT_KEEP_ALIVE: Duration = '30s' as const;
@ -53,22 +52,25 @@ export class RuleMigrationsDataBaseClient {
}
}
protected processResponseHits<T extends object>(
response: SearchResponse<T>,
override?: Partial<T>
): Array<Stored<T>> {
return this.processHits(response.hits.hits, override);
protected processHit<T extends object>(hit: SearchHit<T>, override: Partial<T> = {}): Stored<T> {
const { _id, _source } = hit;
assert(_id, 'document should have _id');
assert(_source, 'document should have _source');
return { ..._source, ...override, id: _id };
}
protected processHits<T extends object>(
hits: Array<SearchHit<T>> = [],
override: Partial<T> = {}
): Array<Stored<T>> {
return hits.map(({ _id, _source }) => {
assert(_id, 'document should have _id');
assert(_source, 'document should have _source');
return { ..._source, ...override, id: _id };
});
return hits.map((hit) => this.processHit(hit, override));
}
protected processResponseHits<T extends object>(
response: SearchResponse<T>,
override?: Partial<T>
): Array<Stored<T>> {
return this.processHits(response.hits.hits, override);
}
protected getTotalHits(response: SearchResponse) {

View file

@ -11,13 +11,11 @@ import { RuleMigrationsDataPrebuiltRulesClient } from './rule_migrations_data_pr
import { RuleMigrationsDataResourcesClient } from './rule_migrations_data_resources_client';
import { RuleMigrationsDataRulesClient } from './rule_migrations_data_rules_client';
import { RuleMigrationsDataLookupsClient } from './rule_migrations_data_lookups_client';
import type { SiemRuleMigrationsClientDependencies } from '../types';
import type { AdapterId } from './rule_migrations_data_service';
export type IndexNameProvider = () => Promise<string>;
export type IndexNameProviders = Record<AdapterId, IndexNameProvider>;
import type { IndexNameProviders, SiemRuleMigrationsClientDependencies } from '../types';
import { RuleMigrationsDataMigrationClient } from './rule_migrations_data_migration_client';
export class RuleMigrationsDataClient {
public readonly migrations: RuleMigrationsDataMigrationClient;
public readonly rules: RuleMigrationsDataRulesClient;
public readonly resources: RuleMigrationsDataResourcesClient;
public readonly integrations: RuleMigrationsDataIntegrationsClient;
@ -32,6 +30,13 @@ export class RuleMigrationsDataClient {
spaceId: string,
dependencies: SiemRuleMigrationsClientDependencies
) {
this.migrations = new RuleMigrationsDataMigrationClient(
indexNameProviders.migrations,
currentUser,
esScopedClient,
logger,
dependencies
);
this.rules = new RuleMigrationsDataRulesClient(
indexNameProviders.rules,
currentUser,

View file

@ -0,0 +1,110 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IScopedClusterClient } from '@kbn/core/server';
import type { SiemRuleMigrationsClientDependencies } from '../types';
import { RuleMigrationsDataMigrationClient } from './rule_migrations_data_migration_client';
import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
import type { AuthenticatedUser } from '@kbn/security-plugin-types-common';
import type IndexApi from '@elastic/elasticsearch/lib/api/api';
import type GetApi from '@elastic/elasticsearch/lib/api/api/get';
describe('RuleMigrationsDataMigrationClient', () => {
let ruleMigrationsDataMigrationClient: RuleMigrationsDataMigrationClient;
const esClient =
elasticsearchServiceMock.createCustomClusterClient() as unknown as IScopedClusterClient;
const logger = loggingSystemMock.createLogger();
const indexNameProvider = jest.fn().mockReturnValue('.kibana-siem-rule-migrations');
const currentUser = {
userName: 'testUser',
profile_uid: 'testProfileUid',
} as unknown as AuthenticatedUser;
const dependencies = {} as unknown as SiemRuleMigrationsClientDependencies;
beforeEach(() => {
ruleMigrationsDataMigrationClient = new RuleMigrationsDataMigrationClient(
indexNameProvider,
currentUser,
esClient,
logger,
dependencies
);
});
afterEach(() => {
jest.clearAllMocks();
});
describe('create', () => {
test('should create a new migration', async () => {
const index = '.kibana-siem-rule-migrations';
const result = await ruleMigrationsDataMigrationClient.create();
expect(result).not.toBeFalsy();
expect(esClient.asInternalUser.create).toHaveBeenCalledWith({
refresh: 'wait_for',
id: result,
index,
document: {
created_by: currentUser.profile_uid,
created_at: expect.any(String),
},
});
});
test('should throw an error if an error occurs', async () => {
(
esClient.asInternalUser.create as unknown as jest.MockedFn<typeof IndexApi>
).mockRejectedValueOnce(new Error('Test error'));
await expect(ruleMigrationsDataMigrationClient.create()).rejects.toThrow('Test error');
expect(esClient.asInternalUser.create).toHaveBeenCalled();
expect(logger.error).toHaveBeenCalled();
});
});
describe('get', () => {
test('should get a migration', async () => {
const index = '.kibana-siem-rule-migrations';
const id = 'testId';
const response = {
_index: index,
found: true,
_source: {
created_by: currentUser.profile_uid,
created_at: new Date().toISOString(),
},
_id: id,
};
(
esClient.asInternalUser.get as unknown as jest.MockedFn<typeof GetApi>
).mockResolvedValueOnce(response);
const result = await ruleMigrationsDataMigrationClient.get({ id });
expect(result).toEqual({
...response._source,
id: response._id,
});
});
test('should throw an error if an error occurs', async () => {
const id = 'testId';
(
esClient.asInternalUser.get as unknown as jest.MockedFn<typeof GetApi>
).mockRejectedValueOnce(new Error('Test error'));
await expect(ruleMigrationsDataMigrationClient.get({ id })).rejects.toThrow('Test error');
expect(esClient.asInternalUser.get).toHaveBeenCalled();
expect(logger.error).toHaveBeenCalledWith(`Error getting migration ${id}: Error: Test error`);
});
});
});

View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { v4 as uuidV4 } from 'uuid';
import type { StoredSiemMigration } from '../types';
import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client';
export class RuleMigrationsDataMigrationClient extends RuleMigrationsDataBaseClient {
async create(): Promise<string> {
const migrationId = uuidV4();
const index = await this.getIndexName();
const profileUid = await this.getProfileUid();
const createdAt = new Date().toISOString();
await this.esClient
.create({
refresh: 'wait_for',
id: migrationId,
index,
document: {
created_by: profileUid,
created_at: createdAt,
},
})
.catch((error) => {
this.logger.error(`Error creating migration ${migrationId}: ${error}`);
throw error;
});
return migrationId;
}
async get({ id }: { id: string }): Promise<StoredSiemMigration> {
const index = await this.getIndexName();
return this.esClient
.get<StoredSiemMigration>({
index,
id,
})
.then((document) => {
return this.processHit(document);
})
.catch((error) => {
this.logger.error(`Error getting migration ${id}: ${error}`);
throw error;
});
}
}

View file

@ -21,14 +21,14 @@ import {
SiemMigrationStatus,
RuleTranslationResult,
} from '../../../../../common/siem_migrations/constants';
import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen';
import {
type RuleMigration,
type RuleMigrationTaskStats,
type RuleMigrationTranslationStats,
} from '../../../../../common/siem_migrations/model/rule_migration.gen';
import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client';
import { getSortingOptions, type RuleMigrationSort } from './sort';
import { conditions as searchConditions } from './search';
import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client';
export type CreateRuleMigrationInput = Omit<
RuleMigration,

View file

@ -12,8 +12,7 @@ import type { InstallParams } from '@kbn/index-adapter';
import { IndexPatternAdapter, IndexAdapter } from '@kbn/index-adapter';
import { loggerMock } from '@kbn/logging-mocks';
import { Subject } from 'rxjs';
import type { SiemRuleMigrationsClientDependencies } from '../types';
import type { IndexNameProviders } from './rule_migrations_data_client';
import type { IndexNameProviders, SiemRuleMigrationsClientDependencies } from '../types';
import { INDEX_PATTERN, RuleMigrationsDataService } from './rule_migrations_data_service';
jest.mock('@kbn/index-adapter');
@ -45,7 +44,7 @@ describe('SiemRuleMigrationsDataService', () => {
describe('constructor', () => {
it('should create IndexPatternAdapters', () => {
new RuleMigrationsDataService(logger, kibanaVersion);
expect(MockedIndexPatternAdapter).toHaveBeenCalledTimes(2);
expect(MockedIndexPatternAdapter).toHaveBeenCalledTimes(3);
expect(MockedIndexAdapter).toHaveBeenCalledTimes(2);
});
@ -118,7 +117,8 @@ describe('SiemRuleMigrationsDataService', () => {
logger: loggerMock.create(),
pluginStop$: new Subject(),
};
const [rulesIndexPatternAdapter, resourcesIndexPatternAdapter] =
const [rulesIndexPatternAdapter, resourcesIndexPatternAdapter, migrationIndexPatternAdapter] =
MockedIndexPatternAdapter.mock.instances;
(rulesIndexPatternAdapter.install as jest.Mock).mockResolvedValueOnce(undefined);
@ -127,12 +127,16 @@ describe('SiemRuleMigrationsDataService', () => {
await mockIndexNameProviders.rules();
await mockIndexNameProviders.resources();
await mockIndexNameProviders.migrations();
expect(rulesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1');
expect(rulesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1');
expect(resourcesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1');
expect(resourcesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1');
expect(migrationIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1');
expect(migrationIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1');
});
});
});

View file

@ -11,11 +11,18 @@ import {
type FieldMap,
type InstallParams,
} from '@kbn/index-adapter';
import type { IndexNameProvider, IndexNameProviders } from './rule_migrations_data_client';
import type {} from './rule_migrations_data_client';
import { RuleMigrationsDataClient } from './rule_migrations_data_client';
import type { SiemRuleMigrationsClientDependencies } from '../types';
import type {
AdapterId,
Adapters,
SiemRuleMigrationsClientDependencies,
IndexNameProvider,
IndexNameProviders,
} from '../types';
import {
integrationsFieldMap,
migrationsFieldMaps,
prebuiltRulesFieldMap,
ruleMigrationResourcesFieldMap,
ruleMigrationsFieldMap,
@ -24,15 +31,6 @@ import {
const TOTAL_FIELDS_LIMIT = 2500;
export const INDEX_PATTERN = '.kibana-siem-rule-migrations';
export interface Adapters {
rules: IndexPatternAdapter;
resources: IndexPatternAdapter;
integrations: IndexAdapter;
prebuiltrules: IndexAdapter;
}
export type AdapterId = keyof Adapters;
interface CreateClientParams {
spaceId: string;
currentUser: AuthenticatedUser;
@ -49,6 +47,10 @@ export class RuleMigrationsDataService {
constructor(private logger: Logger, private kibanaVersion: string) {
this.adapters = {
migrations: this.createIndexPatternAdapter({
adapterId: 'migrations',
fieldMap: migrationsFieldMaps,
}),
rules: this.createIndexPatternAdapter({
adapterId: 'rules',
fieldMap: ruleMigrationsFieldMap,
@ -100,6 +102,7 @@ export class RuleMigrationsDataService {
this.adapters.resources.install({ ...params, logger: this.logger }),
this.adapters.integrations.install({ ...params, logger: this.logger }),
this.adapters.prebuiltrules.install({ ...params, logger: this.logger }),
this.adapters.migrations.install({ ...params, logger: this.logger }),
]);
}
@ -109,6 +112,7 @@ export class RuleMigrationsDataService {
resources: this.createIndexNameProvider(this.adapters.resources, spaceId),
integrations: async () => this.getAdapterIndexName('integrations'),
prebuiltrules: async () => this.getAdapterIndexName('prebuiltrules'),
migrations: this.createIndexNameProvider(this.adapters.migrations, spaceId),
};
return new RuleMigrationsDataClient(

View file

@ -10,7 +10,7 @@ import type {
RuleMigration,
RuleMigrationResource,
} from '../../../../../common/siem_migrations/model/rule_migration.gen';
import type { RuleMigrationIntegration, RuleMigrationPrebuiltRule } from '../types';
import type { SiemMigration, RuleMigrationIntegration, RuleMigrationPrebuiltRule } from '../types';
export const ruleMigrationsFieldMap: FieldMap<SchemaFieldMapKeys<Omit<RuleMigration, 'id'>>> = {
'@timestamp': { type: 'date', required: false },
@ -76,3 +76,8 @@ export const prebuiltRulesFieldMap: FieldMap<SchemaFieldMapKeys<RuleMigrationPre
rule_id: { type: 'keyword', required: true },
mitre_attack_ids: { type: 'keyword', array: true, required: false },
};
export const migrationsFieldMaps: FieldMap<SchemaFieldMapKeys<SiemMigration>> = {
created_at: { type: 'date', required: true },
created_by: { type: 'keyword', required: true },
};

View file

@ -11,18 +11,25 @@ import type { AnalyticsServiceSetup } from '@kbn/core/public';
import type { SavedObjectsClientContract } from '@kbn/core/server';
import type { PackageService } from '@kbn/fleet-plugin/server';
import type { InferenceClient } from '@kbn/inference-plugin/server';
import type { IndexAdapter, IndexPatternAdapter } from '@kbn/index-adapter';
import type {
RuleMigration,
RuleMigrationTranslationResult,
UpdateRuleMigrationData,
} from '../../../../common/siem_migrations/model/rule_migration.gen';
import {
type RuleMigration,
type RuleMigrationResource,
} from '../../../../common/siem_migrations/model/rule_migration.gen';
import { type RuleMigrationResource } from '../../../../common/siem_migrations/model/rule_migration.gen';
import type { RuleVersions } from './data/rule_migrations_data_prebuilt_rules_client';
export type Stored<T extends object> = T & { id: string };
export interface SiemMigration {
/** The moment the migration was created */
created_at: string;
/** The profile id of the user who created the migration */
created_by: string;
}
export type StoredSiemMigration = Stored<SiemMigration>;
export type StoredRuleMigration = Stored<RuleMigration>;
export type StoredRuleMigrationResource = Stored<RuleMigrationResource>;
@ -71,3 +78,16 @@ export type InternalUpdateRuleMigrationData = UpdateRuleMigrationData & {
*
**/
export type SplunkSeverity = '1' | '2' | '3' | '4' | '5';
export interface Adapters {
rules: IndexPatternAdapter;
resources: IndexPatternAdapter;
integrations: IndexAdapter;
prebuiltrules: IndexAdapter;
migrations: IndexPatternAdapter;
}
export type AdapterId = keyof Adapters;
export type IndexNameProvider = () => Promise<string>;
export type IndexNameProviders = Record<AdapterId, IndexNameProvider>;

View file

@ -241,6 +241,7 @@
"@kbn/security-ai-prompts",
"@kbn/scout-security",
"@kbn/custom-icons",
"@kbn/security-plugin-types-common",
"@kbn/management-settings-ids"
]
}