diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.test.ts index e9cc5273b58a..ca54ce647940 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.test.ts @@ -43,7 +43,7 @@ describe('AssetCriticalityDataClient', () => { index: '.asset-criticality.asset-criticality-default', mappings: { _meta: { - version: 2, + version: 3, }, dynamic: 'strict', properties: { @@ -56,6 +56,13 @@ describe('AssetCriticalityDataClient', () => { criticality_level: { type: 'keyword', }, + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, '@timestamp': { type: 'date', ignore_malformed: false, @@ -114,6 +121,9 @@ describe('AssetCriticalityDataClient', () => { }, }, }, + settings: { + default_pipeline: 'entity_analytics_create_eventIngest_from_timestamp-pipeline-default', + }, }, }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.ts index 14b4fb64fbd3..0cc028fc32ec 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.ts @@ -26,6 +26,10 @@ import { import { AssetCriticalityAuditActions } from './audit'; import { AUDIT_CATEGORY, AUDIT_OUTCOME, AUDIT_TYPE } from '../audit'; import { getImplicitEntityFields } from './helpers'; +import { + getIngestPipelineName, + createEventIngestedFromTimestamp, +} from '../utils/create_ingest_pipeline'; interface AssetCriticalityClientOpts { logger: Logger; @@ -62,6 +66,7 @@ export class AssetCriticalityDataClient { * Initialize asset criticality resources. */ public async init() { + await createEventIngestedFromTimestamp(this.options.esClient, this.options.namespace); await this.createOrUpdateIndex(); this.options.auditLogger?.log({ @@ -90,6 +95,9 @@ export class AssetCriticalityDataClient { version: ASSET_CRITICALITY_MAPPINGS_VERSIONS, }, }, + settings: { + default_pipeline: getIngestPipelineName(this.options.namespace), + }, }, }); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_migration_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_migration_client.ts index dfa3b3fdb632..dc79a467f143 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_migration_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_migration_client.ts @@ -95,4 +95,36 @@ export class AssetCriticalityMigrationClient { } ); }; + + public copyTimestampToEventIngestedForAssetCriticality = (abortSignal?: AbortSignal) => { + return this.options.esClient.updateByQuery( + { + index: this.assetCriticalityDataClient.getIndex(), + conflicts: 'proceed', + ignore_unavailable: true, + allow_no_indices: true, + body: { + query: { + bool: { + must_not: { + exists: { + field: 'event.ingested', + }, + }, + }, + }, + script: { + source: 'ctx._source.event.ingested = ctx._source.@timestamp', + lang: 'painless', + }, + }, + }, + { + requestTimeout: '5m', + retryOnTimeout: true, + maxRetries: 2, + signal: abortSignal, + } + ); + }; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.test.ts index deb2a1d5d8d4..c22057ea92d4 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.test.ts @@ -9,7 +9,7 @@ import { ASSET_CRITICALITY_MAPPINGS_VERSIONS, assetCriticalityFieldMap } from '. describe('asset criticality - constants', () => { it("please bump 'ASSET_CRITICALITY_MAPPINGS_VERSIONS' when mappings change", () => { - expect(ASSET_CRITICALITY_MAPPINGS_VERSIONS).toEqual(2); + expect(ASSET_CRITICALITY_MAPPINGS_VERSIONS).toEqual(3); expect(assetCriticalityFieldMap).toMatchInlineSnapshot(` Object { "@timestamp": Object { @@ -27,6 +27,11 @@ describe('asset criticality - constants', () => { "required": false, "type": "keyword", }, + "event.ingested": Object { + "array": false, + "required": false, + "type": "date", + }, "host.asset.criticality": Object { "array": false, "required": false, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.ts index f536e7e64908..3066b1352296 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.ts @@ -17,7 +17,7 @@ const assetCriticalityMapping = { }; // Upgrade this value to force a mappings update on the next Kibana startup -export const ASSET_CRITICALITY_MAPPINGS_VERSIONS = 2; +export const ASSET_CRITICALITY_MAPPINGS_VERSIONS = 3; export const assetCriticalityFieldMap: FieldMap = { '@timestamp': { @@ -58,6 +58,11 @@ export const assetCriticalityFieldMap: FieldMap = { required: false, }, 'user.asset.criticality': assetCriticalityMapping, + 'event.ingested': { + type: 'date', + array: false, + required: false, + }, 'service.name': { type: 'keyword', array: false, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/asset_criticality_copy_timestamp_to_event_ingested.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/asset_criticality_copy_timestamp_to_event_ingested.ts new file mode 100644 index 000000000000..cffd417b907c --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/asset_criticality_copy_timestamp_to_event_ingested.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EntityAnalyticsMigrationsParams } from '.'; +import { AssetCriticalityMigrationClient } from '../asset_criticality/asset_criticality_migration_client'; + +const TASK_TYPE = 'security-solution-ea-asset-criticality-copy-timestamp-to-event-ingested'; +const TASK_ID = `${TASK_TYPE}-task-id`; +const TASK_TIMEOUT = '15m'; +const TASK_SCOPE = ['securitySolution']; + +export const assetCrticalityCopyTimestampToEventIngested = async ({ + auditLogger, + taskManager, + logger, + getStartServices, +}: EntityAnalyticsMigrationsParams) => { + if (!taskManager) { + return; + } + + logger.debug(`Register task "${TASK_TYPE}"`); + + taskManager.registerTaskDefinitions({ + [TASK_TYPE]: { + title: `Copy Asset Criticality @timestamp value to events.ingested`, + timeout: TASK_TIMEOUT, + createTaskRunner: createMigrationTask({ auditLogger, logger, getStartServices }), + }, + }); + + const [_, depsStart] = await getStartServices(); + const taskManagerStart = depsStart.taskManager; + + if (taskManagerStart) { + logger.debug(`Task scheduled: "${TASK_TYPE}"`); + + const now = new Date(); + try { + await taskManagerStart.ensureScheduled({ + id: TASK_ID, + taskType: TASK_TYPE, + scheduledAt: now, + runAt: now, + scope: TASK_SCOPE, + params: {}, + state: {}, + }); + } catch (e) { + logger.error(`Error scheduling ${TASK_ID}, received ${e.message}`); + } + } +}; + +export const createMigrationTask = + ({ + getStartServices, + logger, + auditLogger, + }: Pick) => + () => { + let abortController: AbortController; + return { + run: async () => { + abortController = new AbortController(); + const [coreStart] = await getStartServices(); + const esClient = coreStart.elasticsearch.client.asInternalUser; + const assetCrticalityClient = new AssetCriticalityMigrationClient({ + esClient, + logger, + auditLogger, + }); + + const assetCriticalityResponse = + await assetCrticalityClient.copyTimestampToEventIngestedForAssetCriticality( + abortController.signal + ); + + const failures = assetCriticalityResponse.failures?.map((failure) => failure.cause); + const hasFailures = failures && failures?.length > 0; + + logger.info( + `Task "${TASK_TYPE}" finished. Updated documents: ${ + assetCriticalityResponse.updated + }, failures: ${hasFailures ? failures.join('\n') : 0}` + ); + }, + + cancel: async () => { + abortController.abort(); + logger.debug(`Task cancelled: "${TASK_TYPE}"`); + }, + }; + }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/index.ts index dac3099a0b3f..498d9c954c20 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/index.ts @@ -9,6 +9,8 @@ import type { AuditLogger, Logger, StartServicesAccessor } from '@kbn/core/serve import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; import type { StartPlugins } from '../../../plugin'; import { scheduleAssetCriticalityEcsCompliancyMigration } from '../asset_criticality/migrations/schedule_ecs_compliancy_migration'; +import { assetCrticalityCopyTimestampToEventIngested } from './asset_criticality_copy_timestamp_to_event_ingested'; +import { riskScoreCopyTimestampToEventIngested } from './risk_score_copy_timestamp_to_event_ingested'; import { updateAssetCriticalityMappings } from '../asset_criticality/migrations/update_asset_criticality_mappings'; import { updateRiskScoreMappings } from '../risk_engine/migrations/update_risk_score_mappings'; @@ -43,3 +45,21 @@ export const scheduleEntityAnalyticsMigration = async (params: EntityAnalyticsMi await scheduleAssetCriticalityEcsCompliancyMigration({ ...params, logger: scopedLogger }); await updateRiskScoreMappings({ ...params, logger: scopedLogger }); }; + +export const scheduleAssetCriticalityCopyTimestampToEventIngested = async ( + params: EntityAnalyticsMigrationsParams +) => { + const scopedLogger = params.logger.get( + 'entityAnalytics.assetCriticality.copyTimestampToEventIngested' + ); + + await assetCrticalityCopyTimestampToEventIngested({ ...params, logger: scopedLogger }); +}; + +export const scheduleRiskScoreCopyTimestampToEventIngested = async ( + params: EntityAnalyticsMigrationsParams +) => { + const scopedLogger = params.logger.get('entityAnalytics.riskScore.copyTimestampToEventIngested'); + + await riskScoreCopyTimestampToEventIngested({ ...params, logger: scopedLogger }); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/risk_score_copy_timestamp_to_event_ingested.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/risk_score_copy_timestamp_to_event_ingested.ts new file mode 100644 index 000000000000..3fb5eecae237 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/risk_score_copy_timestamp_to_event_ingested.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EntityAnalyticsMigrationsParams } from '.'; +import { RiskScoreDataClient } from '../risk_score/risk_score_data_client'; +import { buildScopedInternalSavedObjectsClientUnsafe } from '../risk_score/tasks/helpers'; + +const TASK_TYPE = 'security-solution-ea-risk-score-copy-timestamp-to-event-ingested'; +const TASK_ID = `${TASK_TYPE}-task-id`; +const TASK_TIMEOUT = '15m'; +const TASK_SCOPE = ['securitySolution']; + +export const riskScoreCopyTimestampToEventIngested = async ({ + auditLogger, + taskManager, + logger, + getStartServices, +}: EntityAnalyticsMigrationsParams) => { + if (!taskManager) { + return; + } + + logger.debug(`Register task "${TASK_TYPE}"`); + + taskManager.registerTaskDefinitions({ + [TASK_TYPE]: { + title: `Copy Risk Score @timestamp value to events.ingested`, + timeout: TASK_TIMEOUT, + createTaskRunner: createMigrationTask({ auditLogger, logger, getStartServices }), + }, + }); + + const [_, depsStart] = await getStartServices(); + const taskManagerStart = depsStart.taskManager; + + if (taskManagerStart) { + logger.debug(`Task scheduled: "${TASK_TYPE}"`); + + const now = new Date(); + try { + await taskManagerStart.ensureScheduled({ + id: TASK_ID, + taskType: TASK_TYPE, + scheduledAt: now, + runAt: now, + scope: TASK_SCOPE, + params: {}, + state: {}, + }); + } catch (e) { + logger.error(`Error scheduling ${TASK_ID}, received ${e.message}`); + } + } +}; + +export const createMigrationTask = + ({ + getStartServices, + logger, + auditLogger, + }: Pick) => + () => { + let abortController: AbortController; + return { + run: async () => { + abortController = new AbortController(); + const [coreStart] = await getStartServices(); + const esClient = coreStart.elasticsearch.client.asInternalUser; + const soClient = buildScopedInternalSavedObjectsClientUnsafe({ coreStart, namespace: '*' }); + + const riskScoreClient = new RiskScoreDataClient({ + esClient, + logger, + auditLogger, + namespace: '*', + soClient, + kibanaVersion: '*', + }); + const riskScoreResponse = await riskScoreClient.copyTimestampToEventIngestedForRiskScore( + abortController.signal + ); + const failures = riskScoreResponse.failures?.map((failure) => failure.cause); + const hasFailures = failures && failures?.length > 0; + + logger.info( + `Task "${TASK_TYPE}" finished. Updated documents: ${ + riskScoreResponse.updated + }, failures: ${hasFailures ? failures.join('\n') : 0}` + ); + }, + + cancel: async () => { + abortController.abort(); + logger.debug(`Task cancelled: "${TASK_TYPE}"`); + }, + }; + }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.test.ts index 60e9f44965a7..3ce478af30c6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.test.ts @@ -13,7 +13,7 @@ describe('#getDefaultRiskEngineConfiguration', () => { const namespace = 'default'; const config = getDefaultRiskEngineConfiguration({ namespace }); - expect(config._meta.mappingsVersion).toEqual(2); + expect(config._meta.mappingsVersion).toEqual(3); expect(riskScoreFieldMap).toMatchInlineSnapshot(` Object { "@timestamp": Object { @@ -21,6 +21,11 @@ describe('#getDefaultRiskEngineConfiguration', () => { "required": false, "type": "date", }, + "event.ingested": Object { + "array": false, + "required": false, + "type": "date", + }, "host.name": Object { "array": false, "required": false, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.ts index 3af27649f3d5..e9b756043395 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.ts @@ -28,7 +28,7 @@ export const getDefaultRiskEngineConfiguration = ({ range: { start: 'now-30d', end: 'now' }, _meta: { // Upgrade this property when changing mappings - mappingsVersion: 2, + mappingsVersion: 3, }, }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/__snapshots__/risk_score_data_client.test.ts.snap b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/__snapshots__/risk_score_data_client.test.ts.snap index 922ae18a4dfc..6e48a94640cb 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/__snapshots__/risk_score_data_client.test.ts.snap +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/__snapshots__/risk_score_data_client.test.ts.snap @@ -9,6 +9,13 @@ Object { "ignore_malformed": false, "type": "date", }, + "event": Object { + "properties": Object { + "ingested": Object { + "type": "date", + }, + }, + }, "host": Object { "properties": Object { "name": Object { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/configurations.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/configurations.ts index d6f5983ca632..962d6d5002dc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/configurations.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/configurations.ts @@ -102,6 +102,11 @@ export const riskScoreFieldMap: FieldMap = { array: false, required: false, }, + 'event.ingested': { + type: 'date', + array: false, + required: false, + }, 'host.name': { type: 'keyword', array: false, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_data_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_data_client.test.ts index 1e285e27c4fe..393cfd5c7b49 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_data_client.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_data_client.test.ts @@ -210,7 +210,202 @@ const assertIndex = (namespace: string) => { esClient, options: { index: `risk-score.risk-score-latest-${namespace}`, - mappings: expect.any(Object), + mappings: { + dynamic: false, + properties: { + '@timestamp': { + ignore_malformed: false, + type: 'date', + }, + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, + host: { + properties: { + name: { + type: 'keyword', + }, + risk: { + properties: { + calculated_level: { + type: 'keyword', + }, + calculated_score: { + type: 'float', + }, + calculated_score_norm: { + type: 'float', + }, + category_1_count: { + type: 'long', + }, + category_1_score: { + type: 'float', + }, + id_field: { + type: 'keyword', + }, + id_value: { + type: 'keyword', + }, + notes: { + type: 'keyword', + }, + inputs: { + properties: { + id: { + type: 'keyword', + }, + index: { + type: 'keyword', + }, + category: { + type: 'keyword', + }, + description: { + type: 'keyword', + }, + risk_score: { + type: 'float', + }, + timestamp: { + type: 'date', + }, + }, + type: 'object', + }, + }, + type: 'object', + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + }, + risk: { + properties: { + calculated_level: { + type: 'keyword', + }, + calculated_score: { + type: 'float', + }, + calculated_score_norm: { + type: 'float', + }, + category_1_count: { + type: 'long', + }, + category_1_score: { + type: 'float', + }, + id_field: { + type: 'keyword', + }, + id_value: { + type: 'keyword', + }, + inputs: { + properties: { + category: { + type: 'keyword', + }, + description: { + type: 'keyword', + }, + id: { + type: 'keyword', + }, + index: { + type: 'keyword', + }, + risk_score: { + type: 'float', + }, + timestamp: { + type: 'date', + }, + }, + type: 'object', + }, + notes: { + type: 'keyword', + }, + }, + type: 'object', + }, + }, + }, + user: { + properties: { + name: { + type: 'keyword', + }, + risk: { + properties: { + calculated_level: { + type: 'keyword', + }, + calculated_score: { + type: 'float', + }, + calculated_score_norm: { + type: 'float', + }, + category_1_count: { + type: 'long', + }, + category_1_score: { + type: 'float', + }, + id_field: { + type: 'keyword', + }, + id_value: { + type: 'keyword', + }, + notes: { + type: 'keyword', + }, + inputs: { + properties: { + id: { + type: 'keyword', + }, + index: { + type: 'keyword', + }, + category: { + type: 'keyword', + }, + description: { + type: 'keyword', + }, + risk_score: { + type: 'float', + }, + timestamp: { + type: 'date', + }, + }, + type: 'object', + }, + }, + type: 'object', + }, + }, + }, + }, + }, + settings: { + 'index.default_pipeline': `entity_analytics_create_eventIngest_from_timestamp-pipeline-${namespace}`, + }, }, }); }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_data_client.ts index 74391c6bc3c5..2542e33c92be 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_data_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_data_client.ts @@ -45,6 +45,10 @@ import { createOrUpdateIndex } from '../utils/create_or_update_index'; import { retryTransientEsErrors } from '../utils/retry_transient_es_errors'; import { RiskScoreAuditActions } from './audit'; import { AUDIT_CATEGORY, AUDIT_OUTCOME, AUDIT_TYPE } from '../audit'; +import { + createEventIngestedFromTimestamp, + getIngestPipelineName, +} from '../utils/create_ingest_pipeline'; interface RiskScoringDataClientOpts { logger: Logger; @@ -100,6 +104,9 @@ export class RiskScoreDataClient { options: { index: getRiskScoreLatestIndex(this.options.namespace), mappings: mappingFromFieldMap(riskScoreFieldMap, false), + settings: { + 'index.default_pipeline': getIngestPipelineName(this.options.namespace), + }, }, }); }; @@ -130,9 +137,10 @@ export class RiskScoreDataClient { public async init() { const namespace = this.options.namespace; + const esClient = this.options.esClient; try { - const esClient = this.options.esClient; + await createEventIngestedFromTimestamp(esClient, namespace); const indexPatterns = getIndexPatternDataStream(namespace); @@ -169,6 +177,7 @@ export class RiskScoreDataClient { lifecycle: {}, settings: { 'index.mapping.total_fields.limit': totalFieldsLimit, + 'index.default_pipeline': getIngestPipelineName(namespace), }, mappings: { dynamic: false, @@ -340,6 +349,38 @@ export class RiskScoreDataClient { }); } + public copyTimestampToEventIngestedForRiskScore = (abortSignal?: AbortSignal) => { + return this.options.esClient.updateByQuery( + { + index: getRiskScoreLatestIndex(this.options.namespace), + conflicts: 'proceed', + ignore_unavailable: true, + allow_no_indices: true, + body: { + query: { + bool: { + must_not: { + exists: { + field: 'event.ingested', + }, + }, + }, + }, + script: { + source: 'ctx._source.event.ingested = ctx._source.@timestamp', + lang: 'painless', + }, + }, + }, + { + requestTimeout: '5m', + retryOnTimeout: true, + maxRetries: 2, + signal: abortSignal, + } + ); + }; + public async reinstallTransform() { const esClient = this.options.esClient; const namespace = this.options.namespace; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/create_ingest_pipeline.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/create_ingest_pipeline.ts new file mode 100644 index 000000000000..183243e98be9 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/create_ingest_pipeline.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; + +export const getIngestPipelineName = (namespace: string): string => { + return `entity_analytics_create_eventIngest_from_timestamp-pipeline-${namespace}`; +}; + +export const createEventIngestedFromTimestamp = async ( + esClient: ElasticsearchClient, + namespace: string +) => { + const ingestTimestampPipeline = getIngestPipelineName(namespace); + + try { + const pipeline = { + id: ingestTimestampPipeline, + body: { + _meta: { + managed_by: 'entity_analytics', + managed: true, + }, + description: 'Pipeline for adding timestamp value to event.ingested', + processors: [ + { + set: { + field: 'event.ingested', + value: '{{_ingest.timestamp}}', + }, + }, + ], + }, + }; + + await esClient.ingest.putPipeline(pipeline); + } catch (e) { + throw new Error(`Error creating ingest pipeline: ${e}`); + } +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality.ts index d5d802acb392..71f7642982f3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality.ts @@ -71,7 +71,7 @@ export default ({ getService }: FtrProviderContext) => { assetCriticalityIndexResult['.asset-criticality.asset-criticality-default']?.mappings ).to.eql({ _meta: { - version: 2, + version: 3, }, dynamic: 'strict', properties: { @@ -81,6 +81,13 @@ export default ({ getService }: FtrProviderContext) => { criticality_level: { type: 'keyword', }, + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, id_field: { type: 'keyword', }, @@ -160,8 +167,7 @@ export default ({ getService }: FtrProviderContext) => { expect(result['@timestamp']).to.be.a('string'); const doc = await getAssetCriticalityDoc({ idField: 'host.name', idValue: 'host-01', es }); - - expect(doc).to.eql(result); + expect(_.omit(doc, 'event')).to.eql(result); }); it('should return 400 if criticality is invalid', async () => { @@ -372,7 +378,7 @@ export default ({ getService }: FtrProviderContext) => { const doc = await getAssetCriticalityDoc({ idField: 'host.name', idValue: 'host-01', es }); - expect(doc).to.eql(updatedDoc); + expect(_.omit(doc, 'event')).to.eql(_.omit(updatedDoc, 'event')); }); }); @@ -387,7 +393,7 @@ export default ({ getService }: FtrProviderContext) => { idValue: expectedDoc.id_value, }); - expect(omit(esDoc, '@timestamp')).to.eql(expectedDoc); + expect(omit(esDoc, ['@timestamp', 'event'])).to.eql(expectedDoc); }; it('should return 400 if the records array is empty', async () => { @@ -478,9 +484,8 @@ export default ({ getService }: FtrProviderContext) => { await assetCriticalityRoutes.upsert(assetCriticality); const res = await assetCriticalityRoutes.delete('host.name', 'delete-me'); - expect(res.body.deleted).to.eql(true); - expect(_.omit(res.body.record, '@timestamp')).to.eql( + expect(_.omit(res.body.record, ['@timestamp', 'event'])).to.eql( assetCreateTypeToAssetRecord(assetCriticality) ); @@ -494,7 +499,9 @@ export default ({ getService }: FtrProviderContext) => { ...assetCriticality, criticality_level: CRITICALITY_VALUES.DELETED, }; - expect(_.omit(doc, '@timestamp')).to.eql(assetCreateTypeToAssetRecord(deletedDoc)); + expect(_.omit(doc, ['@timestamp', 'event'])).to.eql( + assetCreateTypeToAssetRecord(deletedDoc) + ); }); it('should not return 404 if the asset criticality does not exist', async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts index f87916d1af12..3252621024dc 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts @@ -71,6 +71,8 @@ export default ({ getService }: FtrProviderContext) => { const dataStreamName = 'risk-score.risk-score-default'; const latestIndexName = 'risk-score.risk-score-latest-default'; const transformId = 'risk_score_latest_transform_default'; + const defaultPipeline = + 'entity_analytics_create_eventIngest_from_timestamp-pipeline-default'; await riskEngineRoutes.init(); @@ -89,6 +91,13 @@ export default ({ getService }: FtrProviderContext) => { ignore_malformed: false, type: 'date', }, + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, host: { properties: { name: { @@ -288,6 +297,7 @@ export default ({ getService }: FtrProviderContext) => { expect(indexTemplate.index_template.template!.settings).to.eql({ index: { + default_pipeline: defaultPipeline, mapping: { total_fields: { limit: '1000', @@ -341,6 +351,7 @@ export default ({ getService }: FtrProviderContext) => { const dataStreamName = `risk-score.risk-score-${customSpaceName}`; const latestIndexName = `risk-score.risk-score-latest-${customSpaceName}`; const transformId = `risk_score_latest_transform_${customSpaceName}`; + const defaultPipeline = `entity_analytics_create_eventIngest_from_timestamp-pipeline-${customSpaceName}`; await riskEngineRoutesWithNamespace.init(); @@ -359,6 +370,13 @@ export default ({ getService }: FtrProviderContext) => { ignore_malformed: false, type: 'date', }, + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, host: { properties: { name: { @@ -562,6 +580,7 @@ export default ({ getService }: FtrProviderContext) => { expect(indexTemplate.index_template.template!.settings).to.eql({ index: { + default_pipeline: defaultPipeline, mapping: { total_fields: { limit: '1000', @@ -626,7 +645,7 @@ export default ({ getService }: FtrProviderContext) => { start: 'now-30d', }, _meta: { - mappingsVersion: 2, + mappingsVersion: 3, }, }); });