github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 01:13:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[SecuritySolution] Update Entity analytics BE to support service entity type (#203409)

Browse source 
## Summary

Update Entity Analytics BE to support the new entity type "service".
* Hide all functionality behind an Experimental Flag
(`serviceEntityStoreEnabled`)
* Update asset criticality assignment
* Update Bulk upload logic
* Update Risk score calculation
* Create plugin setup mappings migration
  * Add service to risk score indices and templates
  * Add service to asset criticality index
* Create a reusable migration workflow where we only need to update the
mappings and bump the version
* Add a risk score transform migration when the schedule is now called
  * It will delete and reinstall the transform to apply the changes 

### issues
* I had to update the API doc to include service even though it is
behind an Experimental Flag
* The risk scope mappings migration runs on every space. If the users
have thousands of spaces, it could take some time.

### What is not included?
* UI changes


## Documentation for Entity Analytics future migrations

### How to add a new field to the risk score index and template
mappings?
* Update the mapping object
[here](6f8b5f6c51/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/configurations.ts (L102))
* Pump the `mappingsVersion` version
[here](8333bea86f/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.ts (L31))

### How to add a new field to the asset criticality index?
* Update the mapping object
[here](8333bea86f/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.ts (L22))
* Pump the `ASSET_CRITICALITY_MAPPINGS_VERSIONS` version
[here](8333bea86f/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.ts (L20))

### How to update the risk score transform config?
* Update the transform config
[here](6f8b5f6c51/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/configurations.ts (L162))
* Pump the `version`
[here](6f8b5f6c51/x-pack/plugins/security_solution/server/lib/entity_analytics/risk_score/configurations.ts (L190))

*note: If you change the `latest` property, the transform will reinstall
after the engine task runs

## How to test it?
* Enable the fla `serviceEntityStoreEnabled`
* Start ES and an old version of Kibana
* Populate it with data, start the risk engine
  * You could also run the document generator `yarn start entity-store` 
*  Make sure you have some alerts with `service.name` field populated
* Migrate to the version on this PR
* Run the risk engine
* You should see risk score documents created for service entities
* All asset criticality API should support `service` entities

## Checklist
- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Pablo Machado 2025-01-02 13:50:08 +01:00 • committed by GitHub
parent 529f833ac8
commit 1fbd86f199
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
73 changed files with 1943 additions and 564 deletions
Download patch file Download diff file Expand all files Collapse all files

15
oas_docs/output/kibana.serverless.yaml
View file

@ -46989,6 +46989,20 @@ components:
type: string
required:
- name
service:
type: object
properties:
asset:
type: object
properties:
criticality:
$ref: '#/components/schemas/Security_Entity_Analytics_API_AssetCriticalityLevel'
required:
- criticality
name:
type: string
required:
- name
user:
type: object
properties:
@ -47307,6 +47321,7 @@ components:
enum:
- host.name
- user.name
- service.name
type: string
Security_Entity_Analytics_API_IndexPattern:
type: string

15
oas_docs/output/kibana.yaml
View file

@ -53865,6 +53865,20 @@ components:
type: string
required:
- name
service:
type: object
properties:
asset:
type: object
properties:
criticality:
$ref: '#/components/schemas/Security_Entity_Analytics_API_AssetCriticalityLevel'
required:
- criticality
name:
type: string
required:
- name
user:
type: object
properties:
@ -54183,6 +54197,7 @@ components:
enum:
- host.name
- user.name
- service.name
type: string
Security_Entity_Analytics_API_IndexPattern:
type: string

1
x-pack/platform/plugins/private/translations/translations/fr-FR.json
View file

@ -36783,7 +36783,6 @@
"xpack.securitySolution.assetCriticality.csvUpload.expectedColumnsError": "Trois colonnes attendues, {rowLength} reçues",
"xpack.securitySolution.assetCriticality.csvUpload.idTooLongError": "L’identificateur est trop long. Il devrait contenir moins de {maxChars} caractères, mais en contient {idLength}",
"xpack.securitySolution.assetCriticality.csvUpload.invalidCriticalityError": "Niveau de criticité non valide \"{criticalityLevel}\", un des {validLevels} attendu",
"xpack.securitySolution.assetCriticality.csvUpload.invalidEntityTypeError": "Type d'entité \"{entityType}\" non valide, hôte ou utilisateur attendu",
"xpack.securitySolution.assetCriticality.csvUpload.missingCriticalityError": "Niveau de criticité manquant",
"xpack.securitySolution.assetCriticality.csvUpload.missingEntityTypeError": "Type d'entité manquant",
"xpack.securitySolution.assetCriticality.csvUpload.missingIdError": "Identificateur manquant",

1
x-pack/platform/plugins/private/translations/translations/ja-JP.json
View file

@ -36642,7 +36642,6 @@
"xpack.securitySolution.assetCriticality.csvUpload.expectedColumnsError": "3列でなければなりませんが、{rowLength}列でした",
"xpack.securitySolution.assetCriticality.csvUpload.idTooLongError": "識別子が長すぎます。{maxChars}未満でなければなりませんが、{idLength}でした",
"xpack.securitySolution.assetCriticality.csvUpload.invalidCriticalityError": "無効な重要度レベル\"{criticalityLevel}\"です。{validLevels}のいずれかでなければなりません",
"xpack.securitySolution.assetCriticality.csvUpload.invalidEntityTypeError": "無効なエンティティタイプ\"{entityType}\"です。ホストまたはユーザーでなければなりません",
"xpack.securitySolution.assetCriticality.csvUpload.missingCriticalityError": "重要度レベルがありません",
"xpack.securitySolution.assetCriticality.csvUpload.missingEntityTypeError": "エンティティタイプがありません",
"xpack.securitySolution.assetCriticality.csvUpload.missingIdError": "識別子がありません",

12
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts
View file

@ -17,7 +17,7 @@
import { z } from '@kbn/zod';
export type IdField = z.infer<typeof IdField>;
export const IdField = z.enum(['host.name', 'user.name']);
export const IdField = z.enum(['host.name', 'user.name', 'service.name']);
export type IdFieldEnum = typeof IdField.enum;
export const IdFieldEnum = IdField.enum;
@ -78,6 +78,16 @@ export const AssetCriticalityRecordEcsParts = z.object({
.optional(),
})
.optional(),
service: z
.object({
name: z.string(),
asset: z
.object({
criticality: AssetCriticalityLevel,
})
.optional(),
})
.optional(),
});
export type AssetCriticalityRecord = z.infer<typeof AssetCriticalityRecord>;

15
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml
View file

@ -28,6 +28,7 @@ components:
enum:
- 'host.name'
- 'user.name'
- 'service.name'
AssetCriticalityRecordIdParts:
type: object
properties:
@ -109,5 +110,19 @@ components:
- 'criticality'
required:
- 'name'
'service':
type: object
properties:
'name':
type: string
'asset':
type: object
properties:
'criticality':
$ref: '#/components/schemas/AssetCriticalityLevel'
required:
- 'criticality'
required:
- 'name'
required:
- 'asset'

18
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/common/common.gen.ts
View file

@ -39,6 +39,7 @@ export type AfterKeys = z.infer<typeof AfterKeys>;
export const AfterKeys = z.object({
host: EntityAfterKey.optional(),
user: EntityAfterKey.optional(),
service: EntityAfterKey.optional(),
});
/**
@ -72,7 +73,7 @@ export const DateRange = z.object({
});
export type IdentifierType = z.infer<typeof IdentifierType>;
export const IdentifierType = z.enum(['host', 'user']);
export const IdentifierType = z.enum(['host', 'user', 'service']);
export type IdentifierTypeEnum = typeof IdentifierType.enum;
export const IdentifierTypeEnum = IdentifierType.enum;
@ -169,22 +170,33 @@ export const RiskScoreWeightGlobalShared = z.object({
type: z.literal('global_identifier'),
});
export type RiskScoreWeight = z.infer<typeof RiskScoreWeight>;
export const RiskScoreWeight = z.union([
export const RiskScoreWeightInternal = z.union([
RiskScoreWeightGlobalShared.merge(
z.object({
host: RiskScoreEntityIdentifierWeights,
user: RiskScoreEntityIdentifierWeights.optional(),
service: RiskScoreEntityIdentifierWeights.optional(),
})
),
RiskScoreWeightGlobalShared.merge(
z.object({
host: RiskScoreEntityIdentifierWeights.optional(),
user: RiskScoreEntityIdentifierWeights,
service: RiskScoreEntityIdentifierWeights.optional(),
})
),
RiskScoreWeightGlobalShared.merge(
z.object({
host: RiskScoreEntityIdentifierWeights.optional(),
user: RiskScoreEntityIdentifierWeights.optional(),
service: RiskScoreEntityIdentifierWeights,
})
),
]);
export type RiskScoreWeight = z.infer<typeof RiskScoreWeightInternal>;
export const RiskScoreWeight = RiskScoreWeightInternal as z.ZodType<RiskScoreWeight>;
/**
* A list of weights to be applied to the scoring calculation.
*/

24
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/common/common.schema.yaml
View file

@ -52,11 +52,16 @@ components:
$ref: '#/components/schemas/EntityAfterKey'
user:
$ref: '#/components/schemas/EntityAfterKey'
service:
$ref: '#/components/schemas/EntityAfterKey'
example:
host:
'host.name': 'example.host'
user:
'user.name': 'example_user_name'
service:
'service.name': 'example_service_name'
DataViewId:
description: The identifier of the Kibana data view to be used when generating risk scores.
@ -94,6 +99,7 @@ components:
enum:
- host
- user
- service
RiskScoreInput:
description: A generic representation of a document contributing to a Risk Score.
@ -247,6 +253,8 @@ components:
$ref: '#/components/schemas/RiskScoreEntityIdentifierWeights'
user:
$ref: '#/components/schemas/RiskScoreEntityIdentifierWeights'
service:
$ref: '#/components/schemas/RiskScoreEntityIdentifierWeights'
- allOf:
- $ref: '#/components/schemas/RiskScoreWeightGlobalShared'
@ -258,7 +266,20 @@ components:
$ref: '#/components/schemas/RiskScoreEntityIdentifierWeights'
user:
$ref: '#/components/schemas/RiskScoreEntityIdentifierWeights'
service:
$ref: '#/components/schemas/RiskScoreEntityIdentifierWeights'
- allOf:
- $ref: '#/components/schemas/RiskScoreWeightGlobalShared'
- type: object
required:
- service
properties:
host:
$ref: '#/components/schemas/RiskScoreEntityIdentifierWeights'
user:
$ref: '#/components/schemas/RiskScoreEntityIdentifierWeights'
service:
$ref: '#/components/schemas/RiskScoreEntityIdentifierWeights'
RiskScoreWeights:
description: 'A list of weights to be applied to the scoring calculation.'
type: array
@ -268,6 +289,7 @@ components:
- type: 'global_identifier'
host: 0.5
user: 0.1
service: 0.4
TaskManagerUnavailableResponse:
description: Task manager is unavailable

2
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/common/risk_weights.schema.test.ts
View file

@ -121,7 +121,7 @@ describe('risk weight schema', () => {
expect(decoded.success).toBeFalsy();
expect(stringifyZodError(decoded.error)).toEqual(
'type: Invalid literal value, expected "global_identifier", host: Required, type: Invalid literal value, expected "global_identifier"'
'type: Invalid literal value, expected "global_identifier", host: Required, type: Invalid literal value, expected "global_identifier", type: Invalid literal value, expected "global_identifier", service: Required'
);
});

4
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/calculation_route.gen.ts
View file

@ -42,6 +42,10 @@ export const RiskScoresCalculationResponse = z.object({
* A list of user risk scores
*/
user: z.array(EntityRiskScoreRecord).optional(),
/**
* A list of service risk scores
*/
service: z.array(EntityRiskScoreRecord).optional(),
/**
* If 'wait_for' the request will wait for the index refresh.
*/

5
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/calculation_route.schema.yaml
View file

@ -40,6 +40,11 @@ components:
items:
$ref: '../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord'
description: A list of user risk scores
service:
type: array
items:
$ref: '../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord'
description: A list of service risk scores
refresh:
type: string
enum: [wait_for]

4
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.gen.ts
View file

@ -89,6 +89,10 @@ export const RiskScoresPreviewResponse = z.object({
* A list of user risk scores
*/
user: z.array(EntityRiskScoreRecord).optional(),
/**
* A list of service risk scores
*/
service: z.array(EntityRiskScoreRecord).optional(),
}),
});

5
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/risk_engine/preview_route.schema.yaml
View file

@ -101,3 +101,8 @@ components:
items:
$ref: '../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord'
description: A list of user risk scores
service:
type: array
items:
$ref: '../common/common.schema.yaml#/components/schemas/EntityRiskScoreRecord'
description: A list of service risk scores

4
x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/parse_asset_criticality_csv_row.test.ts
View file

@ -46,7 +46,7 @@ describe('parseAssetCriticalityCsvRow', () => {
// @ts-ignore result can now only be InvalidRecord
expect(result.error).toMatchInlineSnapshot(
`"Invalid entity type \\"invalid\\", expected host or user"`
`"Invalid entity type \\"invalid\\", expected to be one of: user, host, service"`
);
});
@ -57,7 +57,7 @@ describe('parseAssetCriticalityCsvRow', () => {
// @ts-ignore result can now only be InvalidRecord
expect(result.error).toMatchInlineSnapshot(
`"Invalid entity type \\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...\\", expected host or user"`
`"Invalid entity type \\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...\\", expected to be one of: user, host, service"`
);
});

15
x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/parse_asset_criticality_csv_row.ts
View file

@ -7,7 +7,9 @@
import { i18n } from '@kbn/i18n';
import type { CriticalityLevels } from './constants';
import { ValidCriticalityLevels } from './constants';
import type { AssetCriticalityUpsert, CriticalityLevel } from './types';
import { type AssetCriticalityUpsert, type CriticalityLevel } from './types';
import { IDENTITY_FIELD_MAP, getAvailableEntityTypes } from '../entity_store/constants';
import type { EntityType } from '../../api/entity_analytics';
const MAX_COLUMN_CHARS = 1000;
@ -98,16 +100,19 @@ export const parseAssetCriticalityCsvRow = (row: string[]): ReturnType => {
);
}
if (entityType !== 'host' && entityType !== 'user') {
if (!getAvailableEntityTypes().includes(entityType as EntityType)) {
return validationErrorWithMessage(
i18n.translate('xpack.securitySolution.assetCriticality.csvUpload.invalidEntityTypeError', {
defaultMessage: 'Invalid entity type "{entityType}", expected host or user',
values: { entityType: trimColumn(entityType) },
defaultMessage: 'Invalid entity type "{entityType}", expected to be one of: {validTypes}',
values: {
entityType: trimColumn(entityType),
validTypes: getAvailableEntityTypes().join(', '),
},
})
);
}
const idField = entityType === 'host' ? 'host.name' : 'user.name';
const idField = IDENTITY_FIELD_MAP[entityType as EntityType];
return {
valid: true,

12
x-pack/solutions/security/plugins/security_solution/common/entity_analytics/entity_store/constants.ts
View file

@ -5,6 +5,9 @@
* 2.0.
*/
import type { EntityType, IdField } from '../../api/entity_analytics';
import { EntityTypeEnum } from '../../api/entity_analytics';
/**
* Entity Store routes
*/
@ -23,3 +26,12 @@ export const ENTITY_STORE_REQUIRED_ES_CLUSTER_PRIVILEGES = [
// The index pattern for the entity store has to support '.entities.v1.latest.noop' index
export const ENTITY_STORE_INDEX_PATTERN = '.entities.v1.latest.*';
export const IDENTITY_FIELD_MAP: Record<EntityType, IdField> = {
[EntityTypeEnum.host]: 'host.name',
[EntityTypeEnum.user]: 'user.name',
[EntityTypeEnum.service]: 'service.name',
};
export const getAvailableEntityTypes = (): EntityType[] =>
Object.keys(EntityTypeEnum) as EntityType[];

2
x-pack/solutions/security/plugins/security_solution/common/entity_analytics/risk_engine/identifier_types.ts
View file

@ -7,6 +7,6 @@
import * as t from 'io-ts';
export const identifierTypeSchema = t.keyof({ user: null, host: null });
export const identifierTypeSchema = t.keyof({ user: null, host: null, service: null });
export type IdentifierTypeSchema = t.TypeOf<typeof identifierTypeSchema>;
export type IdentifierType = IdentifierTypeSchema;

3
x-pack/solutions/security/plugins/security_solution/common/entity_analytics/risk_engine/types.ts
View file

@ -10,7 +10,10 @@ import type { EntityRiskScoreRecord, RiskScoreInput } from '../../api/entity_ana
export enum RiskScoreEntity {
host = 'host',
user = 'user',
// TODO Add service when FE is updated https://github.com/elastic/security-team/issues/11326
}
// TODO: Remove this when FE is updated https://github.com/elastic/security-team/issues/11326
export const SERVICE_RISK_SCORE_ENTITY = 'service';
export interface InitRiskEngineResult {
legacyRiskEngineDisabled: boolean;

15
x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
View file

@ -800,6 +800,20 @@ components:
type: string
required:
- name
service:
type: object
properties:
asset:
type: object
properties:
criticality:
$ref: '#/components/schemas/AssetCriticalityLevel'
required:
- criticality
name:
type: string
required:
- name
user:
type: object
properties:
@ -1130,6 +1144,7 @@ components:
enum:
- host.name
- user.name
- service.name
type: string
IndexPattern:
type: string

15
x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
View file

@ -800,6 +800,20 @@ components:
type: string
required:
- name
service:
type: object
properties:
asset:
type: object
properties:
criticality:
$ref: '#/components/schemas/AssetCriticalityLevel'
required:
- criticality
name:
type: string
required:
- name
user:
type: object
properties:
@ -1130,6 +1144,7 @@ components:
enum:
- host.name
- user.name
- service.name
type: string
IndexPattern:
type: string

17
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.test.ts
View file

@ -42,6 +42,9 @@ describe('AssetCriticalityDataClient', () => {
options: {
index: '.asset-criticality.asset-criticality-default',
mappings: {
_meta: {
version: 2,
},
dynamic: 'strict',
properties: {
id_field: {
@ -81,6 +84,20 @@ describe('AssetCriticalityDataClient', () => {
},
},
},
service: {
properties: {
asset: {
properties: {
criticality: {
type: 'keyword',
},
},
},
name: {
type: 'keyword',
},
},
},
user: {
properties: {
asset: {

23
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.ts
View file

@ -10,7 +10,6 @@ import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import { mappingFromFieldMap } from '@kbn/alerting-plugin/common';
import type { AuditLogger } from '@kbn/security-plugin-types-server';
import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
import type {
BulkUpsertAssetCriticalityRecordsResponse,
AssetCriticalityUpsert,
@ -19,7 +18,11 @@ import type { AssetCriticalityRecord } from '../../../../common/api/entity_analy
import { createOrUpdateIndex } from '../utils/create_or_update_index';
import { getAssetCriticalityIndex } from '../../../../common/entity_analytics/asset_criticality';
import type { CriticalityValues } from './constants';
import { CRITICALITY_VALUES, assetCriticalityFieldMap } from './constants';
import {
ASSET_CRITICALITY_MAPPINGS_VERSIONS,
CRITICALITY_VALUES,
assetCriticalityFieldMap,
} from './constants';
import { AssetCriticalityAuditActions } from './audit';
import { AUDIT_CATEGORY, AUDIT_OUTCOME, AUDIT_TYPE } from '../audit';
import { getImplicitEntityFields } from './helpers';
@ -81,7 +84,12 @@ export class AssetCriticalityDataClient {
logger: this.options.logger,
options: {
index: this.getIndex(),
mappings: mappingFromFieldMap(assetCriticalityFieldMap, 'strict'),
mappings: {
...mappingFromFieldMap(assetCriticalityFieldMap, 'strict'),
_meta: {
version: ASSET_CRITICALITY_MAPPINGS_VERSIONS,
},
},
},
});
}
@ -178,9 +186,12 @@ export class AssetCriticalityDataClient {
}
public getIndexMappings() {
return this.options.esClient.indices.getMapping({
index: this.getIndex(),
});
return this.options.esClient.indices.getMapping(
{
index: this.getIndex(),
},
{ ignore: [404] }
);
}
public async get(idParts: AssetCriticalityIdParts): Promise<AssetCriticalityRecord | undefined> {

30
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_migration_client.test.ts
View file

@ -5,10 +5,11 @@
* 2.0.
*/
import { AssetCriticalityEcsMigrationClient } from './asset_criticality_migration_client';
import { AssetCriticalityMigrationClient } from './asset_criticality_migration_client';
import { AssetCriticalityDataClient } from './asset_criticality_data_client';
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import type { AuditLogger } from '@kbn/security-plugin-types-server';
import { ASSET_CRITICALITY_MAPPINGS_VERSIONS } from './constants';
jest.mock('./asset_criticality_data_client');
@ -19,12 +20,12 @@ const emptySearchResponse = {
hits: { hits: [] },
};
describe('AssetCriticalityEcsMigrationClient', () => {
describe('AssetCriticalityMigrationClient', () => {
let logger: Logger;
let auditLogger: AuditLogger | undefined;
let esClient: ElasticsearchClient;
let assetCriticalityDataClient: jest.Mocked<AssetCriticalityDataClient>;
let migrationClient: AssetCriticalityEcsMigrationClient;
let migrationClient: AssetCriticalityMigrationClient;
beforeEach(() => {
logger = { info: jest.fn(), error: jest.fn() } as unknown as Logger;
@ -39,27 +40,28 @@ describe('AssetCriticalityEcsMigrationClient', () => {
(AssetCriticalityDataClient as jest.Mock).mockImplementation(() => assetCriticalityDataClient);
migrationClient = new AssetCriticalityEcsMigrationClient({ logger, auditLogger, esClient });
migrationClient = new AssetCriticalityMigrationClient({ logger, auditLogger, esClient });
});
describe('isEcsMappingsMigrationRequired', () => {
it('should return true if any index mappings do not have asset property', async () => {
describe('isMappingsMigrationRequired', () => {
it('should return true if versions are different', async () => {
assetCriticalityDataClient.getIndexMappings.mockResolvedValue({
index1: { mappings: { properties: {} } },
index2: { mappings: { properties: { asset: {} } } },
index2: { mappings: { properties: {}, _meta: { version: '9999' } } },
});
const result = await migrationClient.isEcsMappingsMigrationRequired();
const result = await migrationClient.isMappingsMigrationRequired();
expect(result).toBe(true);
});
it('should return false if all index mappings have asset property', async () => {
it('should return false if versions are equal', async () => {
assetCriticalityDataClient.getIndexMappings.mockResolvedValue({
index1: { mappings: { properties: { asset: {} } } },
index2: { mappings: { properties: { asset: {} } } },
index1: {
mappings: { properties: {}, _meta: { version: ASSET_CRITICALITY_MAPPINGS_VERSIONS } },
},
});
const result = await migrationClient.isEcsMappingsMigrationRequired();
const result = await migrationClient.isMappingsMigrationRequired();
expect(result).toBe(false);
});
});
@ -83,9 +85,9 @@ describe('AssetCriticalityEcsMigrationClient', () => {
});
});
describe('migrateEcsMappings', () => {
describe('migrateMappings', () => {
it('should call createOrUpdateIndex on assetCriticalityDataClient', async () => {
await migrationClient.migrateEcsMappings();
await migrationClient.migrateMappings();
expect(assetCriticalityDataClient.createOrUpdateIndex).toHaveBeenCalled();
});
});

13
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_migration_client.ts
View file

@ -7,8 +7,9 @@
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import type { AuditLogger } from '@kbn/security-plugin-types-server';
import { AssetCriticalityDataClient } from './asset_criticality_data_client';
import { ASSET_CRITICALITY_MAPPINGS_VERSIONS } from './constants';
interface AssetCriticalityEcsMigrationClientOpts {
interface AssetCriticalityMigrationClientOpts {
logger: Logger;
auditLogger: AuditLogger | undefined;
esClient: ElasticsearchClient;
@ -40,20 +41,20 @@ if (ctx._source.id_field == 'user.name') {
ctx._source.host = host;
}`;
export class AssetCriticalityEcsMigrationClient {
export class AssetCriticalityMigrationClient {
private readonly assetCriticalityDataClient: AssetCriticalityDataClient;
constructor(private readonly options: AssetCriticalityEcsMigrationClientOpts) {
constructor(private readonly options: AssetCriticalityMigrationClientOpts) {
this.assetCriticalityDataClient = new AssetCriticalityDataClient({
...options,
namespace: '*', // The migration is applied to all spaces
});
}
public isEcsMappingsMigrationRequired = async () => {
public isMappingsMigrationRequired = async () => {
const indicesMappings = await this.assetCriticalityDataClient.getIndexMappings();
return Object.values(indicesMappings).some(
({ mappings }) => mappings?.properties?.asset === undefined
({ mappings }) => mappings._meta?.version !== ASSET_CRITICALITY_MAPPINGS_VERSIONS
);
};
@ -66,7 +67,7 @@ export class AssetCriticalityEcsMigrationClient {
return resp.hits.hits.length > 0;
};
public migrateEcsMappings = () => {
public migrateMappings = () => {
return this.assetCriticalityDataClient.createOrUpdateIndex();
};

1
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/check_and_init_asset_criticality_resources.ts
View file

@ -30,7 +30,6 @@ export const checkAndInitAssetCriticalityResources = async (
});
const doesIndexExist = await assetCriticalityDataClient.doesIndexExist();
if (!doesIndexExist) {
logger.info('Asset criticality resources are not installed, initialising...');
await assetCriticalityDataClient.init();

78
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.test.ts Normal file
View file

@ -0,0 +1,78 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ASSET_CRITICALITY_MAPPINGS_VERSIONS, assetCriticalityFieldMap } from './constants';
describe('asset criticality - constants', () => {
it("please bump 'ASSET_CRITICALITY_MAPPINGS_VERSIONS' when mappings change", () => {
expect(ASSET_CRITICALITY_MAPPINGS_VERSIONS).toEqual(2);
expect(assetCriticalityFieldMap).toMatchInlineSnapshot(`
Object {
"@timestamp": Object {
"array": false,
"required": false,
"type": "date",
},
"asset.criticality": Object {
"array": false,
"required": true,
"type": "keyword",
},
"criticality_level": Object {
"array": false,
"required": false,
"type": "keyword",
},
"host.asset.criticality": Object {
"array": false,
"required": false,
"type": "keyword",
},
"host.name": Object {
"array": false,
"required": false,
"type": "keyword",
},
"id_field": Object {
"array": false,
"required": false,
"type": "keyword",
},
"id_value": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.asset.criticality": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.name": Object {
"array": false,
"required": false,
"type": "keyword",
},
"updated_at": Object {
"array": false,
"required": false,
"type": "date",
},
"user.asset.criticality": Object {
"array": false,
"required": false,
"type": "keyword",
},
"user.name": Object {
"array": false,
"required": false,
"type": "keyword",
},
}
`);
});
});

9
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/constants.ts
View file

@ -16,6 +16,9 @@ const assetCriticalityMapping = {
required: false,
};
// Upgrade this value to force a mappings update on the next Kibana startup
export const ASSET_CRITICALITY_MAPPINGS_VERSIONS = 2;
export const assetCriticalityFieldMap: FieldMap = {
'@timestamp': {
type: 'date',
@ -55,6 +58,12 @@ export const assetCriticalityFieldMap: FieldMap = {
required: false,
},
'user.asset.criticality': assetCriticalityMapping,
'service.name': {
type: 'keyword',
array: false,
required: false,
},
'service.asset.criticality': assetCriticalityMapping,
} as const;
export const CRITICALITY_VALUES: { readonly [K in CriticalityValues as Uppercase<K>]: K } = {

41
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.test.ts
View file

@ -4,8 +4,8 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { applyCriticalityToScore } from './helpers';
import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
import { applyCriticalityToScore, getMappingForFlattenedField } from './helpers';
describe('applyCriticalityToScore', () => {
describe('integer scores', () => {
@ -60,4 +60,41 @@ describe('applyCriticalityToScore', () => {
expect(result).toEqual(99.9993992827436);
});
});
describe('getMappingForFlattenedField', () => {
const mappingProperty: MappingProperty = { type: 'keyword' };
const mapping: Record<string, MappingProperty> = {
user: {
properties: {
asset: {
properties: {
criticality: mappingProperty,
},
},
},
},
name: mappingProperty,
};
it('returns the correct mapping for a simple field', () => {
const field = 'name';
const result = getMappingForFlattenedField(field, mapping);
expect(result).toEqual(mappingProperty);
});
it('returns the correct mapping for a nested field', () => {
const field = 'user.asset.criticality';
const result = getMappingForFlattenedField(field, mapping);
expect(result).toEqual(mappingProperty);
});
it('returns undefined for a non-existent field', () => {
const field = 'user.asset.nonExistentField';
const result = getMappingForFlattenedField(field, mapping);
expect(result).toBeUndefined();
});
});
});

24
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts
View file

@ -5,6 +5,8 @@
* 2.0.
*/
import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
import { get } from 'lodash/fp';
import { CriticalityModifiers } from '../../../../common/entity_analytics/asset_criticality';
import type {
AssetCriticalityUpsert,
@ -76,8 +78,14 @@ type AssetCriticalityUpsertWithDeleted = {
: AssetCriticalityUpsert[K];
};
const entityTypeByIdField = {
'host.name': 'host',
'user.name': 'user',
'service.name': 'service',
} as const;
export const getImplicitEntityFields = (record: AssetCriticalityUpsertWithDeleted) => {
const entityType = record.idField === 'host.name' ? 'host' : 'user';
const entityType = entityTypeByIdField[record.idField];
return {
[entityType]: {
asset: { criticality: record.criticalityLevel },
@ -85,3 +93,17 @@ export const getImplicitEntityFields = (record: AssetCriticalityUpsertWithDelete
},
};
};
/**
* Finds the mapping for a flatten field name
*
* @example
* const field = `user.asset.criticality`
* const mapping = {user: {properties: {asset: {properties: {criticality: {type: 'keyword'}}}}}};
* getMappingForFlattenedField(field, mapping) // returns {type: 'keyword'}
*
*/
export const getMappingForFlattenedField = (
field: string,
mapping: Record<string, MappingProperty>
) => get(field.replaceAll('.', '.properties.'), mapping);

41
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/migrations/schedule_ecs_compliancy_migration.test.ts
View file

@ -15,18 +15,14 @@ import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks';
import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
import { auditLoggerMock } from '@kbn/core-security-server-mocks';
const mockMigrateEcsMappings = jest.fn().mockResolvedValue(false);
const mockIsEcsMappingsMigrationRequired = jest.fn().mockResolvedValue(false);
const mockIsEcsDataMigrationRequired = jest.fn().mockResolvedValue(false);
const mockMigrateEcsData = jest.fn().mockResolvedValue({
updated: 100,
failures: [],
});
jest.mock('../asset_criticality_migration_client', () => ({
AssetCriticalityEcsMigrationClient: jest.fn().mockImplementation(() => ({
isEcsMappingsMigrationRequired: mockIsEcsMappingsMigrationRequired,
AssetCriticalityMigrationClient: jest.fn().mockImplementation(() => ({
isEcsDataMigrationRequired: mockIsEcsDataMigrationRequired,
migrateEcsMappings: mockMigrateEcsMappings,
migrateEcsData: mockMigrateEcsData,
})),
}));
@ -61,6 +57,7 @@ describe('scheduleAssetCriticalityEcsCompliancyMigration', () => {
taskManager,
logger,
getStartServices,
kibanaVersion: '8.0.0',
});
expect(taskManager.registerTaskDefinitions).toHaveBeenCalledWith({
@ -75,40 +72,11 @@ describe('scheduleAssetCriticalityEcsCompliancyMigration', () => {
taskManager: undefined,
logger,
getStartServices,
kibanaVersion: '8.0.0',
})
).resolves.not.toThrow();
});
it('should migrate mappings if required', async () => {
const taskManager = taskManagerMock.createSetup();
mockIsEcsMappingsMigrationRequired.mockResolvedValue(true);
await scheduleAssetCriticalityEcsCompliancyMigration({
auditLogger,
taskManager,
logger,
getStartServices,
});
expect(mockMigrateEcsMappings).toHaveBeenCalled();
});
it('should not migrate mappings if not required', async () => {
const taskManager = taskManagerMock.createSetup();
mockIsEcsMappingsMigrationRequired.mockResolvedValue(false);
await scheduleAssetCriticalityEcsCompliancyMigration({
auditLogger,
taskManager,
logger,
getStartServices,
});
expect(mockMigrateEcsMappings).not.toHaveBeenCalled();
});
it('should schedule the task if data migration is required', async () => {
mockIsEcsDataMigrationRequired.mockResolvedValue(true);
@ -117,6 +85,7 @@ describe('scheduleAssetCriticalityEcsCompliancyMigration', () => {
taskManager: taskManagerMock.createSetup(),
logger,
getStartServices,
kibanaVersion: '8.0.0',
});
expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalledWith(
@ -136,6 +105,7 @@ describe('scheduleAssetCriticalityEcsCompliancyMigration', () => {
taskManager: taskManagerMock.createSetup(),
logger,
getStartServices,
kibanaVersion: '8.0.0',
});
expect(logger.error).toHaveBeenCalledWith(
@ -151,6 +121,7 @@ describe('scheduleAssetCriticalityEcsCompliancyMigration', () => {
taskManager: taskManagerMock.createSetup(),
logger,
getStartServices,
kibanaVersion: '8.0.0',
});
expect(mockTaskManagerStart.ensureScheduled).not.toHaveBeenCalled();

12
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/migrations/schedule_ecs_compliancy_migration.ts
View file

@ -6,7 +6,7 @@
*/
import type { EntityAnalyticsMigrationsParams } from '../../migrations';
import { AssetCriticalityEcsMigrationClient } from '../asset_criticality_migration_client';
import { AssetCriticalityMigrationClient } from '../asset_criticality_migration_client';
const TASK_TYPE = 'security-solution-ea-asset-criticality-ecs-migration';
const TASK_ID = `${TASK_TYPE}-task-id`;
@ -37,18 +37,12 @@ export const scheduleAssetCriticalityEcsCompliancyMigration = async ({
const taskManagerStart = depsStart.taskManager;
const esClient = coreStart.elasticsearch.client.asInternalUser;
const migrationClient = new AssetCriticalityEcsMigrationClient({
const migrationClient = new AssetCriticalityMigrationClient({
esClient,
logger,
auditLogger,
});
const shouldMigrateMappings = await migrationClient.isEcsMappingsMigrationRequired();
if (shouldMigrateMappings) {
logger.debug('Migrating Asset Criticality mappings');
await migrationClient.migrateEcsMappings();
}
const shouldMigrateData = await migrationClient.isEcsDataMigrationRequired();
if (shouldMigrateData && taskManagerStart) {
logger.debug(`Task scheduled: "${TASK_TYPE}"`);
@ -83,7 +77,7 @@ export const createMigrationTask =
abortController = new AbortController();
const [coreStart] = await getStartServices();
const esClient = coreStart.elasticsearch.client.asInternalUser;
const migrationClient = new AssetCriticalityEcsMigrationClient({
const migrationClient = new AssetCriticalityMigrationClient({
esClient,
logger,
auditLogger,

58
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/migrations/update_asset_criticality_mappings.test.ts Normal file
View file

@ -0,0 +1,58 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { updateAssetCriticalityMappings } from './update_asset_criticality_mappings';
import type { Logger } from '@kbn/core/server';
const mockisMappingsMigrationRequired = jest.fn();
const mockmigrateMappings = jest.fn();
jest.mock('../asset_criticality_migration_client', () => ({
AssetCriticalityMigrationClient: jest.fn().mockImplementation(() => ({
isMappingsMigrationRequired: () => mockisMappingsMigrationRequired(),
migrateMappings: () => mockmigrateMappings(),
})),
}));
describe('updateAssetCriticalityMappings', () => {
const mockLogger = { info: jest.fn(), error: jest.fn() } as unknown as Logger;
const mockGetStartServices = jest
.fn()
.mockResolvedValue([{ elasticsearch: { client: { asInternalUser: {} } } }]);
beforeEach(() => {
jest.clearAllMocks();
});
it('should migrate mappings if migration is required', async () => {
mockisMappingsMigrationRequired.mockResolvedValue(true);
await updateAssetCriticalityMappings({
auditLogger: undefined,
logger: mockLogger,
getStartServices: mockGetStartServices,
kibanaVersion: '8.0.0',
});
expect(mockLogger.info).toHaveBeenCalledWith('Migrating Asset Criticality mappings');
expect(mockmigrateMappings).toHaveBeenCalled();
});
it('should not migrate mappings if migration is not required', async () => {
mockisMappingsMigrationRequired.mockResolvedValue(false);
await updateAssetCriticalityMappings({
auditLogger: undefined,
logger: mockLogger,
getStartServices: mockGetStartServices,
kibanaVersion: '8.0.0',
});
expect(mockisMappingsMigrationRequired).toHaveBeenCalled();
expect(mockLogger.info).not.toHaveBeenCalledWith('Migrating Asset Criticality mappings');
});
});

31
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/migrations/update_asset_criticality_mappings.ts Normal file
View file

@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { EntityAnalyticsMigrationsParams } from '../../migrations';
import { AssetCriticalityMigrationClient } from '../asset_criticality_migration_client';
export const updateAssetCriticalityMappings = async ({
auditLogger,
logger,
getStartServices,
}: EntityAnalyticsMigrationsParams) => {
const [coreStart] = await getStartServices();
const esClient = coreStart.elasticsearch.client.asInternalUser;
const migrationClient = new AssetCriticalityMigrationClient({
esClient,
logger,
auditLogger,
});
const shouldMigrateMappings = await migrationClient.isMappingsMigrationRequired();
if (shouldMigrateMappings) {
logger.info('Migrating Asset Criticality mappings');
await migrationClient.migrateMappings();
}
};

8
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
View file

@ -50,7 +50,7 @@ import type {
} from '../../../../common/api/entity_analytics';
import { EngineDescriptorClient } from './saved_object/engine_descriptor';
import { ENGINE_STATUS, ENTITY_STORE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
import { AssetCriticalityEcsMigrationClient } from '../asset_criticality/asset_criticality_migration_client';
import { AssetCriticalityMigrationClient } from '../asset_criticality/asset_criticality_migration_client';
import { getUnitedEntityDefinition } from './united_entity_definitions';
import {
startEntityStoreFieldRetentionEnrichTask,
@ -128,7 +128,7 @@ interface SearchEntitiesParams {
export class EntityStoreDataClient {
private engineClient: EngineDescriptorClient;
private assetCriticalityMigrationClient: AssetCriticalityEcsMigrationClient;
private assetCriticalityMigrationClient: AssetCriticalityMigrationClient;
private entityClient: EntityClient;
private riskScoreDataClient: RiskScoreDataClient;
private esClient: ElasticsearchClient;
@ -148,7 +148,7 @@ export class EntityStoreDataClient {
namespace,
});
this.assetCriticalityMigrationClient = new AssetCriticalityEcsMigrationClient({
this.assetCriticalityMigrationClient = new AssetCriticalityMigrationClient({
esClient: this.esClient,
logger,
auditLogger,
@ -288,7 +288,7 @@ export class EntityStoreDataClient {
const { config } = this.options;
await this.riskScoreDataClient.createRiskScoreLatestIndex().catch((e) => {
await this.riskScoreDataClient.createOrUpdateRiskScoreLatestIndex().catch((e) => {
if (e.meta.body.error.type === 'resource_already_exists_exception') {
this.options.logger.debug(
`Risk score index for ${entityType} already exists, skipping creation.`

6
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/task/field_retention_enrichment_task.ts
View file

@ -13,6 +13,7 @@ import type {
TaskManagerSetupContract,
TaskManagerStartContract,
} from '@kbn/task-manager-plugin/server';
import { getAvailableEntityTypes } from '../../../../../common/entity_analytics/entity_store/constants';
import {
EngineComponentResourceEnum,
type EntityType,
@ -24,10 +25,7 @@ import {
} from './state';
import { INTERVAL, SCOPE, TIMEOUT, TYPE, VERSION } from './constants';
import type { EntityAnalyticsRoutesDeps } from '../../types';
import {
getAvailableEntityTypes,
getUnitedEntityDefinitionVersion,
} from '../united_entity_definitions';
import { getUnitedEntityDefinitionVersion } from '../united_entity_definitions';
import { executeFieldRetentionEnrichPolicy } from '../elasticsearch_assets';
import { getEntitiesIndexName } from '../utils';

3
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts
View file

@ -67,6 +67,3 @@ const versionByEntityType: Record<EntityType, string> = {
export const getUnitedEntityDefinitionVersion = (entityType: EntityType): string =>
versionByEntityType[entityType];
export const getAvailableEntityTypes = (): EntityType[] =>
Object.keys(unitedDefinitionBuilders) as EntityType[];

14
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts
View file

@ -11,23 +11,15 @@ import {
entitiesIndexPattern,
} from '@kbn/entities-schema';
import type { DataViewsService, DataView } from '@kbn/data-views-plugin/common';
import { IDENTITY_FIELD_MAP } from '../../../../../common/entity_analytics/entity_store/constants';
import type { AppClient } from '../../../../types';
import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine';
import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality';
import {
EntityTypeEnum,
type EntityType,
} from '../../../../../common/api/entity_analytics/entity_store/common.gen';
import { type EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
import { entityEngineDescriptorTypeName } from '../saved_object';
const identityFieldMap: Record<EntityType, string> = {
[EntityTypeEnum.host]: 'host.name',
[EntityTypeEnum.user]: 'user.name',
[EntityTypeEnum.service]: 'service.name',
};
export const getIdentityFieldForEntityType = (entityType: EntityType) => {
return identityFieldMap[entityType];
return IDENTITY_FIELD_MAP[entityType];
};
export const buildIndexPatterns = async (

21
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/migrations/index.ts
View file

@ -9,16 +9,37 @@ import type { AuditLogger, Logger, StartServicesAccessor } from '@kbn/core/serve
import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server';
import type { StartPlugins } from '../../../plugin';
import { scheduleAssetCriticalityEcsCompliancyMigration } from '../asset_criticality/migrations/schedule_ecs_compliancy_migration';
import { updateAssetCriticalityMappings } from '../asset_criticality/migrations/update_asset_criticality_mappings';
import { updateRiskScoreMappings } from '../risk_engine/migrations/update_risk_score_mappings';
export interface EntityAnalyticsMigrationsParams {
taskManager?: TaskManagerSetupContract;
logger: Logger;
getStartServices: StartServicesAccessor<StartPlugins>;
auditLogger: AuditLogger | undefined;
kibanaVersion: string;
}
/**
* ### How to add a new field to the risk score index and template mappings?
* - Update the mapping object [here](../risk_score/configurations.ts)
* - Pump the `mappingsVersion` version [here](../risk_engine/utils/saved_object_configuration.ts)
*
* ### How to add a new field to the asset criticality index?
* - Update the mapping object [here](../asset_criticality/constants.ts)
* - Pump the `ASSET_CRITICALITY_MAPPINGS_VERSIONS` version [here](../asset_criticality/constants.ts)
*
* ### How to update the risk score transform config?
* - Update the transform config [here](../risk_score/configurations.ts)
* - Pump the `version` [here](../risk_score/configurations.ts)
*
* note: If you change the `latest` property, the transform will reinstall after the engine task runs.
*/
export const scheduleEntityAnalyticsMigration = async (params: EntityAnalyticsMigrationsParams) => {
const scopedLogger = params.logger.get('entityAnalytics.migration');
await updateAssetCriticalityMappings({ ...params, logger: scopedLogger });
await scheduleAssetCriticalityEcsCompliancyMigration({ ...params, logger: scopedLogger });
await updateRiskScoreMappings({ ...params, logger: scopedLogger });
};

164
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/migrations/update_risk_score_mappings.test.ts Normal file
View file

@ -0,0 +1,164 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { updateRiskScoreMappings } from './update_risk_score_mappings';
import { coreMock, loggingSystemMock, savedObjectsClientMock } from '@kbn/core/server/mocks';
import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
import { auditLoggerMock } from '@kbn/core-security-server-mocks';
const mockCreateOrUpdateComponentTemplate = jest.fn();
const mockCreateOrUpdateIndex = jest.fn();
const mockUpdateUnderlyingMapping = jest.fn();
jest.mock('@kbn/alerting-plugin/server', () => ({
createOrUpdateComponentTemplate: (...params: unknown[]) =>
mockCreateOrUpdateComponentTemplate(...params),
}));
jest.mock('../../utils/create_or_update_index', () => ({
createOrUpdateIndex: (...params: unknown[]) => mockCreateOrUpdateIndex(...params),
}));
jest.mock('../../utils/create_datastream', () => ({
updateUnderlyingMapping: (...params: unknown[]) => mockUpdateUnderlyingMapping(...params),
}));
const mockGetDefaultRiskEngineConfiguration = jest.fn();
const mockUpdateSavedObjectAttribute = jest.fn();
jest.mock('../utils/saved_object_configuration', () => ({
...jest.requireActual('../utils/saved_object_configuration'),
getDefaultRiskEngineConfiguration: () => mockGetDefaultRiskEngineConfiguration(),
updateSavedObjectAttribute: (...params: unknown[]) => mockUpdateSavedObjectAttribute(...params),
}));
describe('updateRiskScoreMappings', () => {
const logger = loggingSystemMock.createLogger();
const mockAuditLogger = auditLoggerMock.create();
const coreStart = coreMock.createStart();
const soClient = savedObjectsClientMock.create();
const esClient = elasticsearchServiceMock.createElasticsearchClient();
const getStartServicesMock = jest.fn().mockReturnValue([
{
...coreStart,
savedObjects: {
createInternalRepository: jest.fn().mockReturnValue(soClient),
getScopedClient: jest.fn().mockReturnValue(soClient),
},
elasticsearch: { client: { asInternalUser: esClient } },
},
]);
const kibanaVersion = '8.0.0';
const buildSavedObject = (attributes = {}) => ({
namespaces: ['default'],
attributes,
id: 'id',
type: 'type',
references: [],
score: 1,
});
const mockSavedObjectsResponseDefaults = {
total: 1,
page: 1,
per_page: 10,
saved_objects: [],
};
beforeEach(() => {
jest.clearAllMocks();
});
it('should update risk score mappings when versions are different', async () => {
const mockSavedObjectsResponse = {
...mockSavedObjectsResponseDefaults,
saved_objects: [buildSavedObject({ _meta: { mappingsVersion: '1.0.0' } })],
};
soClient.find.mockResolvedValue(mockSavedObjectsResponse);
const newConfig = { _meta: { mappingsVersion: '2.0.0' } };
mockGetDefaultRiskEngineConfiguration.mockResolvedValue(newConfig);
await updateRiskScoreMappings({
auditLogger: mockAuditLogger,
logger,
kibanaVersion,
getStartServices: getStartServicesMock,
});
expect(mockCreateOrUpdateIndex).toHaveBeenCalledWith(
expect.objectContaining({
options: expect.objectContaining({ index: 'risk-score.risk-score-latest-default' }),
})
);
expect(mockCreateOrUpdateComponentTemplate).toHaveBeenCalledWith(
expect.objectContaining({
template: expect.objectContaining({ name: '.risk-score-mappings-default' }),
})
);
expect(mockUpdateUnderlyingMapping).toHaveBeenCalledWith(
expect.objectContaining({ index: 'risk-score.risk-score-default' })
);
expect(mockUpdateSavedObjectAttribute).toHaveBeenCalledWith(
expect.objectContaining({ attributes: { _meta: { mappingsVersion: '2.0.0' } } })
);
});
it('should not update risk score mappings when versions are the same', async () => {
const savedObjectsResponse = {
...mockSavedObjectsResponseDefaults,
saved_objects: [buildSavedObject({ _meta: { mappingsVersion: '2.0.0' } })],
};
soClient.find.mockResolvedValue(savedObjectsResponse);
const newConfig = { _meta: { mappingsVersion: '2.0.0' } };
mockGetDefaultRiskEngineConfiguration.mockResolvedValue(newConfig);
await updateRiskScoreMappings({
auditLogger: mockAuditLogger,
logger,
getStartServices: getStartServicesMock,
kibanaVersion,
});
expect(mockCreateOrUpdateIndex).not.toHaveBeenCalled();
expect(mockCreateOrUpdateComponentTemplate).not.toHaveBeenCalled();
expect(mockUpdateUnderlyingMapping).not.toHaveBeenCalled();
expect(mockUpdateSavedObjectAttribute).not.toHaveBeenCalled();
});
it('should update risk score mappings for every space when versions are different', async () => {
const mockSavedObjectsResponse = {
...mockSavedObjectsResponseDefaults,
saved_objects: [
buildSavedObject({ _meta: { mappingsVersion: '1.0.0' } }),
buildSavedObject({ _meta: { mappingsVersion: '1.0.0' } }),
buildSavedObject({ _meta: { mappingsVersion: '1.0.0' } }),
buildSavedObject({ _meta: { mappingsVersion: '1.0.0' } }),
],
};
soClient.find.mockResolvedValue(mockSavedObjectsResponse);
const newConfig = { _meta: { mappingsVersion: '2.0.0' } };
mockGetDefaultRiskEngineConfiguration.mockResolvedValue(newConfig);
await updateRiskScoreMappings({
auditLogger: mockAuditLogger,
logger,
kibanaVersion,
getStartServices: getStartServicesMock,
});
expect(mockCreateOrUpdateIndex).toHaveBeenCalledTimes(4);
expect(mockCreateOrUpdateComponentTemplate).toHaveBeenCalledTimes(4);
expect(mockUpdateUnderlyingMapping).toHaveBeenCalledTimes(4);
expect(mockUpdateSavedObjectAttribute).toHaveBeenCalledTimes(4);
});
});

84
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/migrations/update_risk_score_mappings.ts Normal file
View file

@ -0,0 +1,84 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { asyncForEach } from '@kbn/std';
import { first } from 'lodash/fp';
import type { EntityAnalyticsMigrationsParams } from '../../migrations';
import { RiskEngineDataClient } from '../risk_engine_data_client';
import { getDefaultRiskEngineConfiguration } from '../utils/saved_object_configuration';
import { RiskScoreDataClient } from '../../risk_score/risk_score_data_client';
import type { RiskEngineConfiguration } from '../../types';
import { riskEngineConfigurationTypeName } from '../saved_object';
import { buildScopedInternalSavedObjectsClientUnsafe } from '../../risk_score/tasks/helpers';
export const MAX_PER_PAGE = 10_000;
export const updateRiskScoreMappings = async ({
auditLogger,
logger,
getStartServices,
kibanaVersion,
}: EntityAnalyticsMigrationsParams) => {
const [coreStart] = await getStartServices();
const soClientKibanaUser = coreStart.savedObjects.createInternalRepository();
// Get all installed Risk Engine Configurations
const savedObjectsResponse = await soClientKibanaUser.find<RiskEngineConfiguration>({
type: riskEngineConfigurationTypeName,
perPage: MAX_PER_PAGE,
namespaces: ['*'],
});
await asyncForEach(savedObjectsResponse.saved_objects, async (savedObject) => {
const namespace = first(savedObject.namespaces); // We install one Risk Engine Configuration object per space
if (!namespace) {
logger.error('Unexpected saved object. Risk Score saved objects must have a namespace');
return;
}
const newConfig = await getDefaultRiskEngineConfiguration({ namespace });
if (savedObject.attributes._meta?.mappingsVersion !== newConfig._meta.mappingsVersion) {
logger.info(
`Starting Risk Score mappings update from version ${savedObject.attributes._meta?.mappingsVersion} to version ${newConfig._meta.mappingsVersion} on namespace ${namespace}`
);
const esClient = coreStart.elasticsearch.client.asInternalUser;
const soClient = buildScopedInternalSavedObjectsClientUnsafe({ coreStart, namespace });
const riskEngineDataClient = new RiskEngineDataClient({
logger,
kibanaVersion,
esClient,
namespace,
soClient,
auditLogger,
});
const riskScoreDataClient = new RiskScoreDataClient({
logger,
kibanaVersion,
esClient,
namespace,
soClient,
auditLogger,
});
await riskScoreDataClient.createOrUpdateRiskScoreLatestIndex();
await riskScoreDataClient.createOrUpdateRiskScoreIndexTemplate();
await riskScoreDataClient.updateRiskScoreTimeSeriesIndexMappings();
await riskEngineDataClient.updateConfiguration({
_meta: {
mappingsVersion: newConfig._meta.mappingsVersion,
},
});
logger.debug(
`Risk score mappings updated to version ${newConfig._meta.mappingsVersion} on namespace ${namespace}`
);
}
});
};

13
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/risk_engine_data_client.ts
View file

@ -9,8 +9,10 @@ import type { Logger, ElasticsearchClient, SavedObjectsClientContract } from '@k
import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
import type { AuditLogger } from '@kbn/security-plugin-types-server';
import { RiskEngineStatusEnum } from '../../../../common/api/entity_analytics';
import type { InitRiskEngineResult } from '../../../../common/entity_analytics/risk_engine';
import { RiskScoreEntity } from '../../../../common/entity_analytics/risk_engine';
import {
RiskScoreEntity,
type InitRiskEngineResult,
} from '../../../../common/entity_analytics/risk_engine';
import { removeLegacyTransforms, getLegacyTransforms } from '../utils/transforms';
import {
updateSavedObjectAttribute,
@ -24,6 +26,7 @@ import { removeRiskScoringTask, startRiskScoringTask } from '../risk_score/tasks
import { RiskEngineAuditActions } from './audit';
import { AUDIT_CATEGORY, AUDIT_OUTCOME, AUDIT_TYPE } from '../audit';
import { getRiskScoringTaskStatus, scheduleNow } from '../risk_score/tasks/risk_scoring_task';
import type { RiskEngineConfiguration } from '../types';
interface InitOpts {
namespace: string;
@ -112,6 +115,12 @@ export class RiskEngineDataClient {
savedObjectsClient: this.options.soClient,
});
public updateConfiguration = (config: Partial<RiskEngineConfiguration>) =>
updateSavedObjectAttribute({
savedObjectsClient: this.options.soClient,
attributes: config,
});
public async getStatus({
namespace,
taskManager,

282
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.test.ts Normal file
View file

@ -0,0 +1,282 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { riskScoreFieldMap } from '../../risk_score/configurations';
import { getDefaultRiskEngineConfiguration } from './saved_object_configuration';
describe('#getDefaultRiskEngineConfiguration', () => {
it("please bump 'mappingsVersion' when mappings change", () => {
const namespace = 'default';
const config = getDefaultRiskEngineConfiguration({ namespace });
expect(config._meta.mappingsVersion).toEqual(2);
expect(riskScoreFieldMap).toMatchInlineSnapshot(`
Object {
"@timestamp": Object {
"array": false,
"required": false,
"type": "date",
},
"host.name": Object {
"array": false,
"required": false,
"type": "keyword",
},
"host.risk": Object {
"array": false,
"required": false,
"type": "object",
},
"host.risk.calculated_level": Object {
"array": false,
"required": false,
"type": "keyword",
},
"host.risk.calculated_score": Object {
"array": false,
"required": false,
"type": "float",
},
"host.risk.calculated_score_norm": Object {
"array": false,
"required": false,
"type": "float",
},
"host.risk.category_1_count": Object {
"array": false,
"required": false,
"type": "long",
},
"host.risk.category_1_score": Object {
"array": false,
"required": false,
"type": "float",
},
"host.risk.id_field": Object {
"array": false,
"required": false,
"type": "keyword",
},
"host.risk.id_value": Object {
"array": false,
"required": false,
"type": "keyword",
},
"host.risk.inputs": Object {
"array": true,
"required": false,
"type": "object",
},
"host.risk.inputs.category": Object {
"array": false,
"required": false,
"type": "keyword",
},
"host.risk.inputs.description": Object {
"array": false,
"required": false,
"type": "keyword",
},
"host.risk.inputs.id": Object {
"array": false,
"required": false,
"type": "keyword",
},
"host.risk.inputs.index": Object {
"array": false,
"required": false,
"type": "keyword",
},
"host.risk.inputs.risk_score": Object {
"array": false,
"required": false,
"type": "float",
},
"host.risk.inputs.timestamp": Object {
"array": false,
"required": false,
"type": "date",
},
"host.risk.notes": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.name": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.risk": Object {
"array": false,
"required": false,
"type": "object",
},
"service.risk.calculated_level": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.risk.calculated_score": Object {
"array": false,
"required": false,
"type": "float",
},
"service.risk.calculated_score_norm": Object {
"array": false,
"required": false,
"type": "float",
},
"service.risk.category_1_count": Object {
"array": false,
"required": false,
"type": "long",
},
"service.risk.category_1_score": Object {
"array": false,
"required": false,
"type": "float",
},
"service.risk.id_field": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.risk.id_value": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.risk.inputs": Object {
"array": true,
"required": false,
"type": "object",
},
"service.risk.inputs.category": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.risk.inputs.description": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.risk.inputs.id": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.risk.inputs.index": Object {
"array": false,
"required": false,
"type": "keyword",
},
"service.risk.inputs.risk_score": Object {
"array": false,
"required": false,
"type": "float",
},
"service.risk.inputs.timestamp": Object {
"array": false,
"required": false,
"type": "date",
},
"service.risk.notes": Object {
"array": false,
"required": false,
"type": "keyword",
},
"user.name": Object {
"array": false,
"required": false,
"type": "keyword",
},
"user.risk": Object {
"array": false,
"required": false,
"type": "object",
},
"user.risk.calculated_level": Object {
"array": false,
"required": false,
"type": "keyword",
},
"user.risk.calculated_score": Object {
"array": false,
"required": false,
"type": "float",
},
"user.risk.calculated_score_norm": Object {
"array": false,
"required": false,
"type": "float",
},
"user.risk.category_1_count": Object {
"array": false,
"required": false,
"type": "long",
},
"user.risk.category_1_score": Object {
"array": false,
"required": false,
"type": "float",
},
"user.risk.id_field": Object {
"array": false,
"required": false,
"type": "keyword",
},
"user.risk.id_value": Object {
"array": false,
"required": false,
"type": "keyword",
},
"user.risk.inputs": Object {
"array": true,
"required": false,
"type": "object",
},
"user.risk.inputs.category": Object {
"array": false,
"required": false,
"type": "keyword",
},
"user.risk.inputs.description": Object {
"array": false,
"required": false,
"type": "keyword",
},
"user.risk.inputs.id": Object {
"array": false,
"required": false,
"type": "keyword",
},
"user.risk.inputs.index": Object {
"array": false,
"required": false,
"type": "keyword",
},
"user.risk.inputs.risk_score": Object {
"array": false,
"required": false,
"type": "float",
},
"user.risk.inputs.timestamp": Object {
"array": false,
"required": false,
"type": "date",
},
"user.risk.notes": Object {
"array": false,
"required": false,
"type": "keyword",
},
}
`);
});
});

13
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_engine/utils/saved_object_configuration.ts
View file

@ -14,7 +14,7 @@ export interface SavedObjectsClientArg {
savedObjectsClient: SavedObjectsClientContract;
}
const getDefaultRiskEngineConfiguration = ({
export const getDefaultRiskEngineConfiguration = ({
namespace,
}: {
namespace: string;
@ -26,6 +26,10 @@ const getDefaultRiskEngineConfiguration = ({
interval: '1h',
pageSize: 3_500,
range: { start: 'now-30d', end: 'now' },
_meta: {
// Upgrade this property when changing mappings
mappingsVersion: 2,
},
});
const getConfigurationSavedObject = async ({
@ -41,12 +45,7 @@ export const updateSavedObjectAttribute = async ({
savedObjectsClient,
attributes,
}: SavedObjectsClientArg & {
attributes: {
enabled?: boolean;
excludeAlertIds?: string[];
range?: { start: string; end: string };
excludeAlertTags?: string[];
};
attributes: Partial<RiskEngineConfiguration>;
}) => {
const savedObjectConfiguration = await getConfigurationSavedObject({
savedObjectsClient,

193
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/__snapshots__/risk_score_data_client.test.ts.snap generated Normal file
View file

@ -0,0 +1,193 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`RiskScoreDataClient init success should initialize risk engine resources in the appropriate space 1`] = `
Object {
"mappings": Object {
"dynamic": "strict",
"properties": Object {
"@timestamp": Object {
"ignore_malformed": false,
"type": "date",
},
"host": Object {
"properties": Object {
"name": Object {
"type": "keyword",
},
"risk": Object {
"properties": Object {
"calculated_level": Object {
"type": "keyword",
},
"calculated_score": Object {
"type": "float",
},
"calculated_score_norm": Object {
"type": "float",
},
"category_1_count": Object {
"type": "long",
},
"category_1_score": Object {
"type": "float",
},
"id_field": Object {
"type": "keyword",
},
"id_value": Object {
"type": "keyword",
},
"inputs": Object {
"properties": Object {
"category": Object {
"type": "keyword",
},
"description": Object {
"type": "keyword",
},
"id": Object {
"type": "keyword",
},
"index": Object {
"type": "keyword",
},
"risk_score": Object {
"type": "float",
},
"timestamp": Object {
"type": "date",
},
},
"type": "object",
},
"notes": Object {
"type": "keyword",
},
},
"type": "object",
},
},
},
"service": Object {
"properties": Object {
"name": Object {
"type": "keyword",
},
"risk": Object {
"properties": Object {
"calculated_level": Object {
"type": "keyword",
},
"calculated_score": Object {
"type": "float",
},
"calculated_score_norm": Object {
"type": "float",
},
"category_1_count": Object {
"type": "long",
},
"category_1_score": Object {
"type": "float",
},
"id_field": Object {
"type": "keyword",
},
"id_value": Object {
"type": "keyword",
},
"inputs": Object {
"properties": Object {
"category": Object {
"type": "keyword",
},
"description": Object {
"type": "keyword",
},
"id": Object {
"type": "keyword",
},
"index": Object {
"type": "keyword",
},
"risk_score": Object {
"type": "float",
},
"timestamp": Object {
"type": "date",
},
},
"type": "object",
},
"notes": Object {
"type": "keyword",
},
},
"type": "object",
},
},
},
"user": Object {
"properties": Object {
"name": Object {
"type": "keyword",
},
"risk": Object {
"properties": Object {
"calculated_level": Object {
"type": "keyword",
},
"calculated_score": Object {
"type": "float",
},
"calculated_score_norm": Object {
"type": "float",
},
"category_1_count": Object {
"type": "long",
},
"category_1_score": Object {
"type": "float",
},
"id_field": Object {
"type": "keyword",
},
"id_value": Object {
"type": "keyword",
},
"inputs": Object {
"properties": Object {
"category": Object {
"type": "keyword",
},
"description": Object {
"type": "keyword",
},
"id": Object {
"type": "keyword",
},
"index": Object {
"type": "keyword",
},
"risk_score": Object {
"type": "float",
},
"timestamp": Object {
"type": "date",
},
},
"type": "object",
},
"notes": Object {
"type": "keyword",
},
},
"type": "object",
},
},
},
},
},
"settings": Object {},
}
`;

3
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.test.ts
View file

@ -14,8 +14,10 @@ import { calculateRiskScores } from './calculate_risk_scores';
import { calculateRiskScoresMock } from './calculate_risk_scores.mock';
import { riskScoreDataClientMock } from './risk_score_data_client.mock';
import type { RiskScoreDataClient } from './risk_score_data_client';
import type { ExperimentalFeatures } from '../../../../common';
jest.mock('./calculate_risk_scores');
const mockExperimentalFeatures = {} as ExperimentalFeatures;
const calculateAndPersistRecentHostRiskScores = (
esClient: ElasticsearchClient,
@ -34,6 +36,7 @@ const calculateAndPersistRecentHostRiskScores = (
riskScoreDataClient,
assetCriticalityService: assetCriticalityServiceMock.create(),
runtimeMappings: {},
experimentalFeatures: mockExperimentalFeatures,
});
};

4
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_and_persist_risk_scores.ts
View file

@ -7,6 +7,7 @@
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
import type { ExperimentalFeatures } from '../../../../common';
import type { RiskScoresCalculationResponse } from '../../../../common/api/entity_analytics';
import type { RiskScoreDataClient } from './risk_score_data_client';
import type { AssetCriticalityService } from '../asset_criticality/asset_criticality_service';
@ -20,6 +21,7 @@ export const calculateAndPersistRiskScores = async (
logger: Logger;
spaceId: string;
riskScoreDataClient: RiskScoreDataClient;
experimentalFeatures: ExperimentalFeatures;
}
): Promise<RiskScoresCalculationResponse> => {
const { riskScoreDataClient, spaceId, returnScores, refresh, ...rest } = params;
@ -29,7 +31,7 @@ export const calculateAndPersistRiskScores = async (
});
const { after_keys: afterKeys, scores } = await calculateRiskScores(rest);
if (!scores.host?.length && !scores.user?.length) {
if (!scores.host?.length && !scores.user?.length && !scores.service?.length) {
return { after_keys: {}, errors: [], scores_written: 0 };
}

7
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.mock.ts
View file

@ -54,6 +54,13 @@ const buildAggregationResponseMock = (
after_key: { 'user.name': 'username' },
buckets: [buildRiskScoreBucketMock(), buildRiskScoreBucketMock()],
},
service: {
after_key: { 'service.name': 'service_name' },
buckets: [
buildRiskScoreBucketMock({ key: { 'service.name': 'serviceName' } }),
buildRiskScoreBucketMock({ key: { 'service.name': 'serviceName' } }),
],
},
...overrides,
});

27
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.test.ts
View file

@ -13,6 +13,7 @@ import { calculateRiskScores } from './calculate_risk_scores';
import { calculateRiskScoresMock } from './calculate_risk_scores.mock';
import { ALERT_WORKFLOW_STATUS } from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names';
import { mockGlobalState } from '../../../../public/common/mock';
describe('calculateRiskScores()', () => {
let params: Parameters<typeof calculateRiskScores>[0];
@ -31,6 +32,7 @@ describe('calculateRiskScores()', () => {
pageSize: 500,
range: { start: 'now - 15d', end: 'now' },
runtimeMappings: {},
experimentalFeatures: mockGlobalState.app.enableExperimental,
};
});
@ -198,6 +200,31 @@ describe('calculateRiskScores()', () => {
expect(response).toHaveProperty('scores');
expect(response.scores.host).toHaveLength(2);
expect(response.scores.user).toHaveLength(2);
expect(response.scores.service).toHaveLength(0);
});
it('calculates risk score for service when the experimental flag is enabled', async () => {
const response = await calculateRiskScores({
...params,
experimentalFeatures: {
...mockGlobalState.app.enableExperimental,
serviceEntityStoreEnabled: true,
},
});
expect(response.scores.service).toHaveLength(2);
});
it('does NOT calculates risk score for service when the experimental flag is disabled', async () => {
const response = await calculateRiskScores({
...params,
experimentalFeatures: {
...mockGlobalState.app.enableExperimental,
serviceEntityStoreEnabled: false,
},
});
expect(response.scores.service).toHaveLength(0);
});
it('returns scores in the expected format', async () => {

23
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.ts
View file

@ -16,6 +16,7 @@ import {
ALERT_WORKFLOW_STATUS,
ALERT_WORKFLOW_TAGS,
} from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names';
import type { ExperimentalFeatures } from '../../../../common';
import type {
AssetCriticalityRecord,
RiskScoresPreviewResponse,
@ -220,11 +221,13 @@ export const calculateRiskScores = async ({
weights,
alertSampleSizePerShard = 10_000,
excludeAlertStatuses = [],
experimentalFeatures,
excludeAlertTags = [],
}: {
assetCriticalityService: AssetCriticalityService;
esClient: ElasticsearchClient;
logger: Logger;
experimentalFeatures: ExperimentalFeatures;
} & CalculateScoresParams): Promise<RiskScoresPreviewResponse> =>
withSecuritySpan('calculateRiskScores', async () => {
const now = new Date().toISOString();
@ -243,7 +246,12 @@ export const calculateRiskScores = async ({
bool: { must_not: { terms: { [ALERT_WORKFLOW_TAGS]: excludeAlertTags } } },
});
}
const identifierTypes: IdentifierType[] = identifierType ? [identifierType] : ['host', 'user'];
const identifierTypes: IdentifierType[] = identifierType
? [identifierType]
: experimentalFeatures.serviceEntityStoreEnabled
? ['host', 'user', 'service']
: ['host', 'user'];
const request = {
size: 0,
_source: false,
@ -297,16 +305,21 @@ export const calculateRiskScores = async ({
scores: {
host: [],
user: [],
service: [],
},
};
}
const userBuckets = response.aggregations.user?.buckets ?? [];
const hostBuckets = response.aggregations.host?.buckets ?? [];
const serviceBuckets = experimentalFeatures.serviceEntityStoreEnabled
? response.aggregations.service?.buckets ?? []
: [];
const afterKeys = {
host: response.aggregations.host?.after_key,
user: response.aggregations.user?.after_key,
service: experimentalFeatures ? response.aggregations.service?.after_key : undefined,
};
const hostScores = await processScores({
@ -323,6 +336,13 @@ export const calculateRiskScores = async ({
logger,
now,
});
const serviceScores = await processScores({
assetCriticalityService,
buckets: serviceBuckets,
identifierField: 'service.name',
logger,
now,
});
return {
...(debug ? { request, response } : {}),
@ -330,6 +350,7 @@ export const calculateRiskScores = async ({
scores: {
host: hostScores,
user: userScores,
service: serviceScores,
},
};
});

16
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/configurations.test.ts
View file

@ -19,7 +19,7 @@ describe('getTransformOptions', () => {
"_meta": Object {
"managed": true,
"managed_by": "security-entity-analytics",
"version": 2,
"version": 3,
},
"dest": Object {
"index": "dest",
@ -30,6 +30,7 @@ describe('getTransformOptions', () => {
"unique_key": Array [
"host.name",
"user.name",
"service.name",
],
},
"settings": Object {
@ -39,6 +40,19 @@ describe('getTransformOptions', () => {
"index": Array [
"source",
],
"query": Object {
"bool": Object {
"filter": Array [
Object {
"range": Object {
"@timestamp": Object {
"gte": "now-24h",
},
},
},
],
},
},
},
"sync": Object {
"time": Object {

31
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/configurations.ts
View file

@ -9,6 +9,7 @@ import type { FieldMap } from '@kbn/alerts-as-data-utils';
import type { IdentifierType } from '../../../../common/entity_analytics/risk_engine';
import {
RiskScoreEntity,
SERVICE_RISK_SCORE_ENTITY,
riskScoreBaseIndexName,
} from '../../../../common/entity_analytics/risk_engine';
import type { IIndexPatternString } from '../utils/create_datastream';
@ -126,6 +127,17 @@ export const riskScoreFieldMap: FieldMap = {
required: false,
},
...buildIdentityRiskFields(RiskScoreEntity.user),
'service.name': {
type: 'keyword',
array: false,
required: false,
},
'service.risk': {
type: 'object',
array: false,
required: false,
},
...buildIdentityRiskFields(SERVICE_RISK_SCORE_ENTITY),
} as const;
export const mappingComponentName = '.risk-score-mappings';
@ -159,10 +171,24 @@ export const getTransformOptions = ({
},
latest: {
sort: '@timestamp',
unique_key: [`host.name`, `user.name`],
unique_key: [`host.name`, `user.name`, `service.name`],
},
source: {
index: source,
query: {
bool: {
filter: [
{
range: {
'@timestamp': {
// It prevents the transform from processing too much data on reinstall
gte: 'now-24h',
},
},
},
],
},
},
},
frequency: '1h', // 1h is the maximum value
sync: {
@ -175,8 +201,7 @@ export const getTransformOptions = ({
unattended: true, // In unattended mode, the transform retries indefinitely in case of an error
},
_meta: {
version: 2, // When this field is updated we automatically update the transform
version: 3, // When this field is updated we automatically update the transform
managed: true, // Metadata that identifies the transform. It has no functionality
managed_by: 'security-entity-analytics', // Metadata that identifies the transform. It has no functionality
},

11
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/helpers.ts
View file

@ -9,8 +9,14 @@ import type { RiskScoresCalculationResponse } from '../../../../common/api/entit
import type { AfterKeys, EntityAfterKey } from '../../../../common/api/entity_analytics/common';
import type { IdentifierType } from '../../../../common/entity_analytics/risk_engine';
const identifierByEntityType = {
host: 'host.name',
user: 'user.name',
service: 'service.name',
};
export const getFieldForIdentifier = (identifierType: IdentifierType): string =>
identifierType === 'host' ? 'host.name' : 'user.name';
identifierByEntityType[identifierType];
export const getAfterKeyForIdentifierType = ({
afterKeys,
@ -22,4 +28,5 @@ export const getAfterKeyForIdentifierType = ({
export const isRiskScoreCalculationComplete = (result: RiskScoresCalculationResponse): boolean =>
Object.keys(result.after_keys.host ?? {}).length === 0 &&
Object.keys(result.after_keys.user ?? {}).length === 0;
Object.keys(result.after_keys.user ?? {}).length === 0 &&
Object.keys(result.after_keys.service ?? {}).length === 0;

11
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_engine_data_writer.ts
View file

@ -19,6 +19,7 @@ interface WriterBulkResponse {
interface BulkParams {
host?: EntityRiskScoreRecord[];
user?: EntityRiskScoreRecord[];
service?: EntityRiskScoreRecord[];
refresh?: 'wait_for';
}
@ -38,7 +39,7 @@ export class RiskEngineDataWriter implements RiskEngineDataWriter {
public bulk = async (params: BulkParams) => {
try {
if (!params.host?.length && !params.user?.length) {
if (!params.host?.length && !params.user?.length && !params.service?.length) {
return { errors: [], docs_written: 0, took: 0 };
}
@ -81,7 +82,13 @@ export class RiskEngineDataWriter implements RiskEngineDataWriter {
this.scoreToEcs(score, 'user'),
]) ?? [];
return hostBody.concat(userBody) as BulkOperationContainer[];
const serviceBody =
params.service?.flatMap((score) => [
{ create: { _index: this.options.index } },
this.scoreToEcs(score, 'service'),
]) ?? [];
return [...hostBody, ...userBody, ...serviceBody] as BulkOperationContainer[];
};
private scoreToEcs = (score: EntityRiskScoreRecord, identifierType: IdentifierType): unknown => {

483
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_data_client.test.ts
View file

@ -38,13 +38,14 @@ jest.mock('../utils/create_or_update_index', () => ({
jest.spyOn(transforms, 'createTransform').mockResolvedValue(Promise.resolve());
jest.spyOn(transforms, 'scheduleTransformNow').mockResolvedValue(Promise.resolve());
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser;
const totalFieldsLimit = 1000;
describe('RiskScoreDataClient', () => {
let riskScoreDataClient: RiskScoreDataClient;
let riskScoreDataClientWithNameSpace: RiskScoreDataClient;
let mockSavedObjectClient: ReturnType<typeof savedObjectsClientMock.create>;
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser;
const totalFieldsLimit = 1000;
beforeEach(() => {
logger = loggingSystemMock.createLogger();
@ -84,244 +85,6 @@ describe('RiskScoreDataClient', () => {
describe('init success', () => {
it('should initialize risk engine resources in the appropriate space', async () => {
const assertComponentTemplate = (namespace: string) => {
expect(createOrUpdateComponentTemplate).toHaveBeenCalledWith(
expect.objectContaining({
logger,
esClient,
template: expect.objectContaining({
name: `.risk-score-mappings-${namespace}`,
_meta: {
managed: true,
},
}),
totalFieldsLimit: 1000,
})
);
};
const assertIndexTemplate = (namespace: string) => {
expect(createOrUpdateIndexTemplate).toHaveBeenCalledWith({
logger,
esClient,
template: {
name: `.risk-score.risk-score-${namespace}-index-template`,
body: {
data_stream: { hidden: true },
index_patterns: [`risk-score.risk-score-${namespace}`],
composed_of: [`.risk-score-mappings-${namespace}`],
template: {
lifecycle: {},
settings: {
'index.mapping.total_fields.limit': totalFieldsLimit,
},
mappings: {
dynamic: false,
_meta: {
kibana: {
version: '8.9.0',
},
managed: true,
namespace,
},
},
},
_meta: {
kibana: {
version: '8.9.0',
},
managed: true,
namespace,
},
},
},
});
};
const assertDataStream = (namespace: string) => {
expect(createDataStream).toHaveBeenCalledWith({
logger,
esClient,
totalFieldsLimit,
indexPatterns: {
template: `.risk-score.risk-score-${namespace}-index-template`,
alias: `risk-score.risk-score-${namespace}`,
},
});
};
const assertIndex = (namespace: string) => {
expect(createOrUpdateIndex).toHaveBeenCalledWith({
logger,
esClient,
options: {
index: `risk-score.risk-score-latest-${namespace}`,
mappings: {
dynamic: false,
properties: {
'@timestamp': {
ignore_malformed: false,
type: 'date',
},
host: {
properties: {
name: {
type: 'keyword',
},
risk: {
properties: {
calculated_level: {
type: 'keyword',
},
calculated_score: {
type: 'float',
},
calculated_score_norm: {
type: 'float',
},
category_1_count: {
type: 'long',
},
category_1_score: {
type: 'float',
},
id_field: {
type: 'keyword',
},
id_value: {
type: 'keyword',
},
inputs: {
properties: {
category: {
type: 'keyword',
},
description: {
type: 'keyword',
},
id: {
type: 'keyword',
},
index: {
type: 'keyword',
},
risk_score: {
type: 'float',
},
timestamp: {
type: 'date',
},
},
type: 'object',
},
notes: {
type: 'keyword',
},
},
type: 'object',
},
},
},
user: {
properties: {
name: {
type: 'keyword',
},
risk: {
properties: {
calculated_level: {
type: 'keyword',
},
calculated_score: {
type: 'float',
},
calculated_score_norm: {
type: 'float',
},
category_1_count: {
type: 'long',
},
category_1_score: {
type: 'float',
},
id_field: {
type: 'keyword',
},
id_value: {
type: 'keyword',
},
inputs: {
properties: {
category: {
type: 'keyword',
},
description: {
type: 'keyword',
},
id: {
type: 'keyword',
},
index: {
type: 'keyword',
},
risk_score: {
type: 'float',
},
timestamp: {
type: 'date',
},
},
type: 'object',
},
notes: {
type: 'keyword',
},
},
type: 'object',
},
},
},
},
},
},
});
};
const assertTransform = (namespace: string) => {
expect(transforms.createTransform).toHaveBeenCalledWith({
logger,
esClient,
transform: {
dest: {
index: `risk-score.risk-score-latest-${namespace}`,
},
frequency: '1h',
latest: {
sort: '@timestamp',
unique_key: ['host.name', 'user.name'],
},
source: {
index: [`risk-score.risk-score-${namespace}`],
},
sync: {
time: {
delay: '0s',
field: '@timestamp',
},
},
transform_id: `risk_score_latest_transform_${namespace}`,
settings: {
unattended: true,
},
_meta: {
version: 2,
managed: true,
managed_by: 'security-entity-analytics',
},
},
});
};
// Default namespace
esClient.cluster.existsComponentTemplate.mockResolvedValue(false);
await riskScoreDataClient.init();
@ -340,139 +103,9 @@ describe('RiskScoreDataClient', () => {
assertIndex('space-1');
assertTransform('space-1');
expect((createOrUpdateComponentTemplate as jest.Mock).mock.lastCall[0].template.template)
.toMatchInlineSnapshot(`
Object {
"mappings": Object {
"dynamic": "strict",
"properties": Object {
"@timestamp": Object {
"ignore_malformed": false,
"type": "date",
},
"host": Object {
"properties": Object {
"name": Object {
"type": "keyword",
},
"risk": Object {
"properties": Object {
"calculated_level": Object {
"type": "keyword",
},
"calculated_score": Object {
"type": "float",
},
"calculated_score_norm": Object {
"type": "float",
},
"category_1_count": Object {
"type": "long",
},
"category_1_score": Object {
"type": "float",
},
"id_field": Object {
"type": "keyword",
},
"id_value": Object {
"type": "keyword",
},
"inputs": Object {
"properties": Object {
"category": Object {
"type": "keyword",
},
"description": Object {
"type": "keyword",
},
"id": Object {
"type": "keyword",
},
"index": Object {
"type": "keyword",
},
"risk_score": Object {
"type": "float",
},
"timestamp": Object {
"type": "date",
},
},
"type": "object",
},
"notes": Object {
"type": "keyword",
},
},
"type": "object",
},
},
},
"user": Object {
"properties": Object {
"name": Object {
"type": "keyword",
},
"risk": Object {
"properties": Object {
"calculated_level": Object {
"type": "keyword",
},
"calculated_score": Object {
"type": "float",
},
"calculated_score_norm": Object {
"type": "float",
},
"category_1_count": Object {
"type": "long",
},
"category_1_score": Object {
"type": "float",
},
"id_field": Object {
"type": "keyword",
},
"id_value": Object {
"type": "keyword",
},
"inputs": Object {
"properties": Object {
"category": Object {
"type": "keyword",
},
"description": Object {
"type": "keyword",
},
"id": Object {
"type": "keyword",
},
"index": Object {
"type": "keyword",
},
"risk_score": Object {
"type": "float",
},
"timestamp": Object {
"type": "date",
},
},
"type": "object",
},
"notes": Object {
"type": "keyword",
},
},
"type": "object",
},
},
},
},
},
"settings": Object {},
}
`);
expect(
(createOrUpdateComponentTemplate as jest.Mock).mock.lastCall[0].template.template
).toMatchSnapshot();
});
});
@ -527,3 +160,105 @@ describe('RiskScoreDataClient', () => {
});
});
});
const assertComponentTemplate = (namespace: string) => {
expect(createOrUpdateComponentTemplate).toHaveBeenCalledWith(
expect.objectContaining({
logger,
esClient,
template: expect.objectContaining({
name: `.risk-score-mappings-${namespace}`,
_meta: {
managed: true,
},
}),
totalFieldsLimit: 1000,
})
);
};
const assertIndexTemplate = (namespace: string) => {
expect(createOrUpdateIndexTemplate).toHaveBeenCalledWith({
logger,
esClient,
template: {
name: `.risk-score.risk-score-${namespace}-index-template`,
body: expect.objectContaining({
data_stream: { hidden: true },
index_patterns: [`risk-score.risk-score-${namespace}`],
composed_of: [`.risk-score-mappings-${namespace}`],
}),
},
});
};
const assertDataStream = (namespace: string) => {
expect(createDataStream).toHaveBeenCalledWith({
logger,
esClient,
totalFieldsLimit,
indexPatterns: {
template: `.risk-score.risk-score-${namespace}-index-template`,
alias: `risk-score.risk-score-${namespace}`,
},
});
};
const assertIndex = (namespace: string) => {
expect(createOrUpdateIndex).toHaveBeenCalledWith({
logger,
esClient,
options: {
index: `risk-score.risk-score-latest-${namespace}`,
mappings: expect.any(Object),
},
});
};
const assertTransform = (namespace: string) => {
expect(transforms.createTransform).toHaveBeenCalledWith({
logger,
esClient,
transform: {
dest: {
index: `risk-score.risk-score-latest-${namespace}`,
},
frequency: '1h',
latest: {
sort: '@timestamp',
unique_key: ['host.name', 'user.name', 'service.name'],
},
source: {
index: [`risk-score.risk-score-${namespace}`],
query: {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-24h',
},
},
},
],
},
},
},
sync: {
time: {
delay: '0s',
field: '@timestamp',
},
},
transform_id: `risk_score_latest_transform_${namespace}`,
settings: {
unattended: true,
},
_meta: {
version: 3,
managed: true,
managed_by: 'security-entity-analytics',
},
},
});
};

97
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_data_client.ts
View file

@ -26,14 +26,19 @@ import {
riskScoreFieldMap,
totalFieldsLimit,
} from './configurations';
import { createDataStream } from '../utils/create_datastream';
import { createDataStream, updateUnderlyingMapping } from '../utils/create_datastream';
import type { RiskEngineDataWriter as Writer } from './risk_engine_data_writer';
import { RiskEngineDataWriter } from './risk_engine_data_writer';
import {
getRiskScoreLatestIndex,
getRiskScoreTimeSeriesIndex,
} from '../../../../common/entity_analytics/risk_engine';
import { createTransform, getLatestTransformId } from '../utils/transforms';
import {
createTransform,
deleteTransform,
stopTransform,
getLatestTransformId,
} from '../utils/transforms';
import { getRiskInputsIndex } from './get_risk_inputs_index';
import { createOrUpdateIndex } from '../utils/create_or_update_index';
@ -88,7 +93,7 @@ export class RiskScoreDataClient {
soClient: this.options.soClient,
});
public createRiskScoreLatestIndex = async () => {
public createOrUpdateRiskScoreLatestIndex = async () => {
await createOrUpdateIndex({
esClient: this.options.esClient,
logger: this.options.logger,
@ -99,6 +104,30 @@ export class RiskScoreDataClient {
});
};
public createOrUpdateRiskScoreIndexTemplate = async () =>
createOrUpdateComponentTemplate({
logger: this.options.logger,
esClient: this.options.esClient,
template: {
name: nameSpaceAwareMappingsComponentName(this.options.namespace),
_meta: {
managed: true,
},
template: {
settings: {},
mappings: mappingFromFieldMap(riskScoreFieldMap, 'strict'),
},
} as ClusterPutComponentTemplateRequest,
totalFieldsLimit,
});
public updateRiskScoreTimeSeriesIndexMappings = async () =>
updateUnderlyingMapping({
esClient: this.options.esClient,
logger: this.options.logger,
index: getRiskScoreTimeSeriesIndex(this.options.namespace),
});
public async init() {
const namespace = this.options.namespace;
@ -116,32 +145,15 @@ export class RiskScoreDataClient {
};
// Check if there are any existing component templates with the namespace in the name
const oldComponentTemplateExists = await esClient.cluster.existsComponentTemplate({
name: mappingComponentName,
});
if (oldComponentTemplateExists) {
await this.updateComponentTemplateNamewithNamespace(namespace);
await this.updateComponentTemplateNameWithNamespace(namespace);
}
// Update the new component template with the required data
await Promise.all([
createOrUpdateComponentTemplate({
logger: this.options.logger,
esClient,
template: {
name: nameSpaceAwareMappingsComponentName(namespace),
_meta: {
managed: true,
},
template: {
settings: {},
mappings: mappingFromFieldMap(riskScoreFieldMap, 'strict'),
},
} as ClusterPutComponentTemplateRequest,
totalFieldsLimit,
}),
]);
await this.createOrUpdateRiskScoreIndexTemplate();
// Reference the new component template in the index template
await createOrUpdateIndexTemplate({
@ -183,7 +195,7 @@ export class RiskScoreDataClient {
indexPatterns,
});
await this.createRiskScoreLatestIndex();
await this.createOrUpdateRiskScoreLatestIndex();
const transformId = getLatestTransformId(namespace);
await createTransform({
@ -226,16 +238,12 @@ export class RiskScoreDataClient {
const errors: Error[] = [];
const addError = (e: Error) => errors.push(e);
await esClient.transform
.deleteTransform(
{
transform_id: getLatestTransformId(namespace),
delete_dest_index: true,
force: true,
},
{ ignore: [404] }
)
.catch(addError);
await deleteTransform({
esClient,
logger: this.options.logger,
transformId: getLatestTransformId(namespace),
deleteData: true,
}).catch(addError);
await esClient.indices
.deleteDataStream(
@ -316,7 +324,7 @@ export class RiskScoreDataClient {
);
}
private async updateComponentTemplateNamewithNamespace(namespace: string): Promise<void> {
private async updateComponentTemplateNameWithNamespace(namespace: string): Promise<void> {
const esClient = this.options.esClient;
const oldComponentTemplateResponse = await esClient.cluster.getComponentTemplate(
{
@ -331,4 +339,25 @@ export class RiskScoreDataClient {
body: oldComponentTemplate.component_template,
});
}
public async reinstallTransform() {
const esClient = this.options.esClient;
const namespace = this.options.namespace;
const transformId = getLatestTransformId(namespace);
const indexPatterns = getIndexPatternDataStream(namespace);
await stopTransform({ esClient, logger: this.options.logger, transformId });
await deleteTransform({ esClient, logger: this.options.logger, transformId });
await createTransform({
esClient,
logger: this.options.logger,
transform: {
transform_id: transformId,
...getTransformOptions({
dest: getRiskScoreLatestIndex(namespace),
source: [indexPatterns.alias],
}),
},
});
}
}

12
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/risk_score_service.ts
View file

@ -6,6 +6,7 @@
*/
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
import type { ExperimentalFeatures } from '../../../../common';
import type {
RiskScoresCalculationResponse,
RiskScoresPreviewResponse,
@ -48,6 +49,7 @@ export interface RiskScoreServiceFactoryParams {
riskScoreDataClient: RiskScoreDataClient;
spaceId: string;
refresh?: 'wait_for';
experimentalFeatures: ExperimentalFeatures;
}
export const riskScoreServiceFactory = ({
@ -57,9 +59,16 @@ export const riskScoreServiceFactory = ({
riskEngineDataClient,
riskScoreDataClient,
spaceId,
experimentalFeatures,
}: RiskScoreServiceFactoryParams): RiskScoreService => ({
calculateScores: (params) =>
calculateRiskScores({ ...params, assetCriticalityService, esClient, logger }),
calculateRiskScores({
...params,
assetCriticalityService,
esClient,
logger,
experimentalFeatures,
}),
calculateAndPersistScores: (params) =>
calculateAndPersistRiskScores({
...params,
@ -68,6 +77,7 @@ export const riskScoreServiceFactory = ({
logger,
riskScoreDataClient,
spaceId,
experimentalFeatures,
}),
getConfigurationWithDefaults: async (entityAnalyticsConfig: EntityAnalyticsConfig) => {
const savedObjectConfig = await riskEngineDataClient.getConfiguration();

2
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/routes/helpers.ts
View file

@ -25,6 +25,7 @@ export function buildRiskScoreServiceForRequest(
});
const riskEngineDataClient = securityContext.getRiskEngineDataClient();
const riskScoreDataClient = securityContext.getRiskScoreDataClient();
const experimentalFeatures = securityContext.getConfig().experimentalFeatures;
return riskScoreServiceFactory({
assetCriticalityService,
@ -33,5 +34,6 @@ export function buildRiskScoreServiceForRequest(
riskEngineDataClient,
riskScoreDataClient,
spaceId,
experimentalFeatures,
});
}

34
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/tasks/risk_scoring_task.test.ts
View file

@ -24,9 +24,12 @@ import {
} from './risk_scoring_task';
import type { ConfigType } from '../../../../config';
import { TaskStatus } from '@kbn/task-manager-plugin/server';
import type { ExperimentalFeatures } from '../../../../../common';
const ISO_8601_PATTERN = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/;
const mockExperimentalFeatures = {} as ExperimentalFeatures;
const entityAnalyticsConfig = {
riskEngine: {
alertSampleSizePerShard: 10_000,
@ -63,6 +66,7 @@ describe('Risk Scoring Task', () => {
telemetry: mockTelemetry,
entityAnalyticsConfig,
auditLogger: undefined,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockTaskManagerSetup.registerTaskDefinitions).toHaveBeenCalled();
});
@ -77,6 +81,7 @@ describe('Risk Scoring Task', () => {
telemetry: mockTelemetry,
entityAnalyticsConfig,
auditLogger: undefined,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockTaskManagerSetup.registerTaskDefinitions).not.toHaveBeenCalled();
});
@ -209,6 +214,9 @@ describe('Risk Scoring Task', () => {
pageSize: 10_000,
range: { start: 'now-30d', end: 'now' },
alertSampleSizePerShard: 10_000,
_meta: {
mappingsVersion: 1,
},
});
mockIsCancelled = jest.fn().mockReturnValue(false);
@ -232,6 +240,7 @@ describe('Risk Scoring Task', () => {
isCancelled: mockIsCancelled,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.calculateAndPersistScores).toHaveBeenCalledTimes(1);
});
@ -260,6 +269,7 @@ describe('Risk Scoring Task', () => {
isCancelled: mockIsCancelled,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.getConfigurationWithDefaults).toHaveBeenCalledTimes(1);
@ -273,6 +283,7 @@ describe('Risk Scoring Task', () => {
isCancelled: mockIsCancelled,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.calculateAndPersistScores).toHaveBeenCalledTimes(2);
});
@ -289,6 +300,9 @@ describe('Risk Scoring Task', () => {
pageSize: 11_111,
range: { start: 'now-30d', end: 'now' },
alertSampleSizePerShard: 10_000,
_meta: {
mappingsVersion: 1,
},
});
await runTask({
getRiskScoreService,
@ -297,6 +311,7 @@ describe('Risk Scoring Task', () => {
isCancelled: mockIsCancelled,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.calculateAndPersistScores).toHaveBeenCalledWith(
@ -325,6 +340,9 @@ describe('Risk Scoring Task', () => {
pageSize: 10_000,
range: { start: 'now-30d', end: 'now' },
alertSampleSizePerShard: 10_000,
_meta: {
mappingsVersion: 1,
},
});
// add additional mock responses for the additional identifier calls
mockRiskScoreService.calculateAndPersistScores
@ -348,6 +366,7 @@ describe('Risk Scoring Task', () => {
isCancelled: mockIsCancelled,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.calculateAndPersistScores).toHaveBeenCalledTimes(4);
@ -373,6 +392,7 @@ describe('Risk Scoring Task', () => {
isCancelled: mockIsCancelled,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(initialState).not.toEqual(nextState);
@ -397,6 +417,9 @@ describe('Risk Scoring Task', () => {
pageSize: 11_111,
range: { start: 'now-30d', end: 'now' },
alertSampleSizePerShard: 10_000,
_meta: {
mappingsVersion: 1,
},
});
await runTask({
getRiskScoreService,
@ -405,6 +428,7 @@ describe('Risk Scoring Task', () => {
isCancelled: mockIsCancelled,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.calculateAndPersistScores).not.toHaveBeenCalled();
@ -420,6 +444,7 @@ describe('Risk Scoring Task', () => {
isCancelled: mockIsCancelled,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.calculateAndPersistScores).not.toHaveBeenCalled();
@ -436,6 +461,7 @@ describe('Risk Scoring Task', () => {
isCancelled: mockIsCancelled,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.calculateAndPersistScores).not.toHaveBeenCalled();
@ -458,6 +484,7 @@ describe('Risk Scoring Task', () => {
taskInstance: riskScoringTaskInstanceMock,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.calculateAndPersistScores).not.toHaveBeenCalled();
@ -471,6 +498,7 @@ describe('Risk Scoring Task', () => {
taskInstance: riskScoringTaskInstanceMock,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockLogger.info).toHaveBeenCalledWith(
@ -488,6 +516,7 @@ describe('Risk Scoring Task', () => {
taskInstance: riskScoringTaskInstanceMock,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockTelemetry.reportEvent).toHaveBeenCalledWith('risk_score_execution_success', {
@ -506,6 +535,7 @@ describe('Risk Scoring Task', () => {
taskInstance: riskScoringTaskInstanceMock,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.scheduleLatestTransformNow).toHaveBeenCalledTimes(1);
@ -519,6 +549,7 @@ describe('Risk Scoring Task', () => {
taskInstance: riskScoringTaskInstanceMock,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockRiskScoreService.refreshRiskScoreIndex).toHaveBeenCalledTimes(1);
@ -542,6 +573,7 @@ describe('Risk Scoring Task', () => {
taskInstance: riskScoringTaskInstanceMock,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
} catch (err) {
expect(mockTelemetry.reportEvent).toHaveBeenCalledTimes(1);
@ -561,6 +593,7 @@ describe('Risk Scoring Task', () => {
taskInstance: riskScoringTaskInstanceMock,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
})
).rejects.toThrow();
@ -577,6 +610,7 @@ describe('Risk Scoring Task', () => {
taskInstance: riskScoringTaskInstanceMock,
telemetry: mockTelemetry,
entityAnalyticsConfig,
experimentalFeatures: mockExperimentalFeatures,
});
expect(mockTelemetry.reportEvent).toHaveBeenCalledWith(

14
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/tasks/risk_scoring_task.ts
View file

@ -16,10 +16,12 @@ import type {
} from '@kbn/task-manager-plugin/server';
import type { AnalyticsServiceSetup } from '@kbn/core-analytics-server';
import type { AuditLogger } from '@kbn/security-plugin-types-server';
import type { ExperimentalFeatures } from '../../../../../common';
import type { AfterKeys } from '../../../../../common/api/entity_analytics/common';
import {
type IdentifierType,
RiskScoreEntity,
SERVICE_RISK_SCORE_ENTITY,
} from '../../../../../common/entity_analytics/risk_engine';
import { type RiskScoreService, riskScoreServiceFactory } from '../risk_score_service';
import { RiskEngineDataClient } from '../../risk_engine/risk_engine_data_client';
@ -62,6 +64,7 @@ export const registerRiskScoringTask = ({
taskManager,
telemetry,
entityAnalyticsConfig,
experimentalFeatures,
}: {
getStartServices: EntityAnalyticsRoutesDeps['getStartServices'];
kibanaVersion: string;
@ -70,6 +73,7 @@ export const registerRiskScoringTask = ({
taskManager: TaskManagerSetupContract | undefined;
telemetry: AnalyticsServiceSetup;
entityAnalyticsConfig: EntityAnalyticsConfig;
experimentalFeatures: ExperimentalFeatures;
}): void => {
if (!taskManager) {
logger.info('Task Manager is unavailable; skipping risk engine task registration.');
@ -117,6 +121,7 @@ export const registerRiskScoringTask = ({
riskEngineDataClient,
riskScoreDataClient,
spaceId: namespace,
experimentalFeatures,
});
});
@ -130,6 +135,7 @@ export const registerRiskScoringTask = ({
getRiskScoreService,
telemetry,
entityAnalyticsConfig,
experimentalFeatures,
}),
},
});
@ -218,6 +224,7 @@ export const runTask = async ({
taskInstance,
telemetry,
entityAnalyticsConfig,
experimentalFeatures,
}: {
logger: Logger;
isCancelled: () => boolean;
@ -225,6 +232,7 @@ export const runTask = async ({
taskInstance: ConcreteTaskInstance;
telemetry: AnalyticsServiceSetup;
entityAnalyticsConfig: EntityAnalyticsConfig;
experimentalFeatures: ExperimentalFeatures;
}): Promise<{
state: RiskScoringTaskState;
}> => {
@ -283,8 +291,11 @@ export const runTask = async ({
const { index, runtimeMappings } = await riskScoreService.getRiskInputsIndex({
dataViewId,
});
const identifierTypes: IdentifierType[] = configuredIdentifierType
? [configuredIdentifierType]
: experimentalFeatures.serviceEntityStoreEnabled
? [RiskScoreEntity.host, RiskScoreEntity.user, SERVICE_RISK_SCORE_ENTITY]
: [RiskScoreEntity.host, RiskScoreEntity.user];
const runs: Array<{
@ -384,11 +395,13 @@ const createTaskRunnerFactory =
getRiskScoreService,
telemetry,
entityAnalyticsConfig,
experimentalFeatures,
}: {
logger: Logger;
getRiskScoreService: GetRiskScoreService;
telemetry: AnalyticsServiceSetup;
entityAnalyticsConfig: EntityAnalyticsConfig;
experimentalFeatures: ExperimentalFeatures;
}) =>
({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => {
let cancelled = false;
@ -402,6 +415,7 @@ const createTaskRunnerFactory =
taskInstance,
telemetry,
entityAnalyticsConfig,
experimentalFeatures,
}),
cancel: async () => {
cancelled = true;

8
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/types.ts
View file

@ -34,6 +34,10 @@ export interface CalculateRiskScoreAggregations {
after_key: EntityAfterKey;
buckets: RiskScoreBucket[];
};
service?: {
after_key: EntityAfterKey;
buckets: RiskScoreBucket[];
};
}
export interface SearchHitRiskInput {
@ -72,6 +76,10 @@ export interface RiskEngineConfiguration {
pageSize: number;
range: Range;
alertSampleSizePerShard?: number;
// When the version changes the engine automatically updates the mappings
_meta: {
mappingsVersion: number;
};
}
export interface CalculateScoresParams {

2
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/create_datastream.ts
View file

@ -66,7 +66,7 @@ const updateTotalFieldLimitSetting = async ({
// is due to the fact settings can be classed as dynamic and static, and static
// updates will fail on an index that isn't closed. New settings *will* be applied as part
// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654
const updateUnderlyingMapping = async ({ logger, esClient, index }: UpdateIndexOpts) => {
export const updateUnderlyingMapping = async ({ logger, esClient, index }: UpdateIndexOpts) => {
let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse;
try {
simulatedIndexMapping = await retryTransientEsErrors(

2
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/create_or_update_index.ts
View file

@ -13,7 +13,7 @@ import type {
import { retryTransientEsErrors } from './retry_transient_es_errors';
/**
* It's check for index existatnce, and create index
* It's check for index existence, and create index
* or update existing index mappings
*/
export const createOrUpdateIndex = async ({

72
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/transforms.test.ts
View file

@ -15,7 +15,11 @@ import {
getRiskScoreTimeSeriesIndex,
} from '../../../../common/entity_analytics/risk_engine';
import { getTransformOptions } from '../risk_score/configurations';
import { scheduleLatestTransformNow, scheduleTransformNow } from './transforms';
import {
scheduleLatestTransformNow,
scheduleTransformNow,
upgradeLatestTransformIfNeeded,
} from './transforms';
const transformId = 'test_transform_id';
@ -74,6 +78,28 @@ const outdatedTransformsMock = {
},
],
} as TransformGetTransformResponse;
const outdatedTransformsRequiredReinstallMock = {
count: 1,
transforms: [
{
...transformConfig,
id: 'test_transform_id_3',
sync: {
time: {
field: '@timestamp',
delay: '2s',
},
},
latest: {
unique_key: ['test'],
sort: 'desc',
},
_meta: {
version: '1',
},
},
],
} as TransformGetTransformResponse;
const logger = loggingSystemMock.createLogger();
@ -135,5 +161,49 @@ describe('transforms utils', () => {
'There was an error upgrading the transform risk_score_latest_transform_tests. Continuing with transform scheduling. Test error'
);
});
it('it calls upgradeLatestTransformIfNeeded', async () => {
const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser;
esClient.transform.getTransformStats.mockResolvedValueOnce(stoppedTransformsMock);
esClient.transform.getTransform.mockResolvedValueOnce(outdatedTransformsMock);
await scheduleLatestTransformNow({ esClient, namespace: 'tests', logger });
expect(esClient.transform.updateTransform).toHaveBeenCalled();
});
});
describe('upgradeLatestTransformIfNeeded', () => {
it('updateTransform the transform if it is outdated', async () => {
const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser;
esClient.transform.getTransform.mockResolvedValueOnce(outdatedTransformsMock);
await upgradeLatestTransformIfNeeded({ esClient, namespace: 'tests', logger });
expect(esClient.transform.updateTransform).toHaveBeenCalled();
});
it('does not reinstall the transform if it is not outdated', async () => {
const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser;
esClient.transform.getTransform.mockResolvedValueOnce(updatedTransformsMock);
await upgradeLatestTransformIfNeeded({ esClient, namespace: 'tests', logger });
expect(esClient.transform.updateTransform).not.toHaveBeenCalled();
});
it('reinstalls the transform if it is outdated and requires reinstall', async () => {
const esClient = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser;
esClient.transform.getTransform
.mockResolvedValueOnce(outdatedTransformsRequiredReinstallMock)
.mockRejectedValueOnce({ statusCode: 404 }); // MAKE IT 404
await upgradeLatestTransformIfNeeded({ esClient, namespace: 'tests', logger });
expect(esClient.transform.stopTransform).toHaveBeenCalled();
expect(esClient.transform.deleteTransform).toHaveBeenCalled();
expect(esClient.transform.putTransform).toHaveBeenCalled();
});
});
});

104
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/utils/transforms.ts
View file

@ -13,6 +13,7 @@ import type {
TransformGetTransformTransformSummary,
TransformPutTransformRequest,
TransformGetTransformStatsTransformStats,
AcknowledgedResponseBase,
} from '@elastic/elasticsearch/lib/api/types';
import {
getRiskScoreLatestIndex,
@ -25,6 +26,7 @@ import {
} from '../../../../common/utils/risk_score_modules';
import type { TransformOptions } from '../risk_score/configurations';
import { getTransformOptions } from '../risk_score/configurations';
import { retryTransientEsErrors } from './retry_transient_es_errors';
export const getLegacyTransforms = async ({
namespace,
@ -106,6 +108,72 @@ export const createTransform = async ({
}
};
export const stopTransform = async ({
esClient,
logger,
transformId,
}: {
esClient: ElasticsearchClient;
logger: Logger;
transformId: string;
}): Promise<AcknowledgedResponseBase> =>
retryTransientEsErrors(
() =>
esClient.transform.stopTransform(
{
transform_id: transformId,
wait_for_completion: true,
force: true,
},
{ ignore: [409, 404] }
),
{ logger }
);
export const deleteTransform = ({
esClient,
logger,
transformId,
deleteData = false,
}: {
esClient: ElasticsearchClient;
logger: Logger;
transformId: string;
deleteData?: boolean;
}): Promise<AcknowledgedResponseBase> =>
retryTransientEsErrors(
() =>
esClient.transform.deleteTransform(
{
transform_id: transformId,
force: true,
delete_dest_index: deleteData,
},
{ ignore: [404] }
),
{ logger }
);
export const reinstallTransform = async ({
esClient,
logger,
config,
}: {
esClient: ElasticsearchClient;
logger: Logger;
config: TransformPutTransformRequest;
}): Promise<void> => {
const transformId = config.transform_id;
await stopTransform({ esClient, logger, transformId });
await deleteTransform({ esClient, logger, transformId });
await createTransform({
esClient,
logger,
transform: config,
});
};
export const getLatestTransformId = (namespace: string): string =>
`risk_score_latest_transform_${namespace}`;
@ -141,9 +209,10 @@ export const scheduleTransformNow = async ({
};
/**
* Whenever we change the latest transform configuration, we must ensure we update the transform in environments where it has already been installed.
* This method updates the transform configuration if it is outdated.
* If the 'latest' property of the transform changes it will reinstall the transform.
*/
const upgradeLatestTransformIfNeeded = async ({
export const upgradeLatestTransformIfNeeded = async ({
esClient,
namespace,
logger,
@ -166,14 +235,22 @@ const upgradeLatestTransformIfNeeded = async ({
});
if (isTransformOutdated(response.transforms[0], newConfig)) {
logger.info(`Upgrading transform ${transformId}`);
if (doesTransformRequireReinstall(response.transforms[0], newConfig)) {
logger.info(`Reinstalling transform ${transformId}`);
await reinstallTransform({
esClient,
logger,
config: { ...newConfig, transform_id: transformId },
});
} else {
logger.info(`Upgrading transform ${transformId}`);
const { latest: _unused, ...changes } = newConfig;
const { latest: _unused, ...changes } = newConfig;
await esClient.transform.updateTransform({
transform_id: transformId,
...changes,
});
await esClient.transform.updateTransform({
transform_id: transformId,
...changes,
});
}
}
};
@ -199,11 +276,12 @@ export const scheduleLatestTransformNow = async ({
await scheduleTransformNow({ esClient, transformId });
};
/**
* Whitelist the transform fields that we can update.
*/
const isTransformOutdated = (
transform: TransformGetTransformTransformSummary,
newConfig: TransformOptions
): boolean => transform._meta?.version !== newConfig._meta?.version;
const doesTransformRequireReinstall = (
transform: TransformGetTransformTransformSummary,
newConfig: TransformOptions
): boolean => JSON.stringify(transform.latest) !== JSON.stringify(newConfig.latest);

2
x-pack/solutions/security/plugins/security_solution/server/plugin.ts
View file

@ -221,6 +221,7 @@ export class Plugin implements ISecuritySolutionPlugin {
taskManager: plugins.taskManager,
telemetry: core.analytics,
entityAnalyticsConfig: config.entityAnalytics,
experimentalFeatures,
});
}
@ -229,6 +230,7 @@ export class Plugin implements ISecuritySolutionPlugin {
taskManager: plugins.taskManager,
logger: this.logger,
auditLogger: plugins.security?.audit.withoutRequest,
kibanaVersion: pluginContext.env.packageInfo.version,
}).catch((err) => {
logger.error(`Error scheduling entity analytics migration: ${err}`);
});

12
x-pack/solutions/security/plugins/security_solution/server/usage/risk_engine/get_risk_engine_metrics.ts
View file

@ -40,6 +40,7 @@ const getEntitiesAggregationData = async ({
logger,
hostMetricField,
userMetricField,
serviceMetricField,
lastDay,
}: {
esClient: ElasticsearchClient;
@ -47,6 +48,7 @@ const getEntitiesAggregationData = async ({
logger: Logger;
hostMetricField: string;
userMetricField: string;
serviceMetricField: string;
lastDay: boolean;
}) => {
try {
@ -72,6 +74,9 @@ const getEntitiesAggregationData = async ({
host_name: {
value: number;
};
service_name: {
value: number;
};
}
>({
index,
@ -81,10 +86,11 @@ const getEntitiesAggregationData = async ({
return {
[userMetricField]: riskScoreAggsResponse?.aggregations?.user_name?.value,
[hostMetricField]: riskScoreAggsResponse?.aggregations?.host_name?.value,
[serviceMetricField]: riskScoreAggsResponse?.aggregations?.service_name?.value,
};
} catch (err) {
logger.error(
`Error while fetching risk score metrics for ${hostMetricField} and ${userMetricField}: ${err}`
`Error while fetching risk score metrics for ${hostMetricField}, ${userMetricField} and ${serviceMetricField} : ${err}`
);
return {};
}
@ -141,6 +147,7 @@ export const getRiskEngineMetrics = async ({
lastDay: false,
hostMetricField: 'unique_host_risk_score_total',
userMetricField: 'unique_user_risk_score_total',
serviceMetricField: 'unique_service_risk_score_total',
}),
getEntitiesAggregationData({
esClient,
@ -149,6 +156,7 @@ export const getRiskEngineMetrics = async ({
lastDay: true,
hostMetricField: 'unique_host_risk_score_day',
userMetricField: 'unique_user_risk_score_day',
serviceMetricField: 'unique_service_risk_score_day',
}),
getEntitiesAggregationData({
esClient,
@ -157,6 +165,7 @@ export const getRiskEngineMetrics = async ({
lastDay: false,
hostMetricField: 'all_host_risk_scores_total',
userMetricField: 'all_user_risk_scores_total',
serviceMetricField: 'all_service_risk_scores_total',
}),
getEntitiesAggregationData({
esClient,
@ -165,6 +174,7 @@ export const getRiskEngineMetrics = async ({
lastDay: true,
hostMetricField: 'all_host_risk_scores_total_day',
userMetricField: 'all_user_risk_scores_total_day',
serviceMetricField: 'all_service_risk_scores_total_day',
}),
getIndexSize({
esClient,

17
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality.ts
View file

@ -70,6 +70,9 @@ export default ({ getService }: FtrProviderContext) => {
expect(
assetCriticalityIndexResult['.asset-criticality.asset-criticality-default']?.mappings
).to.eql({
_meta: {
version: 2,
},
dynamic: 'strict',
properties: {
'@timestamp': {
@ -84,6 +87,20 @@ export default ({ getService }: FtrProviderContext) => {
id_value: {
type: 'keyword',
},
service: {
properties: {
asset: {
properties: {
criticality: {
type: 'keyword',
},
},
},
name: {
type: 'keyword',
},
},
},
updated_at: {
type: 'date',
},

3
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality_csv_upload.ts
View file

@ -108,7 +108,8 @@ export default ({ getService }: FtrProviderContext) => {
},
{
index: 2,
message: 'Invalid entity type "invalid_entity", expected host or user',
message:
'Invalid entity type "invalid_entity", expected to be one of: user, host, service',
},
{
index: 3,

4
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts
View file

@ -17,7 +17,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
...functionalConfig.get('kbnTestServer'),
serverArgs: [
...functionalConfig.get('kbnTestServer.serverArgs'),
`--xpack.securitySolution.enableExperimental=${JSON.stringify([])}`,
`--xpack.securitySolution.enableExperimental=${JSON.stringify([
'serviceEntityStoreEnabled',
])}`,
],
},
testFiles: [require.resolve('..')],

2
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/serverless.config.ts
View file

@ -9,7 +9,7 @@ import { createTestConfig } from '../../../../../config/serverless/config.base';
export default createTestConfig({
kbnTestServerArgs: [
`--xpack.securitySolution.enableExperimental=${JSON.stringify([])}`,
`--xpack.securitySolution.enableExperimental=${JSON.stringify(['serviceEntityStoreEnabled'])}`,
`--xpack.securitySolutionServerless.productTypes=${JSON.stringify([
{ product_line: 'security', product_tier: 'complete' },
{ product_line: 'endpoint', product_tier: 'complete' },

121
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts
View file

@ -161,6 +161,65 @@ export default ({ getService }: FtrProviderContext) => {
},
},
},
service: {
properties: {
name: {
type: 'keyword',
},
risk: {
properties: {
calculated_level: {
type: 'keyword',
},
calculated_score: {
type: 'float',
},
calculated_score_norm: {
type: 'float',
},
category_1_count: {
type: 'long',
},
category_1_score: {
type: 'float',
},
id_field: {
type: 'keyword',
},
id_value: {
type: 'keyword',
},
inputs: {
properties: {
category: {
type: 'keyword',
},
description: {
type: 'keyword',
},
id: {
type: 'keyword',
},
index: {
type: 'keyword',
},
risk_score: {
type: 'float',
},
timestamp: {
type: 'date',
},
},
type: 'object',
},
notes: {
type: 'keyword',
},
},
type: 'object',
},
},
},
user: {
properties: {
name: {
@ -372,6 +431,65 @@ export default ({ getService }: FtrProviderContext) => {
},
},
},
service: {
properties: {
name: {
type: 'keyword',
},
risk: {
properties: {
calculated_level: {
type: 'keyword',
},
calculated_score: {
type: 'float',
},
calculated_score_norm: {
type: 'float',
},
category_1_count: {
type: 'long',
},
category_1_score: {
type: 'float',
},
id_field: {
type: 'keyword',
},
id_value: {
type: 'keyword',
},
inputs: {
properties: {
category: {
type: 'keyword',
},
description: {
type: 'keyword',
},
id: {
type: 'keyword',
},
index: {
type: 'keyword',
},
risk_score: {
type: 'float',
},
timestamp: {
type: 'date',
},
},
type: 'object',
},
notes: {
type: 'keyword',
},
},
type: 'object',
},
},
},
user: {
properties: {
name: {
@ -520,6 +638,9 @@ export default ({ getService }: FtrProviderContext) => {
end: 'now',
start: 'now-30d',
},
_meta: {
mappingsVersion: 2,
},
});
});

1
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/risk_score_preview.ts
View file

@ -622,6 +622,7 @@ export default ({ getService }: FtrProviderContext): void => {
expect(scores).to.eql({
host: [],
service: [],
user: [],
});
});

16
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/telemetry_usage.ts
View file

@ -65,6 +65,14 @@ export default ({ getService }: FtrProviderContext) => {
it('@skipInServerlessMKI should return metrics with expected values when risk engine is enabled', async () => {
expect(await areRiskScoreIndicesEmpty({ log, es })).to.be(true);
const serviceId = uuidv4();
const serviceDocs = Array(10)
.fill(buildDocument({}, serviceId))
.map((event, index) => ({
...event,
'service.name': `service-${index}`,
}));
const hostId = uuidv4();
const hostDocs = Array(10)
.fill(buildDocument({}, hostId))
@ -81,17 +89,17 @@ export default ({ getService }: FtrProviderContext) => {
'user.name': `user-${index}`,
}));
await indexListOfDocuments([...hostDocs, ...userDocs]);
await indexListOfDocuments([...hostDocs, ...userDocs, ...serviceDocs]); // , ...serviceDocs
await createAndSyncRuleAndAlerts({
query: `id: ${userId} or id: ${hostId}`,
alerts: 20,
query: `id: ${userId} or id: ${hostId} or id: ${serviceId}`, //
alerts: 30,
riskScore: 40,
});
await riskEngineRoutes.init();
await waitForRiskScoresToBePresent({ es, log, scoreCount: 20 });
await waitForRiskScoresToBePresent({ es, log, scoreCount: 30 });
await retry.try(async () => {
const {