github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Entity Analytics][Entity Store] Add transform config options to the API (#208062)

Browse source 
## Summary


This PR adds the following parameters to the `INIT` engine API:

* `frequency`: the transform run frequency
* `timeout`: the timeout for the initial creation of the transform
* `docsPerSecond`: transform throttling option. See
[here](https://arc.net/l/quote/vxcmfnhh)
* `delay`: The transform delay duration. See
[here](https://arc.net/l/quote/mzvaexhv)
Coming soon


In addition, the PR adds these fields to the Saved Object with the
engine descriptor, as well as providing a migration with the appropriate
backfilling.

Finally, there are some utility function that were/are helpful in
working with objects.

## How to test

*NOTE*: Always make sure the security default data view exists. Easiest
way it to just navigate to some Security UI.

### Checking the new defaults

1. Initialize an engine via dev tools by calling: `POST
kbn:/api/entity_store/engines/<entity_type>/init {}`
2. Call `GET kbn:/api/entity_store/status`. This response should now
contain all the default optional values.

### Observing the parameters are being applied

1. Initialize an engine via the API. This time pass any of the `timeout,
frequency, delay and docsPerSecond` options in the request body.
2. Once the `status` changes to `started`, query the respective
transform: `GET
_transform/entities-v1-latest-security_<entity_type>_default`
3. Check that the parameters have been applied to the transform

### Checking Saved Object Migration

1. Check out `main`.
2. Initialize the store.
3. Query `GET kbn:/api/entity_store/status`. Note down the fields in the
engine object.
4. Check out this branch.
5. Restart kibana.
6. Query `GET kbn:/api/entity_store/status` again. Observe the new
fields have been added and backfilled

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Tiago Vila Verde 2025-01-29 21:31:47 +01:00 • committed by GitHub
parent 95d863bc8b
commit 5b22aa9b66
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
37 changed files with 1491 additions and 860 deletions
Download patch file Download diff file Expand all files Collapse all files

57
oas_docs/output/kibana.serverless.yaml
View file

@ -9789,6 +9789,14 @@ paths:
schema:
type: object
properties:
delay:
default: 1m
description: The delay before the transform will run.
pattern: '[smdh]$'
type: string
docsPerSecond:
description: The number of documents per second to process.
type: integer
enrichPolicyExecutionInterval:
$ref: '#/components/schemas/Security_Entity_Analytics_API_Interval'
entityTypes:
@ -9801,11 +9809,21 @@ paths:
type: integer
filter:
type: string
frequency:
default: 1m
description: The frequency at which the transform will run.
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
lookbackPeriod:
default: 24h
description: The lookback period for the entity store
description: The amount of time the transform looks back to calculate the aggregations.
pattern: '[smdh]$'
type: string
timeout:
default: 180s
description: The timeout for initializing the aggregating transform.
pattern: '[smdh]$'
type: string
description: Schema for the entity store initialization
@ -9915,6 +9933,14 @@ paths:
schema:
type: object
properties:
delay:
default: 1m
description: The delay before the transform will run.
pattern: '[smdh]$'
type: string
docsPerSecond:
description: The number of documents per second to process.
type: integer
enrichPolicyExecutionInterval:
$ref: '#/components/schemas/Security_Entity_Analytics_API_Interval'
fieldHistoryLength:
@ -9923,8 +9949,23 @@ paths:
type: integer
filter:
type: string
frequency:
default: 1m
description: The frequency at which the transform will run.
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
lookbackPeriod:
default: 24h
description: The amount of time the transform looks back to calculate the aggregations.
pattern: '[smdh]$'
type: string
timeout:
default: 180s
description: The timeout for initializing the aggregating transform.
pattern: '[smdh]$'
type: string
description: Schema for the engine initialization
required: true
responses:
@ -51361,12 +51402,22 @@ components:
Security_Entity_Analytics_API_EngineDescriptor:
type: object
properties:
delay:
default: 1m
pattern: '[smdh]$'
type: string
docsPerSecond:
type: integer
error:
type: object
fieldHistoryLength:
type: integer
filter:
type: string
frequency:
default: 1m
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
lookbackPeriod:
@ -51375,6 +51426,10 @@ components:
type: string
status:
$ref: '#/components/schemas/Security_Entity_Analytics_API_EngineStatus'
timeout:
default: 180s
pattern: '[smdh]$'
type: string
type:
$ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
required:

57
oas_docs/output/kibana.yaml
View file

@ -11879,6 +11879,14 @@ paths:
schema:
type: object
properties:
delay:
default: 1m
description: The delay before the transform will run.
pattern: '[smdh]$'
type: string
docsPerSecond:
description: The number of documents per second to process.
type: integer
enrichPolicyExecutionInterval:
$ref: '#/components/schemas/Security_Entity_Analytics_API_Interval'
entityTypes:
@ -11891,11 +11899,21 @@ paths:
type: integer
filter:
type: string
frequency:
default: 1m
description: The frequency at which the transform will run.
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
lookbackPeriod:
default: 24h
description: The lookback period for the entity store
description: The amount of time the transform looks back to calculate the aggregations.
pattern: '[smdh]$'
type: string
timeout:
default: 180s
description: The timeout for initializing the aggregating transform.
pattern: '[smdh]$'
type: string
description: Schema for the entity store initialization
@ -12001,6 +12019,14 @@ paths:
schema:
type: object
properties:
delay:
default: 1m
description: The delay before the transform will run.
pattern: '[smdh]$'
type: string
docsPerSecond:
description: The number of documents per second to process.
type: integer
enrichPolicyExecutionInterval:
$ref: '#/components/schemas/Security_Entity_Analytics_API_Interval'
fieldHistoryLength:
@ -12009,8 +12035,23 @@ paths:
type: integer
filter:
type: string
frequency:
default: 1m
description: The frequency at which the transform will run.
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
lookbackPeriod:
default: 24h
description: The amount of time the transform looks back to calculate the aggregations.
pattern: '[smdh]$'
type: string
timeout:
default: 180s
description: The timeout for initializing the aggregating transform.
pattern: '[smdh]$'
type: string
description: Schema for the engine initialization
required: true
responses:
@ -58051,12 +58092,22 @@ components:
Security_Entity_Analytics_API_EngineDescriptor:
type: object
properties:
delay:
default: 1m
pattern: '[smdh]$'
type: string
docsPerSecond:
type: integer
error:
type: object
fieldHistoryLength:
type: integer
filter:
type: string
frequency:
default: 1m
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
lookbackPeriod:
@ -58065,6 +58116,10 @@ components:
type: string
status:
$ref: '#/components/schemas/Security_Entity_Analytics_API_EngineStatus'
timeout:
default: 180s
pattern: '[smdh]$'
type: string
type:
$ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
required:

2
src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
View file

@ -97,7 +97,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
"entity-definition": "1c6bff35c423d5dc5650bc806cf2899e4706a0bc",
"entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
"entity-engine-status": "8cb7dcb13f5e2ea8f2e08dd4af72c110e2051120",
"entity-engine-status": "e2de87d84e9f1f72726eb28b7e670ff8021b5eb4",
"epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
"epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
"event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",

2
x-pack/platform/packages/shared/kbn-entities-schema/src/schema/entity_definition.ts
View file

@ -38,6 +38,8 @@ export const entityDefinitionSchema = z.object({
syncField: z.optional(z.string()),
syncDelay: z.optional(durationSchema),
frequency: z.optional(durationSchema),
timeout: z.optional(durationSchema),
docsPerSecond: z.optional(z.number()),
})
),
}),

2
x-pack/platform/plugins/shared/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap generated
View file

@ -148,6 +148,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"docs_per_second": undefined,
"unattended": true,
},
"source": Object {
@ -179,6 +180,7 @@ Object {
"field": "@timestamp",
},
},
"timeout": undefined,
"transform_id": "entities-v1-latest-admin-console-services",
}
`;

7
x-pack/platform/plugins/shared/entity_manager/server/lib/entities/transform/generate_latest_transform.ts
View file

@ -50,6 +50,7 @@ export function generateLatestTransform(
transformId: generateLatestTransformId(definition),
frequency: definition.latest.settings?.frequency ?? ENTITY_DEFAULT_LATEST_FREQUENCY,
syncDelay: definition.latest.settings?.syncDelay ?? ENTITY_DEFAULT_LATEST_SYNC_DELAY,
docsPerSecond: definition.latest.settings?.docsPerSecond,
});
}
@ -59,13 +60,15 @@ const generateTransformPutRequest = ({
transformId,
frequency,
syncDelay,
docsPerSecond,
}: {
definition: EntityDefinition;
transformId: string;
filter: QueryDslQueryContainer[];
frequency: string;
syncDelay: string;
}) => {
docsPerSecond?: number;
}): TransformPutTransformRequest => {
return {
transform_id: transformId,
_meta: {
@ -73,6 +76,7 @@ const generateTransformPutRequest = ({
managed: definition.managed,
},
defer_validation: true,
timeout: definition.latest.settings?.timeout,
source: {
index: definition.indexPatterns,
...(filter.length > 0 && {
@ -97,6 +101,7 @@ const generateTransformPutRequest = ({
settings: {
deduce_mappings: false,
unattended: true,
docs_per_second: docsPerSecond,
},
pivot: {
group_by: {

16
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
View file

@ -41,6 +41,22 @@ export const EngineDescriptor = z.object({
.regex(/[smdh]$/)
.optional()
.default('24h'),
timeout: z
.string()
.regex(/[smdh]$/)
.optional()
.default('180s'),
frequency: z
.string()
.regex(/[smdh]$/)
.optional()
.default('1m'),
delay: z
.string()
.regex(/[smdh]$/)
.optional()
.default('1m'),
docsPerSecond: z.number().int().optional(),
error: z.object({}).optional(),
});

14
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
View file

@ -36,6 +36,20 @@ components:
type: string
default: 24h
pattern: '[smdh]$'
timeout:
type: string
default: 180s
pattern: '[smdh]$'
frequency:
type: string
default: 1m
pattern: '[smdh]$'
delay:
type: string
default: 1m
pattern: '[smdh]$'
docsPerSecond:
type: integer
error:
type: object

38
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts
View file

@ -24,18 +24,46 @@ export const InitEntityStoreRequestBody = z.object({
* The number of historical values to keep for each field.
*/
fieldHistoryLength: z.number().int().optional().default(10),
indexPattern: IndexPattern.optional(),
filter: z.string().optional(),
entityTypes: z.array(EntityType).optional(),
enrichPolicyExecutionInterval: Interval.optional(),
/**
* The lookback period for the entity store
* The amount of time the transform looks back to calculate the aggregations.
*/
lookbackPeriod: z
.string()
.regex(/[smdh]$/)
.optional()
.default('24h'),
indexPattern: IndexPattern.optional(),
filter: z.string().optional(),
entityTypes: z.array(EntityType).optional(),
enrichPolicyExecutionInterval: Interval.optional(),
/**
* The timeout for initializing the aggregating transform.
*/
timeout: z
.string()
.regex(/[smdh]$/)
.optional()
.default('180s'),
/**
* The frequency at which the transform will run.
*/
frequency: z
.string()
.regex(/[smdh]$/)
.optional()
.default('1m'),
/**
* The delay before the transform will run.
*/
delay: z
.string()
.regex(/[smdh]$/)
.optional()
.default('1m'),
/**
* The number of documents per second to process.
*/
docsPerSecond: z.number().int().optional(),
});
export type InitEntityStoreRequestBodyInput = z.input<typeof InitEntityStoreRequestBody>;

28
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml
View file

@ -23,11 +23,6 @@ paths:
type: integer
description: The number of historical values to keep for each field.
default: 10
lookbackPeriod:
type: string
description: The lookback period for the entity store
default: 24h
pattern: '[smdh]$'
indexPattern:
$ref: './common.schema.yaml#/components/schemas/IndexPattern'
filter:
@ -38,6 +33,29 @@ paths:
$ref: './common.schema.yaml#/components/schemas/EntityType'
enrichPolicyExecutionInterval:
$ref: './common.schema.yaml#/components/schemas/Interval'
lookbackPeriod:
type: string
default: 24h
pattern: '[smdh]$'
description: The amount of time the transform looks back to calculate the aggregations.
timeout:
type: string
default: 180s
pattern: '[smdh]$'
description: The timeout for initializing the aggregating transform.
frequency:
type: string
default: 1m
pattern: '[smdh]$'
description: The frequency at which the transform will run.
delay:
type: string
default: 1m
pattern: '[smdh]$'
description: The delay before the transform will run.
docsPerSecond:
type: integer
description: The number of documents per second to process.
responses:
'200':
description: Successful response

36
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.gen.ts
View file

@ -36,6 +36,42 @@ export const InitEntityEngineRequestBody = z.object({
indexPattern: IndexPattern.optional(),
filter: z.string().optional(),
enrichPolicyExecutionInterval: Interval.optional(),
/**
* The amount of time the transform looks back to calculate the aggregations.
*/
lookbackPeriod: z
.string()
.regex(/[smdh]$/)
.optional()
.default('24h'),
/**
* The timeout for initializing the aggregating transform.
*/
timeout: z
.string()
.regex(/[smdh]$/)
.optional()
.default('180s'),
/**
* The frequency at which the transform will run.
*/
frequency: z
.string()
.regex(/[smdh]$/)
.optional()
.default('1m'),
/**
* The delay before the transform will run.
*/
delay: z
.string()
.regex(/[smdh]$/)
.optional()
.default('1m'),
/**
* The number of documents per second to process.
*/
docsPerSecond: z.number().int().optional(),
});
export type InitEntityEngineRequestBodyInput = z.input<typeof InitEntityEngineRequestBody>;

26
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml
View file

@ -35,6 +35,32 @@ paths:
type: string
enrichPolicyExecutionInterval:
$ref: '../common.schema.yaml#/components/schemas/Interval'
lookbackPeriod:
type: string
default: 24h
pattern: '[smdh]$'
description: The amount of time the transform looks back to calculate the aggregations.
timeout:
type: string
default: 180s
pattern: '[smdh]$'
description: The timeout for initializing the aggregating transform.
frequency:
type: string
default: 1m
pattern: '[smdh]$'
description: The frequency at which the transform will run.
delay:
type: string
default: 1m
pattern: '[smdh]$'
description: The delay before the transform will run.
docsPerSecond:
type: integer
description: The number of documents per second to process.
responses:
'200':
description: Successful response

32
x-pack/solutions/security/plugins/security_solution/common/utils/objects/get.ts Normal file
View file

@ -0,0 +1,32 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import fp from 'lodash/fp';
import type { Get } from 'type-fest';
type Path = string | readonly string[];
/** Proxy for lodash fp `get` with better type inference.
* Overloaded to support both imperative and point free style
*
* Dynamic paths are supported if an array is passed as `path`.
* If an array is passed as `path`, it needs to be `const`: `["foo", "bar"] as const`
**/
function get<T extends object, const P extends Path>(obj: T, path: P): Get<T, P>;
function get<T extends object, const P extends Path>(path: P): (obj: T) => Get<T, P>;
function get<T extends object, const P extends Path>(...args: [T, P] | [P]) {
if (args.length === 2) {
const [obj, path] = args;
return fp.get(path)(obj);
}
const [path] = args;
return fp.get(path);
}
export { get };

28
x-pack/solutions/security/plugins/security_solution/common/utils/objects/merge.test.ts Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { merge } from './merge';
describe('merge', () => {
it('merges two objects', () => {
const target = { a: 1 };
const source = { b: 2 };
expect(merge(target, source)).toEqual({ a: 1, b: 2 });
});
it('merges two objects in point free style', () => {
const target = { a: 1 };
const source = { b: 2 };
expect(merge(source)(target)).toEqual({ a: 1, b: 2 });
});
it('overwrites target properties with source properties', () => {
const target = { a: 1, b: 2 };
const source = { b: 3 };
expect(merge(target, source)).toEqual({ a: 1, b: 3 });
});
});

30
x-pack/solutions/security/plugins/security_solution/common/utils/objects/merge.ts Normal file
View file

@ -0,0 +1,30 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import fp from 'lodash/fp';
import type { Expand } from './types';
interface Merge {
<Source extends object, Target extends object>(source: Source): (
target: Target
) => Expand<Target & Source>;
<Source extends object, Target extends object>(target: Target, source: Source): Expand<
Target & Source
>;
}
/**
* Proxy for lodash `merge` with better types
*/
export const merge: Merge = <S, T>(...args: [S] | [S, T]) => {
if (args.length === 2) {
const [target, source] = args;
return fp.merge(target, source);
}
const [source] = args;
return (target) => fp.merge(target, source);
};

41
x-pack/solutions/security/plugins/security_solution/common/utils/objects/modify.ts Normal file
View file

@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import fp from 'lodash/fp';
import type { Get, IsEqual } from 'type-fest';
import type { SetProp } from './set';
/**
* Proxy for lodash fp `update` with better type inference.
* We use `modify` to signal that the type of the object is being modified.
* Overloaded to support both imperative and point free style.
*/
function modify<T extends object, P extends string, A = Get<T, P>, B = A>(
path: P,
updater: (val: A) => B
): (
obj: T
) => IsEqual<A, unknown | never> extends true ? `Could not find path "${P}"` : SetProp<P, B, T>;
function modify<T extends object, P extends string, A = Get<T, P>, B = A>(
obj: T,
path: P,
updater: (val: A) => B
): IsEqual<A, unknown | never> extends true ? `Could not find path "${P}"` : SetProp<P, B, T>;
function modify<T extends object, P extends string, A = Get<T, P>, B = A>(
...args: [P, (val: A) => B] | [T, P, (val: A) => B]
) {
if (args.length === 3) {
const [obj, path, updater] = args;
return fp.update(path)(updater)(obj);
}
const [path, updater] = args;
return fp.update(path)(updater);
}
export { modify };

71
x-pack/solutions/security/plugins/security_solution/common/utils/objects/set.ts Normal file
View file

@ -0,0 +1,71 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import fp from 'lodash/fp';
import type { IsEqual, Get, EmptyObject } from 'type-fest';
import type { Expand } from './types';
type Path = string | readonly string[];
/** Proxy for lodash fp `set` with better type inference. Overloaded to support both imperative and point free style.
*
* If an invalid path is passed, the type of the `value` parameter is `never` which should cause a type error.
* Do **not** do an `as never` assertion to get around this, but instead make sure that the path is correct.
*
* Dynamic paths is supported if an array is passed as `path`.
* If an array is passed as `path`, it needs to be `const`: `["foo", "bar"] as const`
*/
function set<T extends object, P extends Path, V extends Get<T, P>>(
path: P,
value: IsEqual<Get<T, P>, unknown> extends true ? never : V
): (obj: T) => IsEqual<Get<T, P>, unknown> extends true ? unknown : T;
function set<T extends object, P extends Path, V extends Get<T, P>>(
obj: T,
path: P,
value: IsEqual<Get<T, P>, unknown> extends true ? never : V
): IsEqual<Get<T, P>, unknown> extends true ? unknown : T;
function set<T extends object, P extends Path, V extends Get<T, P>>(...args: [P, V] | [T, P, V]) {
if (args.length === 3) {
const [obj, path, value] = args;
return fp.set(path)(value)(obj);
}
const [path, value] = args;
return fp.set(path)(value);
}
export { set };
export type SetProp<P extends string, V, T extends object> = Expand<
P extends `${infer K}.${infer Tail}`
? K extends keyof T
? { [k in keyof T]: k extends K ? SetProp<Tail, V, EmptyObject> : T[k] }
: { [k in K]: SetProp<Tail, V, EmptyObject> } & (T extends EmptyObject ? {} : T)
: P extends keyof T
? { [k in keyof T]: k extends P ? V : T[k] }
: { [k in P]: V } & (T extends EmptyObject ? {} : T)
>;
export function setProp<T extends object, P extends string, V>(
obj: T,
path: P,
value: V
): SetProp<P, V, T>;
export function setProp<T extends object, P extends string, V>(
path: P,
value: V
): (obj: T) => SetProp<P, V, T>;
export function setProp<T extends object, P extends string, V>(...args: [T, P, V] | [P, V]) {
if (args.length === 3) {
const [obj, path, value] = args;
return fp.set(path, value)(obj);
}
const [path, value] = args;
return (obj: T) => fp.set(path, value)(obj);
}

12
x-pack/solutions/security/plugins/security_solution/common/utils/objects/types.ts Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/**
* Type utility to force typescript to early evaluate the type.
* This is useful for clarifying type computations
*/
export type Expand<T> = T extends unknown ? { [K in keyof T]: T[K] } : never;

22
x-pack/solutions/security/plugins/security_solution/common/utils/objects/update.test.ts Normal file
View file

@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { update } from './update';
describe('update', () => {
it('updates a nested property', () => {
const obj = { a: { b: 1 } };
const result = update(obj, 'a.b', (val) => val + 1);
expect(result).toEqual({ a: { b: 2 } });
});
it('updates a nested property in point free style', () => {
const obj = { a: { b: 1 } };
const result = update<typeof obj, 'a.b', number>('a.b', (val) => val + 1)(obj);
expect(result).toEqual({ a: { b: 2 } });
});
});

41
x-pack/solutions/security/plugins/security_solution/common/utils/objects/update.ts Normal file
View file

@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import fp from 'lodash/fp';
import type { Get, IsEqual } from 'type-fest';
type UpdaterFn<T extends object, P extends string, V extends Get<T, P>> = (
val: IsEqual<Get<T, P>, unknown> extends true ? never : V
) => IsEqual<Get<T, P>, unknown> extends true ? never : V;
/**
* Proxy for lodash fp `update` with better type inference.
* Overloaded to support both imperative and point free style.
*/
function update<T extends object, P extends string, V extends Get<T, P>>(
path: P,
updater: UpdaterFn<T, P, V>
): (obj: T) => IsEqual<Get<T, P>, unknown> extends true ? unknown : T;
function update<T extends object, P extends string, V extends Get<T, P>>(
obj: T,
path: P,
updater: UpdaterFn<T, P, V>
): IsEqual<Get<T, P>, unknown> extends true ? unknown : T;
function update<T extends object, P extends string, V extends Get<T, P>>(
...args: [P, UpdaterFn<T, P, V>] | [T, P, UpdaterFn<T, P, V>]
) {
if (args.length === 3) {
const [obj, path, updater] = args;
return fp.update(path)(updater)(obj);
}
const [path, updater] = args;
return fp.update(path)(updater);
}
export { update };

61
x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
View file

@ -307,6 +307,14 @@ paths:
schema:
type: object
properties:
delay:
default: 1m
description: The delay before the transform will run.
pattern: '[smdh]$'
type: string
docsPerSecond:
description: The number of documents per second to process.
type: integer
enrichPolicyExecutionInterval:
$ref: '#/components/schemas/Interval'
entityTypes:
@ -319,11 +327,23 @@ paths:
type: integer
filter:
type: string
frequency:
default: 1m
description: The frequency at which the transform will run.
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/IndexPattern'
lookbackPeriod:
default: 24h
description: The lookback period for the entity store
description: >-
The amount of time the transform looks back to calculate the
aggregations.
pattern: '[smdh]$'
type: string
timeout:
default: 180s
description: The timeout for initializing the aggregating transform.
pattern: '[smdh]$'
type: string
description: Schema for the entity store initialization
@ -429,6 +449,14 @@ paths:
schema:
type: object
properties:
delay:
default: 1m
description: The delay before the transform will run.
pattern: '[smdh]$'
type: string
docsPerSecond:
description: The number of documents per second to process.
type: integer
enrichPolicyExecutionInterval:
$ref: '#/components/schemas/Interval'
fieldHistoryLength:
@ -437,8 +465,25 @@ paths:
type: integer
filter:
type: string
frequency:
default: 1m
description: The frequency at which the transform will run.
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/IndexPattern'
lookbackPeriod:
default: 24h
description: >-
The amount of time the transform looks back to calculate the
aggregations.
pattern: '[smdh]$'
type: string
timeout:
default: 180s
description: The timeout for initializing the aggregating transform.
pattern: '[smdh]$'
type: string
description: Schema for the engine initialization
required: true
responses:
@ -1007,12 +1052,22 @@ components:
EngineDescriptor:
type: object
properties:
delay:
default: 1m
pattern: '[smdh]$'
type: string
docsPerSecond:
type: integer
error:
type: object
fieldHistoryLength:
type: integer
filter:
type: string
frequency:
default: 1m
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/IndexPattern'
lookbackPeriod:
@ -1021,6 +1076,10 @@ components:
type: string
status:
$ref: '#/components/schemas/EngineStatus'
timeout:
default: 180s
pattern: '[smdh]$'
type: string
type:
$ref: '#/components/schemas/EntityType'
required:

61
x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
View file

@ -307,6 +307,14 @@ paths:
schema:
type: object
properties:
delay:
default: 1m
description: The delay before the transform will run.
pattern: '[smdh]$'
type: string
docsPerSecond:
description: The number of documents per second to process.
type: integer
enrichPolicyExecutionInterval:
$ref: '#/components/schemas/Interval'
entityTypes:
@ -319,11 +327,23 @@ paths:
type: integer
filter:
type: string
frequency:
default: 1m
description: The frequency at which the transform will run.
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/IndexPattern'
lookbackPeriod:
default: 24h
description: The lookback period for the entity store
description: >-
The amount of time the transform looks back to calculate the
aggregations.
pattern: '[smdh]$'
type: string
timeout:
default: 180s
description: The timeout for initializing the aggregating transform.
pattern: '[smdh]$'
type: string
description: Schema for the entity store initialization
@ -429,6 +449,14 @@ paths:
schema:
type: object
properties:
delay:
default: 1m
description: The delay before the transform will run.
pattern: '[smdh]$'
type: string
docsPerSecond:
description: The number of documents per second to process.
type: integer
enrichPolicyExecutionInterval:
$ref: '#/components/schemas/Interval'
fieldHistoryLength:
@ -437,8 +465,25 @@ paths:
type: integer
filter:
type: string
frequency:
default: 1m
description: The frequency at which the transform will run.
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/IndexPattern'
lookbackPeriod:
default: 24h
description: >-
The amount of time the transform looks back to calculate the
aggregations.
pattern: '[smdh]$'
type: string
timeout:
default: 180s
description: The timeout for initializing the aggregating transform.
pattern: '[smdh]$'
type: string
description: Schema for the engine initialization
required: true
responses:
@ -1007,12 +1052,22 @@ components:
EngineDescriptor:
type: object
properties:
delay:
default: 1m
pattern: '[smdh]$'
type: string
docsPerSecond:
type: integer
error:
type: object
fieldHistoryLength:
type: integer
filter:
type: string
frequency:
default: 1m
pattern: '[smdh]$'
type: string
indexPattern:
$ref: '#/components/schemas/IndexPattern'
lookbackPeriod:
@ -1021,6 +1076,10 @@ components:
type: string
status:
$ref: '#/components/schemas/EngineStatus'
timeout:
default: 180s
pattern: '[smdh]$'
type: string
type:
$ref: '#/components/schemas/EntityType'
required:

4
x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/entity_store/components/engines_status/components/engine_status_header_action.test.tsx
View file

@ -14,6 +14,7 @@ import type { GetEntityStoreStatusResponse } from '../../../../../../../common/a
import { EntityType } from '../../../../../../../common/entity_analytics/types';
import { TestProviders } from '../../../../../../common/mock';
import type { EngineComponentStatus } from '../../../../../../../common/api/entity_analytics';
import { defaultOptions } from '../../../../../../../server/lib/entity_analytics/entity_store/constants';
jest.mock('../../../hooks/use_entity_store');
jest.mock('../helpers');
@ -28,12 +29,11 @@ const defaultComponent: EngineComponentStatus = {
};
const defaultEngineResponse: GetEntityStoreStatusResponse['engines'][0] = {
...defaultOptions,
type: EntityType.user,
indexPattern: '',
status: 'started',
fieldHistoryLength: 0,
components: [defaultComponent],
lookbackPeriod: '',
};
describe('EngineStatusHeaderAction', () => {

31
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts
View file

@ -5,9 +5,38 @@
* 2.0.
*/
import type { EngineStatus, StoreStatus } from '../../../../common/api/entity_analytics';
import type { Expand } from '../../../../common/utils/objects/types';
import type {
EngineStatus,
InitEntityEngineRequestBody,
StoreStatus,
} from '../../../../common/api/entity_analytics';
import { DEFAULT_INTERVAL } from './tasks/field_retention_enrichment/constants';
export const DEFAULT_LOOKBACK_PERIOD = '24h';
export const DEFAULT_FIELD_HISTORY_LENGTH = 10;
export const DEFAULT_SYNC_DELAY = '1m';
export const DEFAULT_TIMEOUT = '180s';
export const DEFAULT_FREQUENCY = '1m';
export const DEFAULT_DOCS_PER_SECOND = undefined;
export const DEFAULT_INDEX_PATTERNS = '';
export const DEFAULT_KQL_FILTER = '';
export const defaultOptions: Expand<
Required<Omit<InitEntityEngineRequestBody, 'docsPerSecond'>> & {
docsPerSecond: number | undefined;
}
> = {
delay: DEFAULT_SYNC_DELAY,
timeout: DEFAULT_TIMEOUT,
frequency: DEFAULT_FREQUENCY,
docsPerSecond: DEFAULT_DOCS_PER_SECOND,
lookbackPeriod: DEFAULT_LOOKBACK_PERIOD,
fieldHistoryLength: DEFAULT_FIELD_HISTORY_LENGTH,
indexPattern: DEFAULT_INDEX_PATTERNS,
filter: DEFAULT_KQL_FILTER,
enrichPolicyExecutionInterval: DEFAULT_INTERVAL,
};
export const ENGINE_STATUS: Record<Uppercase<EngineStatus>, EngineStatus> = {
INSTALLING: 'installing',

2
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/entity_index.ts
View file

@ -65,7 +65,7 @@ export const getEntityIndexStatus = async ({
export type MappingProperties = NonNullable<MappingTypeMapping['properties']>;
export const generateIndexMappings = (
description: EntityEngineInstallationDescriptor
description: Pick<EntityEngineInstallationDescriptor, 'fields' | 'identityField'>
): MappingTypeMapping => {
const identityFieldMappings: MappingProperties = {
[description.identityField]: {

2
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_manager_conversion.ts
View file

@ -31,6 +31,8 @@ export const convertToEntityManagerDefinition = (
syncField: description.settings.timestampField,
syncDelay: description.settings.syncDelay,
frequency: description.settings.frequency,
timeout: description.settings.timeout,
docsPerSecond: description.settings.docsPerSecond,
},
},
version: description.version,

2
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/types.ts
View file

@ -24,5 +24,5 @@ export type EntityDescription = PickPartial<
| 'settings'
| 'pipeline'
| 'dynamic',
'indexPatterns' | 'indexMappings' | 'settings' | 'pipeline' | 'dynamic'
'indexPatterns' | 'indexMappings' | 'settings' | 'dynamic'
>;

7
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts
View file

@ -21,6 +21,7 @@ import { convertToEntityManagerDefinition } from './entity_definitions/entity_ma
import { EntityType } from '../../../../common/search_strategy';
import type { InitEntityEngineResponse } from '../../../../common/api/entity_analytics';
import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
import { defaultOptions } from './constants';
const definition: EntityDefinition = convertToEntityManagerDefinition(
{
@ -33,6 +34,8 @@ const definition: EntityDefinition = convertToEntityManagerDefinition(
indexMappings: {},
indexPatterns: [],
settings: {
timeout: '180s',
docsPerSecond: undefined,
syncDelay: '1m',
frequency: '1m',
timestampField: '@timestamp',
@ -354,8 +357,8 @@ describe('EntityStoreDataClient', () => {
it('only enable engine for the given entityType', async () => {
await dataClient.enable({
...defaultOptions,
entityTypes: [EntityType.host],
fieldHistoryLength: 1,
});
expect(spyInit).toHaveBeenCalledWith(EntityType.host, expect.anything(), expect.anything());
@ -363,8 +366,8 @@ describe('EntityStoreDataClient', () => {
it('does not enable engine when the given entity type is disabled', async () => {
await dataClient.enable({
...defaultOptions,
entityTypes: [EntityType.universal],
fieldHistoryLength: 1,
});
expect(spyInit).not.toHaveBeenCalled();

96
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
View file

@ -23,6 +23,7 @@ import moment from 'moment';
import type { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types';
import type { EntityDefinition } from '@kbn/entities-schema';
import type { estypes } from '@elastic/elasticsearch';
import { merge } from '../../../../common/utils/objects/merge';
import { getEnabledStoreEntityTypes } from '../../../../common/entity_analytics/entity_store/utils';
import { EntityType } from '../../../../common/entity_analytics/types';
import type { ExperimentalFeatures } from '../../../../common';
@ -47,7 +48,12 @@ import type {
EngineComponentResource,
} from '../../../../common/api/entity_analytics';
import { EngineDescriptorClient } from './saved_object/engine_descriptor';
import { ENGINE_STATUS, ENTITY_STORE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
import {
ENGINE_STATUS,
ENTITY_STORE_STATUS,
MAX_SEARCH_RESPONSE_SIZE,
defaultOptions,
} from './constants';
import { AssetCriticalityMigrationClient } from '../asset_criticality/asset_criticality_migration_client';
import {
startEntityStoreFieldRetentionEnrichTask,
@ -94,7 +100,7 @@ import {
createKeywordBuilderPipeline,
deleteKeywordBuilderPipeline,
} from '../../asset_inventory/ingest_pipelines';
import { DEFAULT_INTERVAL } from './tasks/field_retention_enrichment/constants';
import type { ApiKeyManager } from './auth/api_key';
// Workaround. TransformState type is wrong. The health type should be: TransformHealth from '@kbn/transform-plugin/common/types/transform_stats'
@ -135,18 +141,6 @@ interface SearchEntitiesParams {
sortOrder: SortOrder;
}
export const DEFAULT_INIT_ENTITY_STORE: InitEntityStoreRequestBody = {
indexPattern: '',
lookbackPeriod: '24h',
filter: '',
fieldHistoryLength: 10,
enrichPolicyExecutionInterval: DEFAULT_INTERVAL,
};
const DEFAULT_ENTITY_ENGINE: InitEntityEngineRequestBody & { lookbackPeriod?: string } = {
...DEFAULT_INIT_ENTITY_STORE,
};
export class EntityStoreDataClient {
private engineClient: EngineDescriptorClient;
private assetCriticalityMigrationClient: AssetCriticalityMigrationClient;
@ -234,25 +228,13 @@ export class EntityStoreDataClient {
}
public async enable(
requestBodyOverrides: Partial<InitEntityStoreRequestBody> = {},
requestBodyOverrides: InitEntityStoreRequestBody,
{ pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
): Promise<InitEntityStoreResponse> {
if (!this.options.taskManager) {
throw new Error('Task Manager is not available');
}
const {
indexPattern,
lookbackPeriod,
filter,
fieldHistoryLength,
entityTypes,
enrichPolicyExecutionInterval,
} = {
...DEFAULT_INIT_ENTITY_STORE,
...requestBodyOverrides,
};
// Immediately defer the initialization to the next tick. This way we don't block on the init preflight checks
const run = <T>(fn: () => Promise<T>) =>
new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
@ -261,24 +243,14 @@ export class EntityStoreDataClient {
const enabledEntityTypes = getEnabledStoreEntityTypes(experimentalFeatures);
// When entityTypes param is defined it only enables the engines that are provided
const enginesTypes = entityTypes
? (entityTypes as EntityType[]).filter((type) => enabledEntityTypes.includes(type))
const enginesTypes = requestBodyOverrides.entityTypes
? (requestBodyOverrides.entityTypes as EntityType[]).filter((type) =>
enabledEntityTypes.includes(type)
)
: enabledEntityTypes;
const promises = enginesTypes.map((entity) =>
run(() =>
this.init(
entity,
{
indexPattern,
lookbackPeriod,
filter,
fieldHistoryLength,
enrichPolicyExecutionInterval,
},
{ pipelineDebugMode }
)
)
run(() => this.init(entity, requestBodyOverrides, { pipelineDebugMode }))
);
const engines = await Promise.all(promises);
@ -335,21 +307,9 @@ export class EntityStoreDataClient {
public async init(
entityType: EntityType,
InitEntityEngineRequestBodyOverrides: Partial<typeof DEFAULT_ENTITY_ENGINE> = {},
requestBody: InitEntityEngineRequestBody,
{ pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
): Promise<InitEntityEngineResponse> {
const mergedRequest = {
...DEFAULT_ENTITY_ENGINE,
...InitEntityEngineRequestBodyOverrides,
} as Required<typeof DEFAULT_ENTITY_ENGINE>;
const {
indexPattern,
filter,
fieldHistoryLength,
lookbackPeriod,
enrichPolicyExecutionInterval,
} = mergedRequest;
const { experimentalFeatures } = this.options;
if (entityType === EntityType.universal && !experimentalFeatures.assetInventoryStoreEnabled) {
@ -394,23 +354,14 @@ export class EntityStoreDataClient {
'Initializing entity engine'
);
const descriptor = await this.engineClient.init(entityType, {
filter,
fieldHistoryLength,
lookbackPeriod,
indexPattern,
});
const descriptor = await this.engineClient.init(entityType, requestBody);
this.log('debug', entityType, `Initialized engine saved object`);
this.asyncSetup(
entityType,
fieldHistoryLength,
lookbackPeriod,
enrichPolicyExecutionInterval,
this.options.taskManager,
indexPattern,
filter,
config,
requestBody,
pipelineDebugMode
).catch((e) =>
this.log('error', entityType, `Error during async setup of entity store: ${e.message}`)
@ -421,24 +372,21 @@ export class EntityStoreDataClient {
private async asyncSetup(
entityType: EntityType,
fieldHistoryLength: number,
lookbackPeriod: string,
enrichPolicyExecutionInterval: string,
taskManager: TaskManagerStartContract,
indexPattern: string,
filter: string,
config: EntityStoreConfig,
requestParams: InitEntityEngineRequestBody,
pipelineDebugMode: boolean
) {
const setupStartTime = moment().utc().toISOString();
const { logger, namespace, appClient, dataViewsService } = this.options;
try {
const defaultIndexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
const options = merge(defaultOptions, requestParams);
const description = createEngineDescription({
entityType,
namespace,
requestParams: { indexPattern, fieldHistoryLength, lookbackPeriod },
requestParams,
defaultIndexPatterns,
config,
});
@ -453,7 +401,7 @@ export class EntityStoreDataClient {
// set up the entity manager definition
const definition = convertToEntityManagerDefinition(description, {
namespace,
filter,
filter: options.filter,
});
await this.entityClient.createEntityDefinition({
@ -516,7 +464,7 @@ export class EntityStoreDataClient {
namespace,
logger,
taskManager,
interval: enrichPolicyExecutionInterval,
interval: options.enrichPolicyExecutionInterval,
});
// this task will continuously refresh the Entity Store indices based on the Data View

662
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/__snapshots__/engine_description.test.ts.snap generated Normal file
View file

@ -0,0 +1,662 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`getUnitedEntityDefinition host entityManagerDefinition 1`] = `
Object {
"displayNameTemplate": "{{host.name}}",
"id": "security_host_test",
"identityFields": Array [
Object {
"field": "host.name",
"optional": false,
},
],
"indexPatterns": Array [
"test*",
],
"latest": Object {
"lookbackPeriod": "24h",
"settings": Object {
"docsPerSecond": undefined,
"frequency": "1m",
"syncDelay": "1m",
"syncField": "@timestamp",
"timeout": "180s",
},
"timestampField": "@timestamp",
},
"managed": true,
"metadata": Array [
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.domain",
"source": "host.domain",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.hostname",
"source": "host.hostname",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.id",
"source": "host.id",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.os.name",
"source": "host.os.name",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.os.type",
"source": "host.os.type",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.ip",
"source": "host.ip",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.mac",
"source": "host.mac",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.type",
"source": "host.type",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.architecture",
"source": "host.architecture",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "asc",
},
"type": "top_value",
},
"destination": "entity.source",
"source": "_index",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "asset.criticality",
"source": "asset.criticality",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "host.risk.calculated_level",
"source": "host.risk.calculated_level",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "host.risk.calculated_score",
"source": "host.risk.calculated_score",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "host.risk.calculated_score_norm",
"source": "host.risk.calculated_score_norm",
},
],
"name": "Security 'host' Entity Store Definition",
"type": "host",
"version": "1.0.0",
}
`;
exports[`getUnitedEntityDefinition host mapping 1`] = `
Object {
"properties": Object {
"@timestamp": Object {
"type": "date",
},
"asset.criticality": Object {
"type": "keyword",
},
"entity.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"entity.source": Object {
"type": "keyword",
},
"host.architecture": Object {
"type": "keyword",
},
"host.domain": Object {
"type": "keyword",
},
"host.hostname": Object {
"type": "keyword",
},
"host.id": Object {
"type": "keyword",
},
"host.ip": Object {
"type": "ip",
},
"host.mac": Object {
"type": "keyword",
},
"host.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"host.os.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"host.os.type": Object {
"type": "keyword",
},
"host.risk.calculated_level": Object {
"type": "keyword",
},
"host.risk.calculated_score": Object {
"type": "float",
},
"host.risk.calculated_score_norm": Object {
"type": "float",
},
"host.type": Object {
"type": "keyword",
},
},
}
`;
exports[`getUnitedEntityDefinition service entityManagerDefinition 1`] = `
Object {
"displayNameTemplate": "{{service.name}}",
"id": "security_service_test",
"identityFields": Array [
Object {
"field": "service.name",
"optional": false,
},
],
"indexPatterns": Array [
"test*",
],
"latest": Object {
"lookbackPeriod": "24h",
"settings": Object {
"docsPerSecond": undefined,
"frequency": "1m",
"syncDelay": "1m",
"syncField": "@timestamp",
"timeout": "180s",
},
"timestampField": "@timestamp",
},
"managed": true,
"metadata": Array [
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.address",
"source": "service.address",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.environment",
"source": "service.environment",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.ephemeral_id",
"source": "service.ephemeral_id",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.id",
"source": "service.id",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.node.name",
"source": "service.node.name",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.node.roles",
"source": "service.node.roles",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.node.role",
"source": "service.node.role",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.state",
"source": "service.state",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.type",
"source": "service.type",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.version",
"source": "service.version",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "asc",
},
"type": "top_value",
},
"destination": "entity.source",
"source": "_index",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "asset.criticality",
"source": "asset.criticality",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_level",
"source": "service.risk.calculated_level",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_score",
"source": "service.risk.calculated_score",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_score_norm",
"source": "service.risk.calculated_score_norm",
},
],
"name": "Security 'service' Entity Store Definition",
"type": "service",
"version": "1.0.0",
}
`;
exports[`getUnitedEntityDefinition service mapping 1`] = `
Object {
"properties": Object {
"@timestamp": Object {
"type": "date",
},
"asset.criticality": Object {
"type": "keyword",
},
"entity.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"entity.source": Object {
"type": "keyword",
},
"service.address": Object {
"type": "keyword",
},
"service.environment": Object {
"type": "keyword",
},
"service.ephemeral_id": Object {
"type": "keyword",
},
"service.id": Object {
"type": "keyword",
},
"service.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"service.node.name": Object {
"type": "keyword",
},
"service.node.role": Object {
"type": "keyword",
},
"service.node.roles": Object {
"type": "keyword",
},
"service.risk.calculated_level": Object {
"type": "keyword",
},
"service.risk.calculated_score": Object {
"type": "float",
},
"service.risk.calculated_score_norm": Object {
"type": "float",
},
"service.state": Object {
"type": "keyword",
},
"service.type": Object {
"type": "keyword",
},
"service.version": Object {
"type": "keyword",
},
},
}
`;
exports[`getUnitedEntityDefinition user entityManagerDefinition 1`] = `
Object {
"displayNameTemplate": "{{user.name}}",
"id": "security_user_test",
"identityFields": Array [
Object {
"field": "user.name",
"optional": false,
},
],
"indexPatterns": Array [
"test*",
],
"latest": Object {
"lookbackPeriod": "24h",
"settings": Object {
"docsPerSecond": undefined,
"frequency": "1m",
"syncDelay": "1m",
"syncField": "@timestamp",
"timeout": "180s",
},
"timestampField": "@timestamp",
},
"managed": true,
"metadata": Array [
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.domain",
"source": "user.domain",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.email",
"source": "user.email",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.full_name",
"source": "user.full_name",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.hash",
"source": "user.hash",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.id",
"source": "user.id",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.roles",
"source": "user.roles",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "asc",
},
"type": "top_value",
},
"destination": "entity.source",
"source": "_index",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "asset.criticality",
"source": "asset.criticality",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "user.risk.calculated_level",
"source": "user.risk.calculated_level",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "user.risk.calculated_score",
"source": "user.risk.calculated_score",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "user.risk.calculated_score_norm",
"source": "user.risk.calculated_score_norm",
},
],
"name": "Security 'user' Entity Store Definition",
"type": "user",
"version": "1.0.0",
}
`;
exports[`getUnitedEntityDefinition user mapping 1`] = `
Object {
"properties": Object {
"@timestamp": Object {
"type": "date",
},
"asset.criticality": Object {
"type": "keyword",
},
"entity.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"entity.source": Object {
"type": "keyword",
},
"user.domain": Object {
"type": "keyword",
},
"user.email": Object {
"type": "keyword",
},
"user.full_name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"user.hash": Object {
"type": "keyword",
},
"user.id": Object {
"type": "keyword",
},
"user.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"user.risk.calculated_level": Object {
"type": "keyword",
},
"user.risk.calculated_score": Object {
"type": "float",
},
"user.risk.calculated_score_norm": Object {
"type": "float",
},
"user.roles": Object {
"type": "keyword",
},
},
}
`;

668
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.test.ts
View file

@ -8,6 +8,7 @@
import { duration } from 'moment';
import { createEngineDescription } from './engine_description';
import { convertToEntityManagerDefinition } from '../entity_definitions/entity_manager_conversion';
import { defaultOptions } from '../constants';
describe('getUnitedEntityDefinition', () => {
const defaultIndexPatterns = ['test*'];
@ -15,9 +16,7 @@ describe('getUnitedEntityDefinition', () => {
const description = createEngineDescription({
entityType: 'host',
namespace: 'test',
requestParams: {
fieldHistoryLength: 10,
},
requestParams: defaultOptions,
defaultIndexPatterns,
config: {
syncDelay: duration(60, 'seconds'),
@ -27,78 +26,7 @@ describe('getUnitedEntityDefinition', () => {
});
it('mapping', () => {
expect(description.indexMappings).toMatchInlineSnapshot(`
Object {
"properties": Object {
"@timestamp": Object {
"type": "date",
},
"asset.criticality": Object {
"type": "keyword",
},
"entity.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"entity.source": Object {
"type": "keyword",
},
"host.architecture": Object {
"type": "keyword",
},
"host.domain": Object {
"type": "keyword",
},
"host.hostname": Object {
"type": "keyword",
},
"host.id": Object {
"type": "keyword",
},
"host.ip": Object {
"type": "ip",
},
"host.mac": Object {
"type": "keyword",
},
"host.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"host.os.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"host.os.type": Object {
"type": "keyword",
},
"host.risk.calculated_level": Object {
"type": "keyword",
},
"host.risk.calculated_score": Object {
"type": "float",
},
"host.risk.calculated_score_norm": Object {
"type": "float",
},
"host.type": Object {
"type": "keyword",
},
},
}
`);
expect(description.indexMappings).toMatchSnapshot();
});
it('entityManagerDefinition', () => {
@ -107,167 +35,14 @@ describe('getUnitedEntityDefinition', () => {
filter: '',
});
expect(entityManagerDefinition).toMatchInlineSnapshot(`
Object {
"displayNameTemplate": "{{host.name}}",
"id": "security_host_test",
"identityFields": Array [
Object {
"field": "host.name",
"optional": false,
},
],
"indexPatterns": Array [
"test*",
],
"latest": Object {
"lookbackPeriod": "1d",
"settings": Object {
"frequency": "60s",
"syncDelay": "60s",
"syncField": "@timestamp",
},
"timestampField": "@timestamp",
},
"managed": true,
"metadata": Array [
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.domain",
"source": "host.domain",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.hostname",
"source": "host.hostname",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.id",
"source": "host.id",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.os.name",
"source": "host.os.name",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.os.type",
"source": "host.os.type",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.ip",
"source": "host.ip",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.mac",
"source": "host.mac",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.type",
"source": "host.type",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "host.architecture",
"source": "host.architecture",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "asc",
},
"type": "top_value",
},
"destination": "entity.source",
"source": "_index",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "asset.criticality",
"source": "asset.criticality",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "host.risk.calculated_level",
"source": "host.risk.calculated_level",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "host.risk.calculated_score",
"source": "host.risk.calculated_score",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "host.risk.calculated_score_norm",
"source": "host.risk.calculated_score_norm",
},
],
"name": "Security 'host' Entity Store Definition",
"type": "host",
"version": "1.0.0",
}
`);
expect(entityManagerDefinition).toMatchSnapshot();
});
});
describe('user', () => {
const description = createEngineDescription({
entityType: 'user',
namespace: 'test',
requestParams: {
fieldHistoryLength: 10,
},
requestParams: defaultOptions,
defaultIndexPatterns,
config: {
syncDelay: duration(60, 'seconds'),
@ -277,203 +52,14 @@ describe('getUnitedEntityDefinition', () => {
});
it('mapping', () => {
expect(description.indexMappings).toMatchInlineSnapshot(`
Object {
"properties": Object {
"@timestamp": Object {
"type": "date",
},
"asset.criticality": Object {
"type": "keyword",
},
"entity.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"entity.source": Object {
"type": "keyword",
},
"user.domain": Object {
"type": "keyword",
},
"user.email": Object {
"type": "keyword",
},
"user.full_name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"user.hash": Object {
"type": "keyword",
},
"user.id": Object {
"type": "keyword",
},
"user.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"user.risk.calculated_level": Object {
"type": "keyword",
},
"user.risk.calculated_score": Object {
"type": "float",
},
"user.risk.calculated_score_norm": Object {
"type": "float",
},
"user.roles": Object {
"type": "keyword",
},
},
}
`);
expect(description.indexMappings).toMatchSnapshot();
});
it('entityManagerDefinition', () => {
const entityManagerDefinition = convertToEntityManagerDefinition(description, {
namespace: 'test',
filter: '',
});
expect(entityManagerDefinition).toMatchInlineSnapshot(`
Object {
"displayNameTemplate": "{{user.name}}",
"id": "security_user_test",
"identityFields": Array [
Object {
"field": "user.name",
"optional": false,
},
],
"indexPatterns": Array [
"test*",
],
"latest": Object {
"lookbackPeriod": "1d",
"settings": Object {
"frequency": "60s",
"syncDelay": "60s",
"syncField": "@timestamp",
},
"timestampField": "@timestamp",
},
"managed": true,
"metadata": Array [
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.domain",
"source": "user.domain",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.email",
"source": "user.email",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.full_name",
"source": "user.full_name",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.hash",
"source": "user.hash",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.id",
"source": "user.id",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "user.roles",
"source": "user.roles",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "asc",
},
"type": "top_value",
},
"destination": "entity.source",
"source": "_index",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "asset.criticality",
"source": "asset.criticality",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "user.risk.calculated_level",
"source": "user.risk.calculated_level",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "user.risk.calculated_score",
"source": "user.risk.calculated_score",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "user.risk.calculated_score_norm",
"source": "user.risk.calculated_score_norm",
},
],
"name": "Security 'user' Entity Store Definition",
"type": "user",
"version": "1.0.0",
}
`);
expect(entityManagerDefinition).toMatchSnapshot();
});
});
@ -481,9 +67,7 @@ describe('getUnitedEntityDefinition', () => {
const description = createEngineDescription({
entityType: 'service',
namespace: 'test',
requestParams: {
fieldHistoryLength: 10,
},
requestParams: defaultOptions,
defaultIndexPatterns,
config: {
syncDelay: duration(60, 'seconds'),
@ -493,76 +77,7 @@ describe('getUnitedEntityDefinition', () => {
});
it('mapping', () => {
expect(description.indexMappings).toMatchInlineSnapshot(`
Object {
"properties": Object {
"@timestamp": Object {
"type": "date",
},
"asset.criticality": Object {
"type": "keyword",
},
"entity.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"entity.source": Object {
"type": "keyword",
},
"service.address": Object {
"type": "keyword",
},
"service.environment": Object {
"type": "keyword",
},
"service.ephemeral_id": Object {
"type": "keyword",
},
"service.id": Object {
"type": "keyword",
},
"service.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"service.node.name": Object {
"type": "keyword",
},
"service.node.role": Object {
"type": "keyword",
},
"service.node.roles": Object {
"type": "keyword",
},
"service.risk.calculated_level": Object {
"type": "keyword",
},
"service.risk.calculated_score": Object {
"type": "float",
},
"service.risk.calculated_score_norm": Object {
"type": "float",
},
"service.state": Object {
"type": "keyword",
},
"service.type": Object {
"type": "keyword",
},
"service.version": Object {
"type": "keyword",
},
},
}
`);
expect(description.indexMappings).toMatchSnapshot();
});
it('entityManagerDefinition', () => {
@ -570,170 +85,7 @@ describe('getUnitedEntityDefinition', () => {
namespace: 'test',
filter: '',
});
expect(entityManagerDefinition).toMatchInlineSnapshot(`
Object {
"displayNameTemplate": "{{service.name}}",
"id": "security_service_test",
"identityFields": Array [
Object {
"field": "service.name",
"optional": false,
},
],
"indexPatterns": Array [
"test*",
],
"latest": Object {
"lookbackPeriod": "1d",
"settings": Object {
"frequency": "60s",
"syncDelay": "60s",
"syncField": "@timestamp",
},
"timestampField": "@timestamp",
},
"managed": true,
"metadata": Array [
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.address",
"source": "service.address",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.environment",
"source": "service.environment",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.ephemeral_id",
"source": "service.ephemeral_id",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.id",
"source": "service.id",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.node.name",
"source": "service.node.name",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.node.roles",
"source": "service.node.roles",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.node.role",
"source": "service.node.role",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.state",
"source": "service.state",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.type",
"source": "service.type",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.version",
"source": "service.version",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "asc",
},
"type": "top_value",
},
"destination": "entity.source",
"source": "_index",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "asset.criticality",
"source": "asset.criticality",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_level",
"source": "service.risk.calculated_level",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_score",
"source": "service.risk.calculated_score",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_score_norm",
"source": "service.risk.calculated_score_norm",
},
],
"name": "Security 'service' Entity Store Definition",
"type": "service",
"version": "1.0.0",
}
`);
expect(entityManagerDefinition).toMatchSnapshot();
});
});
});

96
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts
View file

@ -4,17 +4,14 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { pipe } from 'fp-ts/lib/function';
import { assign, concat, map, merge, update } from 'lodash/fp';
import { set } from '@kbn/safer-lodash-set/fp';
import { assign, concat } from 'lodash/fp';
import type { EntityType } from '../../../../../common/api/entity_analytics';
import {
DEFAULT_FIELD_HISTORY_LENGTH,
DEFAULT_LOOKBACK_PERIOD,
DEFAULT_TIMESTAMP_FIELD,
} from '../entity_definitions/constants';
import type {
EntityType,
InitEntityEngineRequestBody,
} from '../../../../../common/api/entity_analytics';
import { DEFAULT_TIMESTAMP_FIELD } from '../entity_definitions/constants';
import { generateIndexMappings } from '../elasticsearch_assets';
import {
hostEntityEngineDescription,
@ -28,6 +25,9 @@ import { buildEntityDefinitionId } from '../utils';
import type { EntityDescription } from '../entity_definitions/types';
import type { EntityEngineInstallationDescriptor } from './types';
import { merge } from '../../../../../common/utils/objects/merge';
import { defaultOptions } from '../constants';
const engineDescriptionRegistry: Record<EntityType, EntityDescription> = {
host: hostEntityEngineDescription,
user: userEntityEngineDescription,
@ -39,62 +39,52 @@ interface EngineDescriptionParams {
entityType: EntityType;
namespace: string;
config: EntityStoreConfig;
requestParams?: {
indexPattern?: string;
fieldHistoryLength?: number;
lookbackPeriod?: string;
};
requestParams?: InitEntityEngineRequestBody;
defaultIndexPatterns: string[];
}
export const createEngineDescription = (options: EngineDescriptionParams) => {
const { entityType, namespace, config, requestParams, defaultIndexPatterns } = options;
const fieldHistoryLength = requestParams?.fieldHistoryLength || DEFAULT_FIELD_HISTORY_LENGTH;
export const createEngineDescription = (params: EngineDescriptionParams) => {
const { entityType, namespace, config, requestParams = {}, defaultIndexPatterns } = params;
const indexPatterns = requestParams?.indexPattern
? defaultIndexPatterns.concat(requestParams?.indexPattern.split(','))
const fileConfig = {
delay: `${config.syncDelay.asSeconds()}s`,
frequency: `${config.frequency.asSeconds()}s`,
};
const options = merge(defaultOptions, merge(fileConfig, requestParams));
const indexPatterns = options.indexPattern
? defaultIndexPatterns.concat(options.indexPattern.split(','))
: defaultIndexPatterns;
const description = engineDescriptionRegistry[entityType];
const settings: EntityEngineInstallationDescriptor['settings'] = {
syncDelay: `${config.syncDelay.asSeconds()}s`,
frequency: `${config.frequency.asSeconds()}s`,
lookbackPeriod:
requestParams?.lookbackPeriod ||
description.settings?.lookbackPeriod ||
DEFAULT_LOOKBACK_PERIOD,
syncDelay: options.delay,
timeout: options.timeout,
frequency: options.frequency,
docsPerSecond: options.docsPerSecond,
lookbackPeriod: options.lookbackPeriod,
timestampField: description.settings?.timestampField || DEFAULT_TIMESTAMP_FIELD,
};
const updatedDescription = pipe(
description,
set('id', buildEntityDefinitionId(entityType, namespace)),
update('settings', assign(settings)),
updateIndexPatterns(indexPatterns),
updateRetentionFields(fieldHistoryLength),
setDefaultDynamic,
addIndexMappings
) as EntityEngineInstallationDescriptor;
const defaults = {
...description,
id: buildEntityDefinitionId(entityType, namespace),
settings: assign(settings, description.settings),
indexPatterns: concat(indexPatterns, (description.indexPatterns || []) as string[]),
fields: description.fields.map(
merge({
retention: { maxLength: options.fieldHistoryLength },
aggregation: { limit: options.fieldHistoryLength },
})
),
dynamic: description.dynamic || false,
};
const updatedDescription: EntityEngineInstallationDescriptor = {
...defaults,
indexMappings: generateIndexMappings(defaults),
};
return updatedDescription;
};
const updateIndexPatterns = (indexPatterns: string[]) =>
update('indexPatterns', (prev = []) => concat(indexPatterns, prev));
const updateRetentionFields = (fieldHistoryLength: number) =>
update(
'fields',
map(
merge({
retention: { maxLength: fieldHistoryLength },
aggregation: { limit: fieldHistoryLength },
})
)
);
const addIndexMappings = (description: EntityEngineInstallationDescriptor) =>
set('indexMappings', generateIndexMappings(description), description);
const setDefaultDynamic = update('dynamic', (dynamic = false) => dynamic);

4
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/types.ts
View file

@ -47,6 +47,8 @@ export interface EntityEngineInstallationDescriptor {
settings: {
syncDelay: string;
frequency: string;
timeout: string;
docsPerSecond?: number;
lookbackPeriod: string;
timestampField: string;
};
@ -56,7 +58,7 @@ export interface EntityEngineInstallationDescriptor {
* This can be an array of processors which get appended to the default pipeline,
* or a function that takes the default processors and returns an array of processors.
**/
pipeline:
pipeline?:
| IngestProcessorContainer[]
| ((defaultProcessors: IngestProcessorContainer[]) => IngestProcessorContainer[]);

27
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor.ts
View file

@ -9,6 +9,8 @@ import type {
SavedObjectsClientContract,
SavedObjectsFindResponse,
} from '@kbn/core-saved-objects-api-server';
import { merge } from 'lodash/fp';
import type { InitEntityEngineRequestBody } from '../../../../../common/api/entity_analytics/entity_store/engine/init.gen';
import type {
EngineDescriptor,
EngineStatus,
@ -17,13 +19,15 @@ import type {
import { entityEngineDescriptorTypeName } from './engine_descriptor_type';
import { getByEntityTypeQuery } from '../utils';
import { ENGINE_STATUS } from '../constants';
import { ENGINE_STATUS, defaultOptions } from '../constants';
interface EngineDescriptorDependencies {
soClient: SavedObjectsClientContract;
namespace: string;
}
type InitOptions = InitEntityEngineRequestBody;
export class EngineDescriptorClient {
constructor(private readonly deps: EngineDescriptorDependencies) {}
@ -31,15 +35,8 @@ export class EngineDescriptorClient {
return `entity-engine-descriptor-${entityType}-${this.deps.namespace}`;
}
async init(
entityType: EntityType,
{
filter,
fieldHistoryLength,
indexPattern,
lookbackPeriod,
}: { filter: string; fieldHistoryLength: number; indexPattern: string; lookbackPeriod: string }
) {
async init(entityType: EntityType, options: InitOptions) {
const opts: typeof defaultOptions = merge(defaultOptions, options);
const engineDescriptor = await this.find(entityType);
if (engineDescriptor.total > 1) {
@ -52,10 +49,7 @@ export class EngineDescriptorClient {
...old,
error: undefined, // if the engine is being re-initialized, clear any previous error
status: ENGINE_STATUS.INSTALLING,
filter,
fieldHistoryLength,
indexPattern,
lookbackPeriod,
...opts,
};
await this.deps.soClient.update<EngineDescriptor>(
entityEngineDescriptorTypeName,
@ -72,10 +66,7 @@ export class EngineDescriptorClient {
{
status: ENGINE_STATUS.INSTALLING,
type: entityType,
indexPattern,
filter,
fieldHistoryLength,
lookbackPeriod,
...opts,
},
{ id: this.getSavedObjectId(entityType) }
);

19
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor_type.ts
View file

@ -8,6 +8,7 @@
import type { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server';
import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
import type { SavedObjectsType } from '@kbn/core/server';
import { defaultOptions } from '../constants';
export const entityEngineDescriptorTypeName = 'entity-engine-status';
@ -55,11 +56,27 @@ const version1: SavedObjectsModelVersion = {
],
};
const version2: SavedObjectsModelVersion = {
changes: [
{
type: 'data_backfill',
backfillFn: (document) => {
return {
attributes: {
...defaultOptions,
...document.attributes,
},
};
},
},
],
};
export const entityEngineDescriptorType: SavedObjectsType = {
name: entityEngineDescriptorTypeName,
indexPattern: SECURITY_SOLUTION_SAVED_OBJECT_INDEX,
hidden: false,
namespaceType: 'multiple-isolated',
mappings: entityEngineDescriptorTypeMappings,
modelVersions: { 1: version1 },
modelVersions: { 1: version1, 2: version2 },
};

24
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store.ts
View file

@ -6,6 +6,8 @@
*/
import expect from 'expect';
import { defaultOptions } from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/constants';
import { omit } from 'lodash/fp';
import { FtrProviderContext } from '../../../../ftr_provider_context';
import { EntityStoreUtils } from '../../utils';
import { dataViewRouteHelpersFactory } from '../../utils/data_view';
@ -17,6 +19,8 @@ export default ({ getService }: FtrProviderContext) => {
describe('@ess @skipInServerlessMKI Entity Store APIs', () => {
const dataView = dataViewRouteHelpersFactory(supertest);
const defaults = omit('docsPerSecond', defaultOptions);
before(async () => {
await utils.cleanEngines();
await dataView.create('security-solution');
@ -85,12 +89,9 @@ export default ({ getService }: FtrProviderContext) => {
.expect(200);
expect(getResponse.body).toEqual({
...defaults,
status: 'started',
type: 'host',
indexPattern: '',
filter: '',
fieldHistoryLength: 10,
lookbackPeriod: '24h',
});
});
@ -102,12 +103,9 @@ export default ({ getService }: FtrProviderContext) => {
.expect(200);
expect(getResponse.body).toEqual({
...defaults,
status: 'started',
type: 'user',
indexPattern: '',
filter: '',
fieldHistoryLength: 10,
lookbackPeriod: '24h',
});
});
});
@ -121,20 +119,14 @@ export default ({ getService }: FtrProviderContext) => {
expect(sortedEngines).toEqual([
{
...defaults,
status: 'started',
type: 'host',
indexPattern: '',
filter: '',
fieldHistoryLength: 10,
lookbackPeriod: '24h',
},
{
...defaults,
status: 'started',
type: 'user',
indexPattern: '',
filter: '',
fieldHistoryLength: 10,
lookbackPeriod: '24h',
},
]);
});

23
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entity_store_nondefault_spaces.ts
View file

@ -7,6 +7,8 @@
import expect from '@kbn/expect';
import { v4 as uuidv4 } from 'uuid';
import { defaultOptions } from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/constants';
import { omit } from 'lodash/fp';
import { FtrProviderContextWithSpaces } from '../../../../ftr_provider_context_with_spaces';
import { EntityStoreUtils } from '../../utils';
import { dataViewRouteHelpersFactory } from '../../utils/data_view';
@ -21,6 +23,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
describe('@ess Entity Store Engine APIs in non-default space', () => {
const dataView = dataViewRouteHelpersFactory(supertest, namespace);
const defaults = omit('docsPerSecond', defaultOptions);
before(async () => {
await utils.cleanEngines();
await spaces.create({
@ -73,12 +76,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
.expect(200);
expect(getResponse.body).to.eql({
...defaults,
status: 'started',
type: 'host',
filter: '',
fieldHistoryLength: 10,
lookbackPeriod: '24h',
indexPattern: '',
});
});
@ -93,12 +93,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
.expect(200);
expect(getResponse.body).to.eql({
...defaults,
status: 'started',
type: 'user',
filter: '',
fieldHistoryLength: 10,
lookbackPeriod: '24h',
indexPattern: '',
});
});
});
@ -112,20 +109,14 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
expect(sortedEngines).to.eql([
{
...defaults,
status: 'started',
type: 'host',
filter: '',
fieldHistoryLength: 10,
lookbackPeriod: '24h',
indexPattern: '',
},
{
...defaults,
status: 'started',
type: 'user',
filter: '',
fieldHistoryLength: 10,
lookbackPeriod: '24h',
indexPattern: '',
},
]);
});