github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 01:13:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Entity Store] [Asset Inventory] Universal entity definition (#202888)

Browse source 
## Summary

This PR adds a universal entity definition.
A universal entity uses `related.entity` as an identifier field and
includes an extra processor step that parses the field
`entities.keyword` and extracts all the entities in said field (whose
original data comes from `related.entities`).

See this
[doc](https://docs.google.com/document/d/1D8xDtn3HHP65i1Y3eIButacD6ZizyjZZRJB7mxlXzQY/edit?tab=t.0#heading=h.9fz3qtlfzjg7)
for more details.

To accomplish this, we need to allow describing an entity along with
extra entity store resources required for that entity's engine.
This PR reworks the current entity store by introducing an `Entity
Description`, which has all that required information. From it, we can
build an `EntityEngineDescription` which adds all the needed data that
must be computed (as opposed to hardcoded) and is then used to generate
all the resources needed for that Entity's engine (entity definition,
pipeline, enrich policy, index mappings, etc).

<img width="3776" alt="EntityDescriptions"
src="https://github.com/user-attachments/assets/bdf7915f-1981-47e6-a815-31163f24ad03">

This required a refactoring of the current `UnitedEntityDefinition`,
which has now been removed in favour of more contextual functions for
all the different parts.
The intention is to decouple the Entity Description schema from the
schemas required for field retention, entity manager and pipeline. We
can then freely expand on our Entity Description as required, and simply
alter the conversion functions when needed.

## How to test

1. On a fresh ES cluster, add some entity data
* For hosts and user, use the [security documents
generator](https://github.com/elastic/security-documents-generator)
   * For universal, there are a few more steps:
      1. Create the `entity.keyword` builder pipeline
      2. Add it to a index template
      3. Post some docs to the corresponding index 
2. Initialise the universal entity engine via: `POST
kbn:/api/entity_store/engines/universal/init {}`
* Note that using the UI does not work, as we've specifically removed
the Universal engine from the normal Entity Store workflow
3. Check the status of the store is `running` via `GET
kbn:/api/entity_store/status`
4. Once the transform runs, you can query `GET entities*/_search` to see
the created entities

Note that universal entities do not show up in the dashboard Entities
List.


### Code to ingest data
<details>
<summary>Pipeline</summary>

```js
PUT _ingest/pipeline/entities-keyword-builder
{
   "description":"Serialize entities.metadata into a keyword field",
   "processors":[
      {
         "script":{
            "lang":"painless",
            "source":"""
String jsonFromMap(Map map) {
    StringBuilder json = new StringBuilder("{");
    boolean first = true;

    for (entry in map.entrySet()) {
        if (!first) {
            json.append(",");
        }
        first = false;

        String key = entry.getKey().replace("\"", "\\\"");
        Object value = entry.getValue();

        json.append("\"").append(key).append("\":");

        if (value instanceof String) {
            String escapedValue = ((String) value).replace("\"", "\\\"").replace("=", ":");
            json.append("\"").append(escapedValue).append("\"");
        } else if (value instanceof Map) {
            json.append(jsonFromMap((Map) value));
        } else if (value instanceof List) {
            json.append(jsonFromList((List) value));
        } else if (value instanceof Boolean || value instanceof Number) {
            json.append(value.toString());
        } else {
            // For other types, treat as string
            String escapedValue = value.toString().replace("\"", "\\\"").replace("=", ":");
            json.append("\"").append(escapedValue).append("\"");
        }
    }

    json.append("}");
    return json.toString();
}

String jsonFromList(List list) {

    StringBuilder json = new StringBuilder("[");
    boolean first = true;

    for (item in list) {
        if (!first) {
            json.append(",");
        }
        first = false;

        if (item instanceof String) {
            String escapedItem = ((String) item).replace("\"", "\\\"").replace("=", ":");
            json.append("\"").append(escapedItem).append("\"");
        } else if (item instanceof Map) {
            json.append(jsonFromMap((Map) item));
        } else if (item instanceof List) {
            json.append(jsonFromList((List) item));
        } else if (item instanceof Boolean || item instanceof Number) {
            json.append(item.toString());
        } else {
            // For other types, treat as string
            String escapedItem = item.toString().replace("\"", "\\\"").replace("=", ":");
            json.append("\"").append(escapedItem).append("\"");
        }
    }

    json.append("]");
    return json.toString();
}

def metadata = jsonFromMap(ctx['entities']['metadata']);
ctx['entities']['keyword'] = metadata;
"""

            }
        }
    ]
}
```
</details>

<details>
<summary>Index template</summary>

```js
PUT /_index_template/entity_store_index_template
{
   "index_patterns":[
      "logs-store"
   ],
   "template":{
      "settings":{
         "index":{
            "default_pipeline":"entities-keyword-builder"
         }
      },
      "mappings":{
         "properties":{
            "@timestamp":{
               "type":"date"
            },
            "message":{
               "type":"text"
            },
            "event":{
               "properties":{
                  "action":{
                     "type":"keyword"
                  },
                  "category":{
                     "type":"keyword"
                  },
                  "type":{
                     "type":"keyword"
                  },
                  "outcome":{
                     "type":"keyword"
                  },
                  "provider":{
                     "type":"keyword"
                  },
                  "ingested":{
                    "type": "date"
                  }
               }
            },
            "related":{
               "properties":{
                  "entity":{
                     "type":"keyword"
                  }
               }
            },
            "entities":{
               "properties":{
                  "metadata":{
                     "type":"flattened"
                  },
                  "keyword":{
                     "type":"keyword"
                  }
               }
            }
         }
      }
   }
}
```
</details>

<details>
<summary>Example source doc</summary>

```js
POST /logs-store/_doc/
{
   "@timestamp":"2024-11-29T10:01:00Z",
   "message":"Eddie",
   "event": {
      "type":[
         "creation"
      ],
      "ingested": "2024-12-03T10:01:00Z"
   },
   "related":{
      "entity":[
         "AKIAI44QH8DHBEXAMPLE"
      ]
   },
   "entities":{
      "metadata":{
         "AKIAI44QH8DHBEXAMPLE":{
            "entity":{
               "id":"AKIAI44QH8DHBEXAMPLE",
               "category":"Access Management",
               "type":"AWS IAM Access Key"
            },
            "cloud":{
               "account":{
                  "id":"444455556666"
               }
            }
         }
      }
   }
}
```
</details>

### To do

- [x] Add/Improve [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
- [x] Feature flag


----
#### Update:

Added `assetInventoryStoreEnabled` Feature Flag. It is disabled by
default and even when enabled, the `/api/entity_store/enable` route does
not initialize the Universal Entity Engine.
`/api/entity_store/engines/universal/init` needs to be manually called
to initialize it

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Rômulo Farias <romulodefarias@gmail.com>
Co-authored-by: jaredburgettelastic <jared.burgett@elastic.co>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Tiago Vila Verde 2025-01-03 16:43:16 +00:00 • committed by GitHub
parent e51b581e25
commit c6b0a31d8e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
54 changed files with 1128 additions and 1128 deletions
Download patch file Download diff file Expand all files Collapse all files

2
oas_docs/output/kibana.serverless.yaml
View file

@ -47250,6 +47250,7 @@ components:
- user
- host
- service
- universal
type: string
Security_Entity_Analytics_API_HostEntity:
type: object
@ -47320,6 +47321,7 @@ components:
- host.name
- user.name
- service.name
- related.entity
type: string
Security_Entity_Analytics_API_IndexPattern:
type: string

2
oas_docs/output/kibana.yaml
View file

@ -54126,6 +54126,7 @@ components:
- user
- host
- service
- universal
type: string
Security_Entity_Analytics_API_HostEntity:
type: object
@ -54196,6 +54197,7 @@ components:
- host.name
- user.name
- service.name
- related.entity
type: string
Security_Entity_Analytics_API_IndexPattern:
type: string

2
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts
View file

@ -17,7 +17,7 @@
import { z } from '@kbn/zod';
export type IdField = z.infer<typeof IdField>;
export const IdField = z.enum(['host.name', 'user.name', 'service.name']);
export const IdField = z.enum(['host.name', 'user.name', 'service.name', 'related.entity']);
export type IdFieldEnum = typeof IdField.enum;
export const IdFieldEnum = IdField.enum;

1
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml
View file

@ -29,6 +29,7 @@ components:
- 'host.name'
- 'user.name'
- 'service.name'
- 'related.entity'
AssetCriticalityRecordIdParts:
type: object
properties:

2
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
View file

@ -17,7 +17,7 @@
import { z } from '@kbn/zod';
export type EntityType = z.infer<typeof EntityType>;
export const EntityType = z.enum(['user', 'host', 'service']);
export const EntityType = z.enum(['user', 'host', 'service', 'universal']);
export type EntityTypeEnum = typeof EntityType.enum;
export const EntityTypeEnum = EntityType.enum;

1
x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
View file

@ -12,6 +12,7 @@ components:
- user
- host
- service
- universal
EngineDescriptor:
type: object

4
x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/parse_asset_criticality_csv_row.test.ts
View file

@ -46,7 +46,7 @@ describe('parseAssetCriticalityCsvRow', () => {
// @ts-ignore result can now only be InvalidRecord
expect(result.error).toMatchInlineSnapshot(
`"Invalid entity type \\"invalid\\", expected to be one of: user, host, service"`
`"Invalid entity type \\"invalid\\", expected to be one of: user, host, service, universal"`
);
});
@ -57,7 +57,7 @@ describe('parseAssetCriticalityCsvRow', () => {
// @ts-ignore result can now only be InvalidRecord
expect(result.error).toMatchInlineSnapshot(
`"Invalid entity type \\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...\\", expected to be one of: user, host, service"`
`"Invalid entity type \\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...\\", expected to be one of: user, host, service, universal"`
);
});

1
x-pack/solutions/security/plugins/security_solution/common/entity_analytics/entity_store/constants.ts
View file

@ -31,6 +31,7 @@ export const IDENTITY_FIELD_MAP: Record<EntityType, IdField> = {
[EntityTypeEnum.host]: 'host.name',
[EntityTypeEnum.user]: 'user.name',
[EntityTypeEnum.service]: 'service.name',
[EntityTypeEnum.universal]: 'related.entity',
};
export const getAvailableEntityTypes = (): EntityType[] =>

6
x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts
View file

@ -268,6 +268,12 @@ export const allowedExperimentalValues = Object.freeze({
*/
crowdstrikeRunScriptEnabled: false,
/**
* Enables the Asset Inventory Entity Store feature.
* Allows initializing the Universal Entity Store via the API.
*/
assetInventoryStoreEnabled: false,
/**
* Enables the Asset Inventory feature
*/

2
x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
View file

@ -1075,6 +1075,7 @@ components:
- user
- host
- service
- universal
type: string
HostEntity:
type: object
@ -1145,6 +1146,7 @@ components:
- host.name
- user.name
- service.name
- related.entity
type: string
IndexPattern:
type: string

2
x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
View file

@ -1075,6 +1075,7 @@ components:
- user
- host
- service
- universal
type: string
HostEntity:
type: object
@ -1145,6 +1146,7 @@ components:
- host.name
- user.name
- service.name
- related.entity
type: string
IndexPattern:
type: string

1
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts
View file

@ -82,6 +82,7 @@ const entityTypeByIdField = {
'host.name': 'host',
'user.name': 'user',
'service.name': 'service',
'related.entity': 'universal',
} as const;
export const getImplicitEntityFields = (record: AssetCriticalityUpsertWithDeleted) => {

21
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/component_template.ts
View file

@ -10,18 +10,24 @@ import {
EngineComponentResourceEnum,
type EngineComponentStatus,
} from '../../../../../common/api/entity_analytics';
import type { UnitedEntityDefinition } from '../united_entity_definitions';
import type { EntityEngineInstallationDescriptor } from '../installation/types';
const getComponentTemplateName = (definitionId: string) => `${definitionId}-latest@platform`;
interface Options {
unitedDefinition: UnitedEntityDefinition;
/**
* The entity engine description id
**/
id: string;
esClient: ElasticsearchClient;
}
export const createEntityIndexComponentTemplate = ({ unitedDefinition, esClient }: Options) => {
const { entityManagerDefinition, indexMappings } = unitedDefinition;
const name = getComponentTemplateName(entityManagerDefinition.id);
export const createEntityIndexComponentTemplate = (
description: EntityEngineInstallationDescriptor,
esClient: ElasticsearchClient
) => {
const { id, indexMappings } = description;
const name = getComponentTemplateName(id);
return esClient.cluster.putComponentTemplate({
name,
body: {
@ -35,9 +41,8 @@ export const createEntityIndexComponentTemplate = ({ unitedDefinition, esClient
});
};
export const deleteEntityIndexComponentTemplate = ({ unitedDefinition, esClient }: Options) => {
const { entityManagerDefinition } = unitedDefinition;
const name = getComponentTemplateName(entityManagerDefinition.id);
export const deleteEntityIndexComponentTemplate = ({ id, esClient }: Options) => {
const name = getComponentTemplateName(id);
return esClient.cluster.deleteComponentTemplate(
{ name },
{

72
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts
View file

@ -6,12 +6,14 @@
*/
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
import type { EnrichPutPolicyRequest } from '@elastic/elasticsearch/lib/api/types';
import type { EntityType } from '../../../../../common/api/entity_analytics';
import { EngineComponentResourceEnum } from '../../../../../common/api/entity_analytics';
import { getEntitiesIndexName } from '../utils';
import type { UnitedEntityDefinition } from '../united_entity_definitions';
import type { EntityEngineInstallationDescriptor } from '../installation/types';
type DefinitionMetadata = Pick<UnitedEntityDefinition, 'namespace' | 'entityType' | 'version'>;
type DefinitionMetadata = Pick<EntityEngineInstallationDescriptor, 'entityType' | 'version'> & {
namespace: string;
};
export const getFieldRetentionEnrichPolicyName = ({
namespace,
@ -21,41 +23,47 @@ export const getFieldRetentionEnrichPolicyName = ({
return `entity_store_field_retention_${entityType}_${namespace}_v${version}`;
};
const getFieldRetentionEnrichPolicy = (
unitedDefinition: UnitedEntityDefinition
): EnrichPutPolicyRequest => {
const { namespace, entityType, fieldRetentionDefinition } = unitedDefinition;
return {
name: getFieldRetentionEnrichPolicyName(unitedDefinition),
match: {
indices: getEntitiesIndexName(entityType, namespace),
match_field: fieldRetentionDefinition.matchField,
enrich_fields: fieldRetentionDefinition.fields.map(({ field }) => field),
},
};
};
export const createFieldRetentionEnrichPolicy = async ({
esClient,
unitedDefinition,
description,
options,
}: {
esClient: ElasticsearchClient;
unitedDefinition: UnitedEntityDefinition;
description: EntityEngineInstallationDescriptor;
options: { namespace: string };
}) => {
const policy = getFieldRetentionEnrichPolicy(unitedDefinition);
return esClient.enrich.putPolicy(policy);
return esClient.enrich.putPolicy({
name: getFieldRetentionEnrichPolicyName({
namespace: options.namespace,
entityType: description.entityType,
version: description.version,
}),
match: {
indices: getEntitiesIndexName(description.entityType, options.namespace),
match_field: description.identityField,
enrich_fields: description.fields.map(({ destination }) => destination),
},
});
};
export const executeFieldRetentionEnrichPolicy = async ({
esClient,
unitedDefinition,
entityType,
version,
logger,
options,
}: {
unitedDefinition: DefinitionMetadata;
entityType: EntityType;
version: string;
esClient: ElasticsearchClient;
logger: Logger;
options: { namespace: string };
}): Promise<{ executed: boolean }> => {
const name = getFieldRetentionEnrichPolicyName(unitedDefinition);
const name = getFieldRetentionEnrichPolicyName({
namespace: options.namespace,
entityType,
version,
});
try {
await esClient.enrich.executePolicy({ name });
return { executed: true };
@ -63,27 +71,31 @@ export const executeFieldRetentionEnrichPolicy = async ({
if (e.statusCode === 404) {
return { executed: false };
}
logger.error(
`Error executing field retention enrich policy for ${unitedDefinition.entityType}: ${e.message}`
);
logger.error(`Error executing field retention enrich policy for ${entityType}: ${e.message}`);
throw e;
}
};
export const deleteFieldRetentionEnrichPolicy = async ({
unitedDefinition,
description,
options,
esClient,
logger,
attempts = 5,
delayMs = 2000,
}: {
unitedDefinition: DefinitionMetadata;
description: EntityEngineInstallationDescriptor;
options: { namespace: string };
esClient: ElasticsearchClient;
logger: Logger;
attempts?: number;
delayMs?: number;
}) => {
const name = getFieldRetentionEnrichPolicyName(unitedDefinition);
const name = getFieldRetentionEnrichPolicyName({
namespace: options.namespace,
entityType: description.entityType,
version: description.version,
});
let currentAttempt = 1;
while (currentAttempt <= attempts) {
try {

51
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/entity_index.ts
View file

@ -6,6 +6,7 @@
*/
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
import {
EngineComponentResourceEnum,
type EngineComponentStatus,
@ -14,6 +15,8 @@ import {
import { getEntitiesIndexName } from '../utils';
import { createOrUpdateIndex } from '../../utils/create_or_update_index';
import type { EntityEngineInstallationDescriptor } from '../installation/types';
interface Options {
entityType: EntityType;
esClient: ElasticsearchClient;
@ -58,3 +61,51 @@ export const getEntityIndexStatus = async ({
return { id: index, installed: exists, resource: EngineComponentResourceEnum.index };
};
export type MappingProperties = NonNullable<MappingTypeMapping['properties']>;
export const generateIndexMappings = (
description: EntityEngineInstallationDescriptor
): MappingTypeMapping => {
const identityFieldMappings: MappingProperties = {
[description.identityField]: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
};
const otherFieldMappings = description.fields
.filter(({ mapping }) => mapping)
.reduce((acc, { destination, mapping }) => {
acc[destination] = mapping;
return acc;
}, {} as MappingProperties);
return {
properties: { ...BASE_ENTITY_INDEX_MAPPING, ...identityFieldMappings, ...otherFieldMappings },
};
};
export const BASE_ENTITY_INDEX_MAPPING: MappingProperties = {
'@timestamp': {
type: 'date',
},
'asset.criticality': {
type: 'keyword',
},
'entity.name': {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
'entity.source': {
type: 'keyword',
},
};

86
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_pipeline.ts
View file

@ -7,22 +7,22 @@
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
import type { EntityDefinition } from '@kbn/entities-schema';
import { EngineComponentResourceEnum } from '../../../../../common/api/entity_analytics';
import { type FieldRetentionDefinition } from '../field_retention_definition';
import {
debugDeepCopyContextStep,
getDotExpanderSteps,
getRemoveEmptyFieldSteps,
removeEntityDefinitionFieldsStep,
retentionDefinitionToIngestProcessorSteps,
} from './ingest_processor_steps';
import { getIdentityFieldForEntityType } from '../utils';
import { getFieldRetentionEnrichPolicyName } from './enrich_policy';
import type { UnitedEntityDefinition } from '../united_entity_definitions';
const getPlatformPipelineId = (definition: EntityDefinition) => {
return `${definition.id}-latest@platform`;
import { getFieldRetentionEnrichPolicyName } from './enrich_policy';
import { fieldOperatorToIngestProcessor } from '../field_retention';
import type { EntityEngineInstallationDescriptor } from '../installation/types';
const getPlatformPipelineId = (descriptionId: string) => {
return `${descriptionId}-latest@platform`;
};
// the field that the enrich processor writes to
@ -38,30 +38,28 @@ export const ENRICH_FIELD = 'historical';
* and the context field in the document to help with debugging.
*/
const buildIngestPipeline = ({
version,
fieldRetentionDefinition,
allEntityFields,
debugMode,
namespace,
description,
}: {
fieldRetentionDefinition: FieldRetentionDefinition;
allEntityFields: string[];
debugMode?: boolean;
namespace: string;
version: string;
description: EntityEngineInstallationDescriptor;
}): IngestProcessorContainer[] => {
const { entityType, matchField } = fieldRetentionDefinition;
const enrichPolicyName = getFieldRetentionEnrichPolicyName({
namespace,
entityType,
version,
entityType: description.entityType,
version: description.version,
});
return [
...(debugMode ? [debugDeepCopyContextStep()] : []),
const processors = [
{
enrich: {
policy_name: enrichPolicyName,
field: matchField,
field: description.identityField,
target_field: ENRICH_FIELD,
},
},
@ -74,14 +72,14 @@ const buildIngestPipeline = ({
{
set: {
field: 'entity.name',
value: `{{${getIdentityFieldForEntityType(entityType)}}}`,
value: `{{${description.identityField}}}`,
},
},
...getDotExpanderSteps(allEntityFields),
...retentionDefinitionToIngestProcessorSteps(fieldRetentionDefinition, {
enrichField: ENRICH_FIELD,
}),
...getRemoveEmptyFieldSteps([...allEntityFields, 'asset', `${entityType}.risk`]),
...description.fields.map((field) =>
fieldOperatorToIngestProcessor(field, { enrichField: ENRICH_FIELD })
),
...getRemoveEmptyFieldSteps([...allEntityFields, 'asset', `${description.entityType}.risk`]),
removeEntityDefinitionFieldsStep(),
...(!debugMode
? [
@ -94,43 +92,45 @@ const buildIngestPipeline = ({
]
: []),
];
const extraSteps =
(typeof description.pipeline === 'function'
? description.pipeline(processors)
: description.pipeline) ?? [];
return [...(debugMode ? [debugDeepCopyContextStep()] : []), ...processors, ...extraSteps];
};
// developing the pipeline is a bit tricky, so we have a debug mode
// set xpack.securitySolution.entityAnalytics.entityStore.developer.pipelineDebugMode
// to true to keep the enrich field and the context field in the document to help with debugging.
export const createPlatformPipeline = async ({
unitedDefinition,
logger,
esClient,
debugMode,
description,
options,
}: {
unitedDefinition: UnitedEntityDefinition;
description: EntityEngineInstallationDescriptor;
options: { namespace: string };
logger: Logger;
esClient: ElasticsearchClient;
debugMode?: boolean;
}) => {
const { fieldRetentionDefinition, entityManagerDefinition } = unitedDefinition;
const allEntityFields: string[] = (entityManagerDefinition?.metadata || []).map((m) => {
if (typeof m === 'string') {
return m;
}
return m.destination;
});
const allEntityFields = description.fields.map(({ destination }) => destination);
const pipeline = {
id: getPlatformPipelineId(entityManagerDefinition),
id: getPlatformPipelineId(description.id),
body: {
_meta: {
managed_by: 'entity_store',
managed: true,
},
description: `Ingest pipeline for entity definition ${entityManagerDefinition.id}`,
description: `Ingest pipeline for entity definition ${description.id}`,
processors: buildIngestPipeline({
namespace: unitedDefinition.namespace,
version: unitedDefinition.version,
fieldRetentionDefinition,
namespace: options.namespace,
description,
version: description.version,
allEntityFields,
debugMode,
}),
@ -143,15 +143,15 @@ export const createPlatformPipeline = async ({
};
export const deletePlatformPipeline = ({
unitedDefinition,
description,
logger,
esClient,
}: {
unitedDefinition: UnitedEntityDefinition;
description: EntityEngineInstallationDescriptor;
logger: Logger;
esClient: ElasticsearchClient;
}) => {
const pipelineId = getPlatformPipelineId(unitedDefinition.entityManagerDefinition);
const pipelineId = getPlatformPipelineId(description.id);
logger.debug(`Attempting to delete pipeline: ${pipelineId}`);
return esClient.ingest.deletePipeline(
{
@ -164,13 +164,13 @@ export const deletePlatformPipeline = ({
};
export const getPlatformPipelineStatus = async ({
definition,
engineId,
esClient,
}: {
definition: EntityDefinition;
engineId: string;
esClient: ElasticsearchClient;
}) => {
const pipelineId = getPlatformPipelineId(definition);
const pipelineId = getPlatformPipelineId(engineId);
const pipeline = await esClient.ingest.getPipeline(
{
id: pipelineId,

1
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/index.ts
View file

@ -9,4 +9,3 @@ export { debugDeepCopyContextStep } from './debug_deep_copy_context_step';
export { getDotExpanderSteps } from './get_dot_expander_steps';
export { getRemoveEmptyFieldSteps } from './get_remove_empty_field_steps';
export { removeEntityDefinitionFieldsStep } from './remove_entity_definition_fields_step';
export { retentionDefinitionToIngestProcessorSteps } from './retention_definition_to_ingest_processor_steps';

26
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/retention_definition_to_ingest_processor_steps.ts
View file

@ -1,26 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
import type {
FieldRetentionDefinition,
FieldRetentionOperatorBuilderOptions,
} from '../../field_retention_definition';
import { fieldOperatorToIngestProcessor } from '../../field_retention_definition';
/**
* Converts a field retention definition to the ingest processor steps
* required to apply the field retention policy.
*/
export const retentionDefinitionToIngestProcessorSteps = (
fieldRetentionDefinition: FieldRetentionDefinition,
options: FieldRetentionOperatorBuilderOptions
): IngestProcessorContainer[] => {
return fieldRetentionDefinition.fields.map((field) =>
fieldOperatorToIngestProcessor(field, options)
);
};

27
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/constants.ts Normal file
View file

@ -0,0 +1,27 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store';
import {
HOST_DEFINITION_VERSION,
UNIVERSAL_DEFINITION_VERSION,
USER_DEFINITION_VERSION,
SERVICE_DEFINITION_VERSION,
} from './entity_descriptions';
export const VERSIONS_BY_ENTITY_TYPE: Record<EntityType, string> = {
host: HOST_DEFINITION_VERSION,
user: USER_DEFINITION_VERSION,
universal: UNIVERSAL_DEFINITION_VERSION,
service: SERVICE_DEFINITION_VERSION,
};
export const DEFAULT_FIELD_HISTORY_LENGTH = 10;
export const DEFAULT_SYNC_DELAY = '1m';
export const DEFAULT_FREQUENCY = '1m';
export const DEFAULT_LOOKBACK_PERIOD = '1d';
export const DEFAULT_TIMESTAMP_FIELD = '@timestamp';

36
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/common.ts Normal file
View file

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { EntityType } from '../../../../../../common/api/entity_analytics/entity_store';
import type { FieldDescription } from '../../installation/types';
import { oldestValue, newestValue } from './field_utils';
export const getCommonFieldDescriptions = (entityType: EntityType): FieldDescription[] => {
return [
oldestValue({
source: '_index',
destination: 'entity.source',
}),
newestValue({ source: 'asset.criticality' }),
newestValue({
source: `${entityType}.risk.calculated_level`,
}),
newestValue({
source: `${entityType}.risk.calculated_score`,
mapping: {
type: 'float',
},
}),
newestValue({
source: `${entityType}.risk.calculated_score_norm`,
mapping: {
type: 'float',
},
}),
];
};

79
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/field_utils.ts Normal file
View file

@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
import type { FieldDescription } from '../../installation/types';
export const collectValues = ({
destination,
source,
fieldHistoryLength = 10,
mapping = { type: 'keyword' },
}: {
source: string;
destination?: string;
mapping?: MappingProperty;
fieldHistoryLength?: number;
}): FieldDescription => ({
destination: destination ?? source,
source,
retention: {
operation: 'collect_values',
maxLength: fieldHistoryLength,
},
aggregation: {
type: 'terms',
limit: fieldHistoryLength,
},
mapping,
});
export const newestValue = ({
destination,
mapping = { type: 'keyword' },
source,
sort,
}: {
source: string;
destination?: string;
mapping?: MappingProperty;
sort?: Record<string, 'asc' | 'desc'>;
}): FieldDescription => ({
destination: destination ?? source,
source,
retention: { operation: 'prefer_newest_value' },
aggregation: {
type: 'top_value',
sort: sort ?? {
'@timestamp': 'desc',
},
},
mapping,
});
export const oldestValue = ({
source,
destination,
mapping = { type: 'keyword' },
sort,
}: {
source: string;
destination?: string;
mapping?: MappingProperty;
sort?: Record<string, 'asc' | 'desc'>;
}): FieldDescription => ({
destination: destination ?? source,
source,
retention: { operation: 'prefer_oldest_value' },
aggregation: {
type: 'top_value',
sort: sort ?? {
'@timestamp': 'asc',
},
},
mapping,
});

48
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/host.ts Normal file
View file

@ -0,0 +1,48 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { collectValues as collect } from './field_utils';
import type { EntityDescription } from '../types';
import { getCommonFieldDescriptions } from './common';
export const HOST_DEFINITION_VERSION = '1.0.0';
export const HOST_IDENTITY_FIELD = 'host.name';
export const hostEntityEngineDescription: EntityDescription = {
entityType: 'host',
version: HOST_DEFINITION_VERSION,
identityField: HOST_IDENTITY_FIELD,
settings: {
timestampField: '@timestamp',
},
fields: [
collect({ source: 'host.domain' }),
collect({ source: 'host.hostname' }),
collect({ source: 'host.id' }),
collect({
source: 'host.os.name',
mapping: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
}),
collect({ source: 'host.os.type' }),
collect({
source: 'host.ip',
mapping: {
type: 'ip',
},
}),
collect({ source: 'host.mac' }),
collect({ source: 'host.type' }),
collect({ source: 'host.architecture' }),
...getCommonFieldDescriptions('host'),
],
};

3
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/index.ts → x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/index.ts
View file

@ -8,4 +8,5 @@
export * from './host';
export * from './user';
export * from './service';
export { getCommonUnitedFieldDefinitions } from './common';
export * from './universal';
export { getCommonFieldDescriptions } from './common';

32
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/service.ts Normal file
View file

@ -0,0 +1,32 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { EntityDescription } from '../types';
import { collectValues as collect, newestValue } from './field_utils';
export const SERVICE_DEFINITION_VERSION = '1.0.0';
export const SERVICE_IDENTITY_FIELD = 'service.name';
export const serviceEntityEngineDescription: EntityDescription = {
entityType: 'service',
version: SERVICE_DEFINITION_VERSION,
identityField: SERVICE_IDENTITY_FIELD,
settings: {
timestampField: '@timestamp',
},
fields: [
collect({ source: 'service.address' }),
collect({ source: 'service.environment' }),
collect({ source: 'service.ephemeral_id' }),
collect({ source: 'service.id' }),
collect({ source: 'service.node.name' }),
collect({ source: 'service.node.roles' }),
newestValue({ source: 'service.state' }),
collect({ source: 'service.type' }),
newestValue({ source: 'service.version' }),
],
};

60
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/universal.ts Normal file
View file

@ -0,0 +1,60 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { EntityDescription } from '../types';
import { collectValues as collect } from './field_utils';
export const UNIVERSAL_DEFINITION_VERSION = '1.0.0';
export const UNIVERSAL_IDENTITY_FIELD = 'related.entity';
const entityMetadataExtractorProcessor = {
script: {
tag: 'entity_metadata_extractor',
on_failure: [
{
set: {
field: 'error.message',
value:
'Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}',
},
},
],
lang: 'painless',
source: /* java */ `
Map merged = ctx;
def id = ctx.entity.id;
for (meta in ctx.collected.metadata) {
Object json = Processors.json(meta);
if (((Map)json)[id] == null) {
continue;
}
for (entry in ((Map)json)[id].entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
merged.put(key, value);
}
}
merged.entity.id = id;
ctx = merged;
`,
},
};
export const universalEntityEngineDescription: EntityDescription = {
version: UNIVERSAL_DEFINITION_VERSION,
entityType: 'universal',
identityField: UNIVERSAL_IDENTITY_FIELD,
fields: [collect({ source: 'entities.keyword', destination: 'collected.metadata' })],
settings: {
timestampField: 'event.ingested',
},
pipeline: [entityMetadataExtractorProcessor],
};

39
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/user.ts Normal file
View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { EntityDescription } from '../types';
import { getCommonFieldDescriptions } from './common';
import { collectValues as collect } from './field_utils';
export const USER_DEFINITION_VERSION = '1.0.0';
export const USER_IDENTITY_FIELD = 'user.name';
export const userEntityEngineDescription: EntityDescription = {
entityType: 'user',
version: USER_DEFINITION_VERSION,
identityField: USER_IDENTITY_FIELD,
settings: {
timestampField: '@timestamp',
},
fields: [
collect({ source: 'user.domain' }),
collect({ source: 'user.email' }),
collect({
source: 'user.full_name',
mapping: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
}),
collect({ source: 'user.hash' }),
collect({ source: 'user.id' }),
collect({ source: 'user.roles' }),
...getCommonFieldDescriptions('user'),
],
};

40
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_manager_conversion.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { pick } from 'lodash/fp';
import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema';
import { buildEntityDefinitionId } from '../utils';
import type { EntityEngineInstallationDescriptor } from '../installation/types';
export const convertToEntityManagerDefinition = (
description: EntityEngineInstallationDescriptor,
options: { namespace: string; filter: string }
): EntityDefinition => {
const metadata = description.fields.map(pick(['source', 'destination', 'aggregation']));
const definition = {
id: buildEntityDefinitionId(description.entityType, options.namespace),
name: `Security '${description.entityType}' Entity Store Definition`,
type: description.entityType,
indexPatterns: description.indexPatterns,
identityFields: [description.identityField],
displayNameTemplate: `{{${description.identityField}}}`,
metadata,
latest: {
timestampField: description.settings.timestampField,
lookbackPeriod: description.settings.lookbackPeriod,
settings: {
syncDelay: description.settings.syncDelay,
frequency: description.settings.frequency,
},
},
version: description.version,
managed: true,
};
return entityDefinitionSchema.parse(definition);
};

27
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/types.ts Normal file
View file

@ -0,0 +1,27 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { EntityEngineInstallationDescriptor } from '../installation/types';
type PickPartial<T, K extends keyof T, Optional extends K = never> = {
[P in K as P extends Optional ? never : P]: T[P];
} & {
[P in K as P extends Optional ? P : never]?: Partial<T[P]>;
};
export type EntityDescription = PickPartial<
EntityEngineInstallationDescriptor,
| 'version'
| 'entityType'
| 'fields'
| 'identityField'
| 'indexPatterns'
| 'indexMappings'
| 'settings'
| 'pipeline',
'indexPatterns' | 'indexMappings' | 'settings' | 'pipeline'
>;

30
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts
View file

@ -18,17 +18,27 @@ import type { AppClient } from '../../..';
import type { EntityStoreConfig } from './types';
import { mockGlobalState } from '../../../../public/common/mock';
import type { EntityDefinition } from '@kbn/entities-schema';
import { getUnitedEntityDefinition } from './united_entity_definitions';
import { convertToEntityManagerDefinition } from './entity_definitions/entity_manager_conversion';
const unitedDefinition = getUnitedEntityDefinition({
entityType: 'host',
namespace: 'test',
fieldHistoryLength: 10,
indexPatterns: [],
syncDelay: '1m',
frequency: '1m',
});
const definition: EntityDefinition = unitedDefinition.entityManagerDefinition;
const definition: EntityDefinition = convertToEntityManagerDefinition(
{
id: 'host_engine',
entityType: 'host',
pipeline: [],
version: '0.0.1',
fields: [],
identityField: 'host.name',
indexMappings: {},
indexPatterns: [],
settings: {
syncDelay: '1m',
frequency: '1m',
timestampField: '@timestamp',
lookbackPeriod: '24h',
},
},
{ namespace: 'test', filter: '' }
);
describe('EntityStoreDataClient', () => {
const mockSavedObjectClient = savedObjectsClientMock.create();

103
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
View file

@ -51,7 +51,6 @@ import type {
import { EngineDescriptorClient } from './saved_object/engine_descriptor';
import { ENGINE_STATUS, ENTITY_STORE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
import { AssetCriticalityMigrationClient } from '../asset_criticality/asset_criticality_migration_client';
import { getUnitedEntityDefinition } from './united_entity_definitions';
import {
startEntityStoreFieldRetentionEnrichTask,
removeEntityStoreFieldRetentionEnrichTask,
@ -88,6 +87,8 @@ import {
ENTITY_ENGINE_RESOURCE_INIT_FAILURE_EVENT,
} from '../../telemetry/event_based/events';
import { CRITICALITY_VALUES } from '../asset_criticality/constants';
import { createEngineDescription } from './installation/engine_description';
import { convertToEntityManagerDefinition } from './entity_definitions/entity_manager_conversion';
// Workaround. TransformState type is wrong. The health type should be: TransformHealth from '@kbn/transform-plugin/common/types/transform_stats'
export interface TransformHealth extends estypes.TransformGetTransformStatsTransformStatsHealth {
@ -175,7 +176,7 @@ export class EntityStoreDataClient {
? [getEntityStoreFieldRetentionEnrichTaskStatus({ namespace, taskManager })]
: []),
getPlatformPipelineStatus({
definition,
engineId: definition.id,
esClient: this.esClient,
}),
getFieldRetentionEnrichPolicyStatus({
@ -212,9 +213,12 @@ export class EntityStoreDataClient {
new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
const { experimentalFeatures } = this.options;
const enginesTypes = experimentalFeatures.serviceEntityStoreEnabled
? [EntityTypeEnum.host, EntityTypeEnum.user, EntityTypeEnum.service]
: [EntityTypeEnum.host, EntityTypeEnum.user];
const enginesTypes: EntityType[] = [EntityTypeEnum.host, EntityTypeEnum.user];
if (experimentalFeatures.serviceEntityStoreEnabled) {
enginesTypes.push(EntityTypeEnum.service);
}
// NOTE: Whilst the Universal Entity Store is also behind a feature flag, we do not want to enable it as part of this flow as of 8.18
const promises = enginesTypes.map((entity) =>
run(() =>
@ -246,18 +250,15 @@ export class EntityStoreDataClient {
if (withComponents) {
const enginesWithComponents = await Promise.all(
engines.map(async (engine) => {
const entityDefinitionId = buildEntityDefinitionId(engine.type, namespace);
const id = buildEntityDefinitionId(engine.type, namespace);
const {
definitions: [definition],
} = await this.entityClient.getEntityDefinitions({
id: entityDefinitionId,
id,
includeState: withComponents,
});
const definitionComponents = this.getComponentFromEntityDefinition(
entityDefinitionId,
definition
);
const definitionComponents = this.getComponentFromEntityDefinition(id, definition);
const entityStoreComponents = await this.getEngineComponentsState(
engine.type,
@ -282,6 +283,19 @@ export class EntityStoreDataClient {
{ indexPattern = '', filter = '', fieldHistoryLength = 10 }: InitEntityEngineRequestBody,
{ pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
): Promise<InitEntityEngineResponse> {
const { experimentalFeatures } = this.options;
if (
entityType === EntityTypeEnum.universal &&
!experimentalFeatures.assetInventoryStoreEnabled
) {
throw new Error('Universal entity store is not enabled');
}
if (entityType === EntityTypeEnum.service && !experimentalFeatures.serviceEntityStoreEnabled) {
throw new Error('Service entity store is not enabled');
}
if (!this.options.taskManager) {
throw new Error('Task Manager is not available');
}
@ -350,40 +364,34 @@ export class EntityStoreDataClient {
const setupStartTime = moment().utc().toISOString();
const { logger, namespace, appClient, dataViewsService } = this.options;
try {
const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
const defaultIndexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
const unitedDefinition = getUnitedEntityDefinition({
indexPatterns,
const description = createEngineDescription({
entityType,
namespace,
fieldHistoryLength,
syncDelay: `${config.syncDelay.asSeconds()}s`,
frequency: `${config.frequency.asSeconds()}s`,
requestParams: { indexPattern, fieldHistoryLength },
defaultIndexPatterns,
config,
});
const { entityManagerDefinition } = unitedDefinition;
// clean up any existing entity store
await this.delete(entityType, taskManager, { deleteData: false, deleteEngine: false });
// set up the entity manager definition
const definition = convertToEntityManagerDefinition(description, {
namespace,
filter,
});
await this.entityClient.createEntityDefinition({
definition: {
...entityManagerDefinition,
filter,
indexPatterns: indexPattern
? [...entityManagerDefinition.indexPatterns, ...indexPattern.split(',')]
: entityManagerDefinition.indexPatterns,
},
definition,
installOnly: true,
});
this.log(`debug`, entityType, `Created entity definition`);
// the index must be in place with the correct mapping before the enrich policy is created
// this is because the enrich policy will fail if the index does not exist with the correct fields
await createEntityIndexComponentTemplate({
unitedDefinition,
esClient: this.esClient,
});
await createEntityIndexComponentTemplate(description, this.esClient);
this.log(`debug`, entityType, `Created entity index component template`);
await createEntityIndex({
entityType,
@ -396,20 +404,24 @@ export class EntityStoreDataClient {
// we must create and execute the enrich policy before the pipeline is created
// this is because the pipeline will fail if the enrich index does not exist
await createFieldRetentionEnrichPolicy({
unitedDefinition,
description,
options: { namespace },
esClient: this.esClient,
});
this.log(`debug`, entityType, `Created field retention enrich policy`);
await executeFieldRetentionEnrichPolicy({
unitedDefinition,
entityType,
version: description.version,
options: { namespace },
esClient: this.esClient,
logger,
});
this.log(`debug`, entityType, `Executed field retention enrich policy`);
await createPlatformPipeline({
description,
options: { namespace },
debugMode: pipelineDebugMode,
unitedDefinition,
logger,
esClient: this.esClient,
});
@ -599,18 +611,18 @@ export class EntityStoreDataClient {
const { deleteData, deleteEngine } = options;
const descriptor = await this.engineClient.maybeGet(entityType);
const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
const defaultIndexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
// TODO delete unitedDefinition from this method. we only need the id for deletion
const unitedDefinition = getUnitedEntityDefinition({
indexPatterns,
const description = createEngineDescription({
entityType,
namespace: this.options.namespace,
fieldHistoryLength: descriptor?.fieldHistoryLength ?? 10,
syncDelay: `${config.syncDelay.asSeconds()}s`,
frequency: `${config.frequency.asSeconds()}s`,
namespace,
defaultIndexPatterns,
config,
});
const { entityManagerDefinition } = unitedDefinition;
if (!description.id) {
throw new Error(`Unable to find entity definition for ${entityType}`);
}
this.log('info', entityType, `Deleting entity store`);
this.audit(
@ -623,7 +635,7 @@ export class EntityStoreDataClient {
try {
await this.entityClient
.deleteEntityDefinition({
id: entityManagerDefinition.id,
id: description.id,
deleteData,
})
// Swallowing the error as it is expected to fail if no entity definition exists
@ -633,20 +645,21 @@ export class EntityStoreDataClient {
this.log('debug', entityType, `Deleted entity definition`);
await deleteEntityIndexComponentTemplate({
unitedDefinition,
id: description.id,
esClient: this.esClient,
});
this.log('debug', entityType, `Deleted entity index component template`);
await deletePlatformPipeline({
unitedDefinition,
description,
logger,
esClient: this.esClient,
});
this.log('debug', entityType, `Deleted platform pipeline`);
await deleteFieldRetentionEnrichPolicy({
unitedDefinition,
description,
options: { namespace },
esClient: this.esClient,
logger,
});

27
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/collect_values.ts → x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/collect_values.ts
View file

@ -6,29 +6,28 @@
*/
import { isFieldMissingOrEmpty } from '../painless';
import type { BaseFieldRetentionOperator, FieldRetentionOperatorBuilder } from './types';
export interface CollectValues extends BaseFieldRetentionOperator {
operation: 'collect_values';
maxLength: number;
}
import type { FieldRetentionOperatorBuilder } from './types';
/**
* A field retention operator that collects up to `maxLength` values of the specified field.
* Values are first collected from the field, then from the enrich field if the field is not present or empty.
*/
export const collectValuesProcessor: FieldRetentionOperatorBuilder<CollectValues> = (
{ field, maxLength },
export const collectValuesProcessor: FieldRetentionOperatorBuilder = (
{ destination, retention },
{ enrichField }
) => {
const ctxField = `ctx.${field}`;
const ctxEnrichField = `ctx.${enrichField}.${field}`;
if (retention?.operation !== 'collect_values') {
throw new Error('Wrong operation for collectValuesProcessor');
}
const ctxField = `ctx.${destination}`;
const ctxEnrichField = `ctx.${enrichField}.${destination}`;
return {
script: {
lang: 'painless',
source: `
Set uniqueVals = new HashSet();
if (!(${isFieldMissingOrEmpty(ctxField)})) {
if(${ctxField} instanceof Collection) {
uniqueVals.addAll(${ctxField});
@ -36,17 +35,17 @@ export const collectValuesProcessor: FieldRetentionOperatorBuilder<CollectValues
uniqueVals.add(${ctxField});
}
}
if (uniqueVals.size() < params.max_length && !(${isFieldMissingOrEmpty(ctxEnrichField)})) {
int remaining = params.max_length - uniqueVals.size();
List historicalVals = ${ctxEnrichField}.subList(0, (int) Math.min(remaining, ${ctxEnrichField}.size()));
uniqueVals.addAll(historicalVals);
}
${ctxField} = new ArrayList(uniqueVals).subList(0, (int) Math.min(params.max_length, uniqueVals.size()));
`,
params: {
max_length: maxLength,
max_length: retention.maxLength,
},
},
};

0
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/index.ts → x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/index.ts
View file

22
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/operator_to_ingest_processor.ts → x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/operator_to_ingest_processor.ts
View file

@ -5,24 +5,30 @@
* 2.0.
*/
import type { FieldRetentionOperator, FieldRetentionOperatorBuilder } from './types';
import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
import { preferNewestValueProcessor } from './prefer_newest_value';
import { preferOldestValueProcessor } from './prefer_oldest_value';
import { collectValuesProcessor } from './collect_values';
import type { FieldDescription } from '../installation/types';
/**
* Converts a field retention operator to an ingest processor.
* An ingest processor is a step that can be added to an ingest pipeline.
*/
export const fieldOperatorToIngestProcessor: FieldRetentionOperatorBuilder<
FieldRetentionOperator
> = (fieldOperator, options) => {
switch (fieldOperator.operation) {
export const fieldOperatorToIngestProcessor = (
field: FieldDescription,
options: { enrichField: string }
): IngestProcessorContainer => {
if (!field.retention) {
throw new Error('Field retention operator is required');
}
switch (field.retention.operation) {
case 'prefer_newest_value':
return preferNewestValueProcessor(fieldOperator, options);
return preferNewestValueProcessor(field, options);
case 'prefer_oldest_value':
return preferOldestValueProcessor(fieldOperator, options);
return preferOldestValueProcessor(field, options);
case 'collect_values':
return collectValuesProcessor(fieldOperator, options);
return collectValuesProcessor(field, options);
}
};

16
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/prefer_newest_value.ts → x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/prefer_newest_value.ts
View file

@ -5,29 +5,25 @@
* 2.0.
*/
import type { BaseFieldRetentionOperator, FieldRetentionOperatorBuilder } from './types';
import type { FieldRetentionOperatorBuilder } from './types';
import { isFieldMissingOrEmpty } from '../painless';
export interface PreferNewestValue extends BaseFieldRetentionOperator {
operation: 'prefer_newest_value';
}
/**
* A field retention operator that prefers the newest value of the specified field.
* If the field is missing or empty, the value from the enrich field is used.
*/
export const preferNewestValueProcessor: FieldRetentionOperatorBuilder<PreferNewestValue> = (
{ field },
export const preferNewestValueProcessor: FieldRetentionOperatorBuilder = (
{ destination },
{ enrichField }
) => {
const latestField = `ctx.${field}`;
const historicalField = `${enrichField}.${field}`;
const latestField = `ctx.${destination}`;
const historicalField = `${enrichField}.${destination}`;
return {
set: {
if: `(${isFieldMissingOrEmpty(latestField)}) && !(${isFieldMissingOrEmpty(
`ctx.${historicalField}`
)})`,
field,
field: destination,
value: `{{${historicalField}}}`,
},
};

14
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/prefer_oldest_value.ts → x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/prefer_oldest_value.ts
View file

@ -5,26 +5,22 @@
* 2.0.
*/
import type { BaseFieldRetentionOperator, FieldRetentionOperatorBuilder } from './types';
import type { FieldRetentionOperatorBuilder } from './types';
import { isFieldMissingOrEmpty } from '../painless';
export interface PreferOldestValue extends BaseFieldRetentionOperator {
operation: 'prefer_oldest_value';
}
/**
* A field retention operator that prefers the oldest value of the specified field.
* If the historical field is missing or empty, the latest value is used.
*/
export const preferOldestValueProcessor: FieldRetentionOperatorBuilder<PreferOldestValue> = (
{ field },
export const preferOldestValueProcessor: FieldRetentionOperatorBuilder = (
{ destination },
{ enrichField }
) => {
const historicalField = `${enrichField}.${field}`;
const historicalField = `${enrichField}.${destination}`;
return {
set: {
if: `!(${isFieldMissingOrEmpty(`ctx.${historicalField}`)})`,
field,
field: destination,
value: `{{${historicalField}}}`,
},
};

18
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/types.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
import type { FieldDescription } from '../installation/types';
interface Options {
enrichField: string;
}
export type FieldRetentionOperatorBuilder = (
field: FieldDescription,
options: Options
) => IngestProcessorContainer;

34
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/types.ts
View file

@ -1,34 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store';
import type { CollectValues } from './collect_values';
import type { PreferNewestValue } from './prefer_newest_value';
import type { PreferOldestValue } from './prefer_oldest_value';
export interface FieldRetentionDefinition {
entityType: EntityType;
matchField: string;
fields: FieldRetentionOperator[];
}
export interface BaseFieldRetentionOperator {
field: string;
operation: string;
}
export interface FieldRetentionOperatorBuilderOptions {
enrichField: string;
}
export type FieldRetentionOperator = PreferNewestValue | PreferOldestValue | CollectValues;
export type FieldRetentionOperatorBuilder<O extends BaseFieldRetentionOperator> = (
operator: O,
options: FieldRetentionOperatorBuilderOptions
) => IngestProcessorContainer;

363
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts → x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.test.ts
View file

@ -5,22 +5,29 @@
* 2.0.
*/
import { getUnitedEntityDefinition } from './get_united_definition';
import { duration } from 'moment';
import { createEngineDescription } from './engine_description';
import { convertToEntityManagerDefinition } from '../entity_definitions/entity_manager_conversion';
describe('getUnitedEntityDefinition', () => {
const indexPatterns = ['test*'];
const defaultIndexPatterns = ['test*'];
describe('host', () => {
const unitedDefinition = getUnitedEntityDefinition({
const description = createEngineDescription({
entityType: 'host',
namespace: 'test',
fieldHistoryLength: 10,
indexPatterns,
syncDelay: '1m',
frequency: '1m',
requestParams: {
fieldHistoryLength: 10,
},
defaultIndexPatterns,
config: {
syncDelay: duration(60, 'seconds'),
frequency: duration(60, 'seconds'),
developer: { pipelineDebugMode: false },
},
});
it('mapping', () => {
expect(unitedDefinition.indexMappings).toMatchInlineSnapshot(`
expect(description.indexMappings).toMatchInlineSnapshot(`
Object {
"properties": Object {
"@timestamp": Object {
@ -93,83 +100,14 @@ describe('getUnitedEntityDefinition', () => {
}
`);
});
it('fieldRetentionDefinition', () => {
expect(unitedDefinition.fieldRetentionDefinition).toMatchInlineSnapshot(`
Object {
"entityType": "host",
"fields": Array [
Object {
"field": "host.domain",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "host.hostname",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "host.id",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "host.os.name",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "host.os.type",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "host.ip",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "host.mac",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "host.type",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "host.architecture",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "entity.source",
"operation": "prefer_oldest_value",
},
Object {
"field": "asset.criticality",
"operation": "prefer_newest_value",
},
Object {
"field": "host.risk.calculated_level",
"operation": "prefer_newest_value",
},
Object {
"field": "host.risk.calculated_score",
"operation": "prefer_newest_value",
},
Object {
"field": "host.risk.calculated_score_norm",
"operation": "prefer_newest_value",
},
],
"matchField": "host.name",
}
`);
});
it('entityManagerDefinition', () => {
expect(unitedDefinition.entityManagerDefinition).toMatchInlineSnapshot(`
const entityManagerDefinition = convertToEntityManagerDefinition(description, {
namespace: 'test',
filter: '',
});
expect(entityManagerDefinition).toMatchInlineSnapshot(`
Object {
"displayNameTemplate": "{{host.name}}",
"id": "security_host_test",
@ -183,10 +121,10 @@ describe('getUnitedEntityDefinition', () => {
"test*",
],
"latest": Object {
"lookbackPeriod": "24h",
"lookbackPeriod": "1d",
"settings": Object {
"frequency": "1m",
"syncDelay": "1m",
"frequency": "60s",
"syncDelay": "60s",
},
"timestampField": "@timestamp",
},
@ -323,17 +261,22 @@ describe('getUnitedEntityDefinition', () => {
});
});
describe('user', () => {
const unitedDefinition = getUnitedEntityDefinition({
const description = createEngineDescription({
entityType: 'user',
namespace: 'test',
fieldHistoryLength: 10,
indexPatterns,
syncDelay: '1m',
frequency: '1m',
requestParams: {
fieldHistoryLength: 10,
},
defaultIndexPatterns,
config: {
syncDelay: duration(60, 'seconds'),
frequency: duration(60, 'seconds'),
developer: { pipelineDebugMode: false },
},
});
it('mapping', () => {
expect(unitedDefinition.indexMappings).toMatchInlineSnapshot(`
expect(description.indexMappings).toMatchInlineSnapshot(`
Object {
"properties": Object {
"@timestamp": Object {
@ -397,68 +340,12 @@ describe('getUnitedEntityDefinition', () => {
}
`);
});
it('fieldRetentionDefinition', () => {
expect(unitedDefinition.fieldRetentionDefinition).toMatchInlineSnapshot(`
Object {
"entityType": "user",
"fields": Array [
Object {
"field": "user.domain",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "user.email",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "user.full_name",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "user.hash",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "user.id",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "user.roles",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "entity.source",
"operation": "prefer_oldest_value",
},
Object {
"field": "asset.criticality",
"operation": "prefer_newest_value",
},
Object {
"field": "user.risk.calculated_level",
"operation": "prefer_newest_value",
},
Object {
"field": "user.risk.calculated_score",
"operation": "prefer_newest_value",
},
Object {
"field": "user.risk.calculated_score_norm",
"operation": "prefer_newest_value",
},
],
"matchField": "user.name",
}
`);
});
it('entityManagerDefinition', () => {
expect(unitedDefinition.entityManagerDefinition).toMatchInlineSnapshot(`
const entityManagerDefinition = convertToEntityManagerDefinition(description, {
namespace: 'test',
filter: '',
});
expect(entityManagerDefinition).toMatchInlineSnapshot(`
Object {
"displayNameTemplate": "{{user.name}}",
"id": "security_user_test",
@ -472,10 +359,10 @@ describe('getUnitedEntityDefinition', () => {
"test*",
],
"latest": Object {
"lookbackPeriod": "24h",
"lookbackPeriod": "1d",
"settings": Object {
"frequency": "1m",
"syncDelay": "1m",
"frequency": "60s",
"syncDelay": "60s",
},
"timestampField": "@timestamp",
},
@ -589,17 +476,22 @@ describe('getUnitedEntityDefinition', () => {
});
describe('service', () => {
const unitedDefinition = getUnitedEntityDefinition({
const description = createEngineDescription({
entityType: 'service',
namespace: 'test',
fieldHistoryLength: 10,
indexPatterns,
syncDelay: '1m',
frequency: '1m',
requestParams: {
fieldHistoryLength: 10,
},
defaultIndexPatterns,
config: {
syncDelay: duration(60, 'seconds'),
frequency: duration(60, 'seconds'),
developer: { pipelineDebugMode: false },
},
});
it('mapping', () => {
expect(unitedDefinition.indexMappings).toMatchInlineSnapshot(`
expect(description.indexMappings).toMatchInlineSnapshot(`
Object {
"properties": Object {
"@timestamp": Object {
@ -645,15 +537,6 @@ describe('getUnitedEntityDefinition', () => {
"service.node.roles": Object {
"type": "keyword",
},
"service.risk.calculated_level": Object {
"type": "keyword",
},
"service.risk.calculated_score": Object {
"type": "float",
},
"service.risk.calculated_score_norm": Object {
"type": "float",
},
"service.state": Object {
"type": "keyword",
},
@ -667,81 +550,13 @@ describe('getUnitedEntityDefinition', () => {
}
`);
});
it('fieldRetentionDefinition', () => {
expect(unitedDefinition.fieldRetentionDefinition).toMatchInlineSnapshot(`
Object {
"entityType": "service",
"fields": Array [
Object {
"field": "service.address",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.environment",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.ephemeral_id",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.id",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.node.name",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.node.roles",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.state",
"operation": "prefer_newest_value",
},
Object {
"field": "service.type",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.version",
"operation": "prefer_newest_value",
},
Object {
"field": "entity.source",
"operation": "prefer_oldest_value",
},
Object {
"field": "asset.criticality",
"operation": "prefer_newest_value",
},
Object {
"field": "service.risk.calculated_level",
"operation": "prefer_newest_value",
},
Object {
"field": "service.risk.calculated_score",
"operation": "prefer_newest_value",
},
Object {
"field": "service.risk.calculated_score_norm",
"operation": "prefer_newest_value",
},
],
"matchField": "service.name",
}
`);
});
it('entityManagerDefinition', () => {
expect(unitedDefinition.entityManagerDefinition).toMatchInlineSnapshot(`
const entityManagerDefinition = convertToEntityManagerDefinition(description, {
namespace: 'test',
filter: '',
});
expect(entityManagerDefinition).toMatchInlineSnapshot(`
Object {
"displayNameTemplate": "{{service.name}}",
"id": "security_service_test",
@ -755,10 +570,10 @@ describe('getUnitedEntityDefinition', () => {
"test*",
],
"latest": Object {
"lookbackPeriod": "24h",
"lookbackPeriod": "1d",
"settings": Object {
"frequency": "1m",
"syncDelay": "1m",
"frequency": "60s",
"syncDelay": "60s",
},
"timestampField": "@timestamp",
},
@ -840,56 +655,6 @@ describe('getUnitedEntityDefinition', () => {
"destination": "service.version",
"source": "service.version",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "asc",
},
"type": "top_value",
},
"destination": "entity.source",
"source": "_index",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "asset.criticality",
"source": "asset.criticality",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_level",
"source": "service.risk.calculated_level",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_score",
"source": "service.risk.calculated_score",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_score_norm",
"source": "service.risk.calculated_score_norm",
},
],
"name": "Security 'service' Entity Store Definition",
"type": "service",

95
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts Normal file
View file

@ -0,0 +1,95 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { pipe } from 'fp-ts/lib/function';
import { assign, concat, map, merge, update } from 'lodash/fp';
import { set } from '@kbn/safer-lodash-set/fp';
import type { EntityType } from '../../../../../common/api/entity_analytics';
import {
DEFAULT_FIELD_HISTORY_LENGTH,
DEFAULT_LOOKBACK_PERIOD,
DEFAULT_TIMESTAMP_FIELD,
} from '../entity_definitions/constants';
import { generateIndexMappings } from '../elasticsearch_assets';
import {
hostEntityEngineDescription,
userEntityEngineDescription,
universalEntityEngineDescription,
serviceEntityEngineDescription,
} from '../entity_definitions/entity_descriptions';
import type { EntityStoreConfig } from '../types';
import { buildEntityDefinitionId } from '../utils';
import type { EntityDescription } from '../entity_definitions/types';
import type { EntityEngineInstallationDescriptor } from './types';
const engineDescriptionRegistry: Record<EntityType, EntityDescription> = {
host: hostEntityEngineDescription,
user: userEntityEngineDescription,
universal: universalEntityEngineDescription,
service: serviceEntityEngineDescription,
};
export const getAvailableEntityDescriptions = () =>
Object.keys(engineDescriptionRegistry) as EntityType[];
interface EngineDescriptionParams {
entityType: EntityType;
namespace: string;
config: EntityStoreConfig;
requestParams?: {
indexPattern?: string;
fieldHistoryLength?: number;
};
defaultIndexPatterns: string[];
}
export const createEngineDescription = (options: EngineDescriptionParams) => {
const { entityType, namespace, config, requestParams, defaultIndexPatterns } = options;
const fieldHistoryLength = requestParams?.fieldHistoryLength || DEFAULT_FIELD_HISTORY_LENGTH;
const indexPatterns = requestParams?.indexPattern
? defaultIndexPatterns.concat(requestParams?.indexPattern.split(','))
: defaultIndexPatterns;
const description = engineDescriptionRegistry[entityType];
const settings: EntityEngineInstallationDescriptor['settings'] = {
syncDelay: `${config.syncDelay.asSeconds()}s`,
frequency: `${config.frequency.asSeconds()}s`,
lookbackPeriod: description.settings?.lookbackPeriod || DEFAULT_LOOKBACK_PERIOD,
timestampField: description.settings?.timestampField || DEFAULT_TIMESTAMP_FIELD,
};
const updatedDescription = pipe(
description,
set('id', buildEntityDefinitionId(entityType, namespace)),
update('settings', assign(settings)),
updateIndexPatterns(indexPatterns),
updateRetentionFields(fieldHistoryLength),
addIndexMappings
) as EntityEngineInstallationDescriptor;
return updatedDescription;
};
const updateIndexPatterns = (indexPatterns: string[]) =>
update('indexPatterns', (prev = []) => concat(indexPatterns, prev));
const updateRetentionFields = (fieldHistoryLength: number) =>
update(
'fields',
map(
merge({
retention: { maxLength: fieldHistoryLength },
aggregation: { limit: fieldHistoryLength },
})
)
);
const addIndexMappings = (description: EntityEngineInstallationDescriptor) =>
set('indexMappings', generateIndexMappings(description), description);

73
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/types.ts Normal file
View file

@ -0,0 +1,73 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type {
MappingTypeMapping,
IngestProcessorContainer,
MappingProperty,
} from '@elastic/elasticsearch/lib/api/types';
import type { EntityDefinition } from '@kbn/entities-schema';
import type { EntityType } from '../../../../../common/api/entity_analytics';
export type EntityDefinitionMetadataElement = NonNullable<EntityDefinition['metadata']>[number];
export interface EntityEngineInstallationDescriptor {
id: string;
version: string;
entityType: EntityType;
identityField: string;
/**
* Default static index patterns to use as the source of entity data.
* By default, the Security Data View patterns will be added to this list.
* API parameters can be used to add additional patterns.
**/
indexPatterns: string[];
/**
* The mappings for the entity store index.
*/
indexMappings: MappingTypeMapping;
/**
* Field descriptions for the entity.
* Identity fields are not included here as they are treated separately.
*/
fields: FieldDescription[];
/**
* Entity manager default pivot transform settings.
* Any kibana.yml configuration will override these settings.
*/
settings: {
syncDelay: string;
frequency: string;
lookbackPeriod: string;
timestampField: string;
};
/**
* The ingest pipeline to apply to the entity data.
* This can be an array of processors which get appended to the default pipeline,
* or a function that takes the default processors and returns an array of processors.
**/
pipeline:
| IngestProcessorContainer[]
| ((defaultProcessors: IngestProcessorContainer[]) => IngestProcessorContainer[]);
}
export type FieldDescription = EntityDefinitionMetadataElement & {
mapping: MappingProperty;
retention: FieldRetentionOp;
};
export type FieldRetentionOp =
| { operation: 'collect_values'; maxLength: number }
| { operation: 'prefer_newest_value' }
| { operation: 'prefer_oldest_value' };

13
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/task/field_retention_enrichment_task.ts
View file

@ -13,7 +13,6 @@ import type {
TaskManagerSetupContract,
TaskManagerStartContract,
} from '@kbn/task-manager-plugin/server';
import { getAvailableEntityTypes } from '../../../../../common/entity_analytics/entity_store/constants';
import {
EngineComponentResourceEnum,
type EntityType,
@ -25,7 +24,7 @@ import {
} from './state';
import { INTERVAL, SCOPE, TIMEOUT, TYPE, VERSION } from './constants';
import type { EntityAnalyticsRoutesDeps } from '../../types';
import { getUnitedEntityDefinitionVersion } from '../united_entity_definitions';
import { executeFieldRetentionEnrichPolicy } from '../elasticsearch_assets';
import { getEntitiesIndexName } from '../utils';
@ -33,6 +32,8 @@ import {
FIELD_RETENTION_ENRICH_POLICY_EXECUTION_EVENT,
ENTITY_STORE_USAGE_EVENT,
} from '../../../telemetry/event_based/events';
import { VERSIONS_BY_ENTITY_TYPE } from '../entity_definitions/constants';
import { getAvailableEntityDescriptions } from '../installation/engine_description';
const logFactory =
(logger: Logger, taskId: string) =>
@ -79,10 +80,10 @@ export const registerEntityStoreFieldRetentionEnrichTask = ({
const [coreStart, _] = await getStartServices();
const esClient = coreStart.elasticsearch.client.asInternalUser;
const unitedDefinitionVersion = getUnitedEntityDefinitionVersion(entityType);
return executeFieldRetentionEnrichPolicy({
unitedDefinition: { namespace, entityType, version: unitedDefinitionVersion },
entityType,
version: VERSIONS_BY_ENTITY_TYPE[entityType],
options: { namespace },
esClient,
logger,
});
@ -202,7 +203,7 @@ export const runTask = async ({
return { state: updatedState };
}
const entityTypes = getAvailableEntityTypes();
const entityTypes = getAvailableEntityDescriptions();
for (const entityType of entityTypes) {
const start = Date.now();

28
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/constants.ts
View file

@ -1,28 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { MappingProperties } from './types';
export const BASE_ENTITY_INDEX_MAPPING: MappingProperties = {
'@timestamp': {
type: 'date',
},
'asset.criticality': {
type: 'keyword',
},
'entity.name': {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
'entity.source': {
type: 'keyword',
},
};

86
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/definition_utils.ts
View file

@ -1,86 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
import type { UnitedDefinitionField } from './types';
export const collectValues = ({
field,
mapping = { type: 'keyword' },
sourceField,
fieldHistoryLength,
}: {
field: string;
mapping?: MappingProperty;
sourceField?: string;
fieldHistoryLength: number;
}): UnitedDefinitionField => ({
field,
definition: {
source: sourceField ?? field,
destination: field,
aggregation: {
type: 'terms',
limit: fieldHistoryLength,
},
},
retention_operator: { operation: 'collect_values', field, maxLength: fieldHistoryLength },
mapping,
});
export const collectValuesWithLength =
(fieldHistoryLength: number) =>
(opts: { field: string; mapping?: MappingProperty; sourceField?: string }) =>
collectValues({ ...opts, fieldHistoryLength });
export const newestValue = ({
field,
mapping = { type: 'keyword' },
sourceField,
}: {
field: string;
mapping?: MappingProperty;
sourceField?: string;
}): UnitedDefinitionField => ({
field,
definition: {
source: sourceField ?? field,
destination: field,
aggregation: {
type: 'top_value',
sort: {
'@timestamp': 'desc',
},
},
},
retention_operator: { operation: 'prefer_newest_value', field },
mapping,
});
export const oldestValue = ({
field,
mapping = { type: 'keyword' },
sourceField,
}: {
field: string;
mapping?: MappingProperty;
sourceField?: string;
}): UnitedDefinitionField => ({
field,
definition: {
source: sourceField ?? field,
destination: field,
aggregation: {
type: 'top_value',
sort: {
'@timestamp': 'asc',
},
},
},
retention_operator: { operation: 'prefer_oldest_value', field },
mapping,
});

46
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/common.ts
View file

@ -1,46 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { EntityType } from '../../../../../../common/api/entity_analytics/entity_store';
import { getIdentityFieldForEntityType } from '../../utils';
import { oldestValue, newestValue } from '../definition_utils';
import type { UnitedDefinitionField } from '../types';
export const getCommonUnitedFieldDefinitions = ({
entityType,
fieldHistoryLength,
}: {
entityType: EntityType;
fieldHistoryLength: number;
}): UnitedDefinitionField[] => {
const identityField = getIdentityFieldForEntityType(entityType);
return [
oldestValue({
sourceField: '_index',
field: 'entity.source',
}),
newestValue({ field: 'asset.criticality' }),
newestValue({
field: `${entityType}.risk.calculated_level`,
}),
newestValue({
field: `${entityType}.risk.calculated_score`,
mapping: {
type: 'float',
},
}),
newestValue({
field: `${entityType}.risk.calculated_score_norm`,
mapping: {
type: 'float',
},
}),
{
field: identityField,
},
];
};

44
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/host.ts
View file

@ -1,44 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { collectValuesWithLength } from '../definition_utils';
import type { UnitedDefinitionBuilder } from '../types';
export const HOST_DEFINITION_VERSION = '1.0.0';
export const getHostUnitedDefinition: UnitedDefinitionBuilder = (fieldHistoryLength: number) => {
const collect = collectValuesWithLength(fieldHistoryLength);
return {
entityType: 'host',
version: HOST_DEFINITION_VERSION,
fields: [
collect({ field: 'host.domain' }),
collect({ field: 'host.hostname' }),
collect({ field: 'host.id' }),
collect({
field: 'host.os.name',
mapping: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
}),
collect({ field: 'host.os.type' }),
collect({
field: 'host.ip',
mapping: {
type: 'ip',
},
}),
collect({ field: 'host.mac' }),
collect({ field: 'host.type' }),
collect({ field: 'host.architecture' }),
],
};
};

29
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/service.ts
View file

@ -1,29 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { collectValuesWithLength, newestValue } from '../definition_utils';
import type { UnitedDefinitionBuilder } from '../types';
export const SERVICE_DEFINITION_VERSION = '1.0.0';
export const getServiceUnitedDefinition: UnitedDefinitionBuilder = (fieldHistoryLength: number) => {
const collect = collectValuesWithLength(fieldHistoryLength);
return {
entityType: 'service',
version: SERVICE_DEFINITION_VERSION,
fields: [
collect({ field: 'service.address' }),
collect({ field: 'service.environment' }),
collect({ field: 'service.ephemeral_id' }),
collect({ field: 'service.id' }),
collect({ field: 'service.node.name' }),
collect({ field: 'service.node.roles' }),
newestValue({ field: 'service.state' }),
collect({ field: 'service.type' }),
newestValue({ field: 'service.version' }),
],
};
};

35
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/user.ts
View file

@ -1,35 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { collectValuesWithLength } from '../definition_utils';
import type { UnitedDefinitionBuilder } from '../types';
export const USER_DEFINITION_VERSION = '1.0.0';
export const getUserUnitedDefinition: UnitedDefinitionBuilder = (fieldHistoryLength: number) => {
const collect = collectValuesWithLength(fieldHistoryLength);
return {
entityType: 'user',
version: USER_DEFINITION_VERSION,
fields: [
collect({ field: 'user.domain' }),
collect({ field: 'user.email' }),
collect({
field: 'user.full_name',
mapping: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
}),
collect({ field: 'user.hash' }),
collect({ field: 'user.id' }),
collect({ field: 'user.roles' }),
],
};
};

69
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts
View file

@ -1,69 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { memoize } from 'lodash';
import type { EntityType } from '../../../../../common/api/entity_analytics';
import {
getHostUnitedDefinition,
getUserUnitedDefinition,
getCommonUnitedFieldDefinitions,
USER_DEFINITION_VERSION,
HOST_DEFINITION_VERSION,
getServiceUnitedDefinition,
} from './entity_types';
import type { UnitedDefinitionBuilder } from './types';
import { UnitedEntityDefinition } from './united_entity_definition';
const unitedDefinitionBuilders: Record<EntityType, UnitedDefinitionBuilder> = {
host: getHostUnitedDefinition,
user: getUserUnitedDefinition,
service: getServiceUnitedDefinition,
};
interface Options {
entityType: EntityType;
namespace: string;
fieldHistoryLength: number;
indexPatterns: string[];
syncDelay: string;
frequency: string;
}
export const getUnitedEntityDefinition = memoize(
({
entityType,
namespace,
fieldHistoryLength,
indexPatterns,
syncDelay,
frequency,
}: Options): UnitedEntityDefinition => {
const unitedDefinition = unitedDefinitionBuilders[entityType](fieldHistoryLength);
unitedDefinition.fields.push(
...getCommonUnitedFieldDefinitions({
entityType,
fieldHistoryLength,
})
);
return new UnitedEntityDefinition({
...unitedDefinition,
namespace,
indexPatterns,
syncDelay,
frequency,
});
}
);
const versionByEntityType: Record<EntityType, string> = {
host: HOST_DEFINITION_VERSION,
user: USER_DEFINITION_VERSION,
service: USER_DEFINITION_VERSION,
};
export const getUnitedEntityDefinitionVersion = (entityType: EntityType): string =>
versionByEntityType[entityType];

10
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/index.ts
View file

@ -1,10 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export * from './get_united_definition';
export { UnitedEntityDefinition } from './united_entity_definition';

30
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/types.ts
View file

@ -1,30 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { MappingProperty, MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
import type { EntityDefinition } from '@kbn/entities-schema';
import type { EntityType } from '../../../../../common/api/entity_analytics';
import type { FieldRetentionOperator } from '../field_retention_definition';
export type MappingProperties = NonNullable<MappingTypeMapping['properties']>;
type EntityDefinitionMetadataElement = NonNullable<EntityDefinition['metadata']>[number];
export interface UnitedDefinitionField {
field: string;
retention_operator?: FieldRetentionOperator;
mapping?: MappingProperty;
definition?: EntityDefinitionMetadataElement;
}
export interface UnitedEntityDefinitionConfig {
version: string;
entityType: EntityType;
fields: UnitedDefinitionField[];
}
export type UnitedDefinitionBuilder = (fieldHistoryLength: number) => UnitedEntityDefinitionConfig;

126
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts
View file

@ -1,126 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema';
import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
import { DEFAULT_LOOKBACK_PERIOD } from '../constants';
import { buildEntityDefinitionId, getIdentityFieldForEntityType } from '../utils';
import type {
FieldRetentionDefinition,
FieldRetentionOperator,
} from '../field_retention_definition';
import type { MappingProperties, UnitedDefinitionField } from './types';
import { BASE_ENTITY_INDEX_MAPPING } from './constants';
export class UnitedEntityDefinition {
version: string;
entityType: EntityType;
indexPatterns: string[];
fields: UnitedDefinitionField[];
namespace: string;
entityManagerDefinition: EntityDefinition;
fieldRetentionDefinition: FieldRetentionDefinition;
indexMappings: MappingTypeMapping;
syncDelay: string;
frequency: string;
constructor(opts: {
version: string;
entityType: EntityType;
indexPatterns: string[];
fields: UnitedDefinitionField[];
namespace: string;
syncDelay: string;
frequency: string;
}) {
this.version = opts.version;
this.entityType = opts.entityType;
this.indexPatterns = opts.indexPatterns;
this.fields = opts.fields;
this.frequency = opts.frequency;
this.syncDelay = opts.syncDelay;
this.namespace = opts.namespace;
this.entityManagerDefinition = this.toEntityManagerDefinition();
this.fieldRetentionDefinition = this.toFieldRetentionDefinition();
this.indexMappings = this.toIndexMappings();
}
private toEntityManagerDefinition(): EntityDefinition {
const { entityType, namespace, indexPatterns, syncDelay, frequency } = this;
const identityField = getIdentityFieldForEntityType(this.entityType);
const metadata = this.fields
.filter((field) => field.definition)
.map((field) => field.definition!); // eslint-disable-line @typescript-eslint/no-non-null-assertion
return entityDefinitionSchema.parse({
id: buildEntityDefinitionId(entityType, namespace),
name: `Security '${entityType}' Entity Store Definition`,
type: entityType,
indexPatterns,
identityFields: [identityField],
displayNameTemplate: `{{${identityField}}}`,
metadata,
latest: {
timestampField: '@timestamp',
lookbackPeriod: DEFAULT_LOOKBACK_PERIOD,
settings: {
syncDelay,
frequency,
},
},
version: this.version,
managed: true,
});
}
private toFieldRetentionDefinition(): FieldRetentionDefinition {
return {
entityType: this.entityType,
matchField: getIdentityFieldForEntityType(this.entityType),
fields: this.fields
.filter((field) => field.retention_operator !== undefined)
.map((field) => field.retention_operator as FieldRetentionOperator),
};
}
private toIndexMappings(): MappingTypeMapping {
const identityField = getIdentityFieldForEntityType(this.entityType);
const initialMappings: MappingProperties = {
...BASE_ENTITY_INDEX_MAPPING,
[identityField]: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
};
const properties = this.fields.reduce((acc, { field, mapping }) => {
if (!mapping) {
return acc;
}
acc[field] = mapping;
return acc;
}, initialMappings);
properties[identityField] = {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
};
return {
properties,
};
}
}

5
x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts
View file

@ -11,17 +11,12 @@ import {
entitiesIndexPattern,
} from '@kbn/entities-schema';
import type { DataViewsService, DataView } from '@kbn/data-views-plugin/common';
import { IDENTITY_FIELD_MAP } from '../../../../../common/entity_analytics/entity_store/constants';
import type { AppClient } from '../../../../types';
import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine';
import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality';
import { type EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
import { entityEngineDescriptorTypeName } from '../saved_object';
export const getIdentityFieldForEntityType = (entityType: EntityType) => {
return IDENTITY_FIELD_MAP[entityType];
};
export const buildIndexPatterns = async (
space: string,
appClient: AppClient,

264
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/field_retention_operators.ts
View file

@ -6,10 +6,8 @@
*/
import expect from '@kbn/expect';
import {
FieldRetentionOperator,
fieldOperatorToIngestProcessor,
} from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/field_retention_definition';
import { fieldOperatorToIngestProcessor } from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/field_retention';
import { FieldDescription } from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/installation/types';
import { FtrProviderContext } from '../../../../ftr_provider_context';
import { applyIngestProcessorToDoc } from '../utils/ingest';
export default ({ getService }: FtrProviderContext) => {
@ -22,10 +20,7 @@ export default ({ getService }: FtrProviderContext) => {
expect(aSorted).to.eql(bSorted);
};
const applyOperatorToDoc = async (
operator: FieldRetentionOperator,
docSource: any
): Promise<any> => {
const applyOperatorToDoc = async (operator: FieldDescription, docSource: any): Promise<any> => {
const step = fieldOperatorToIngestProcessor(operator, { enrichField: 'historical' });
return applyIngestProcessorToDoc([step], docSource, es, log);
@ -34,10 +29,16 @@ export default ({ getService }: FtrProviderContext) => {
describe('@ess @serverless @skipInServerlessMKI Entity store - Field Retention Pipeline Steps', () => {
describe('collect_values operator', () => {
it('should return value if no history', async () => {
const op: FieldRetentionOperator = {
operation: 'collect_values',
field: 'test_field',
maxLength: 10,
const op: FieldDescription = {
retention: { operation: 'collect_values', maxLength: 10 },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -50,10 +51,16 @@ export default ({ getService }: FtrProviderContext) => {
});
it('should not take from history if latest field has maxLength values', async () => {
const op: FieldRetentionOperator = {
operation: 'collect_values',
field: 'test_field',
maxLength: 1,
const op: FieldDescription = {
retention: { operation: 'collect_values', maxLength: 1 },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -69,10 +76,16 @@ export default ({ getService }: FtrProviderContext) => {
});
it("should take from history if latest field doesn't have maxLength values", async () => {
const op: FieldRetentionOperator = {
operation: 'collect_values',
field: 'test_field',
maxLength: 10,
const op: FieldDescription = {
retention: { operation: 'collect_values', maxLength: 10 },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -88,10 +101,16 @@ export default ({ getService }: FtrProviderContext) => {
});
it('should only take from history up to maxLength values', async () => {
const op: FieldRetentionOperator = {
operation: 'collect_values',
field: 'test_field',
maxLength: 2,
const op: FieldDescription = {
retention: { operation: 'collect_values', maxLength: 2 },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -107,10 +126,16 @@ export default ({ getService }: FtrProviderContext) => {
});
it('should handle value not being an array', async () => {
const op: FieldRetentionOperator = {
operation: 'collect_values',
field: 'test_field',
maxLength: 2,
const op: FieldDescription = {
retention: { operation: 'collect_values', maxLength: 2 },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
test_field: 'foo',
@ -125,10 +150,16 @@ export default ({ getService }: FtrProviderContext) => {
});
it('should handle missing values', async () => {
const op: FieldRetentionOperator = {
operation: 'collect_values',
field: 'test_field',
maxLength: 2,
const op: FieldDescription = {
retention: { operation: 'collect_values', maxLength: 2 },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {};
@ -139,9 +170,16 @@ export default ({ getService }: FtrProviderContext) => {
});
describe('prefer_newest_value operator', () => {
it('should return latest value if no history value', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_newest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_newest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -153,9 +191,16 @@ export default ({ getService }: FtrProviderContext) => {
expect(resultDoc.test_field).to.eql('latest');
});
it('should return history value if no latest value (undefined)', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_newest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_newest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -169,9 +214,16 @@ export default ({ getService }: FtrProviderContext) => {
expect(resultDoc.test_field).to.eql('historical');
});
it('should return history value if no latest value (empty string)', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_newest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_newest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -186,9 +238,16 @@ export default ({ getService }: FtrProviderContext) => {
expect(resultDoc.test_field).to.eql('historical');
});
it('should return history value if no latest value (empty array)', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_newest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_newest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -203,9 +262,16 @@ export default ({ getService }: FtrProviderContext) => {
expect(resultDoc.test_field).to.eql('historical');
});
it('should return history value if no latest value (empty object)', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_newest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_newest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -220,9 +286,16 @@ export default ({ getService }: FtrProviderContext) => {
expect(resultDoc.test_field).to.eql('historical');
});
it('should return latest value if both latest and history values', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_newest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_newest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -238,9 +311,16 @@ export default ({ getService }: FtrProviderContext) => {
});
it('should handle missing values', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_newest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_newest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {};
@ -251,9 +331,16 @@ export default ({ getService }: FtrProviderContext) => {
});
describe('prefer_oldest_value operator', () => {
it('should return history value if no latest value', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_oldest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_oldest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -267,9 +354,16 @@ export default ({ getService }: FtrProviderContext) => {
expect(resultDoc.test_field).to.eql('historical');
});
it('should return latest value if no history value (undefined)', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_oldest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_oldest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -284,9 +378,16 @@ export default ({ getService }: FtrProviderContext) => {
expect(resultDoc.test_field).to.eql('latest');
});
it('should return latest value if no history value (empty string)', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_oldest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_oldest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -301,9 +402,16 @@ export default ({ getService }: FtrProviderContext) => {
expect(resultDoc.test_field).to.eql('latest');
});
it('should return latest value if no history value (empty array)', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_oldest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_oldest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -318,9 +426,16 @@ export default ({ getService }: FtrProviderContext) => {
expect(resultDoc.test_field).to.eql('latest');
});
it('should return latest value if no history value (empty object)', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_oldest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_oldest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {
@ -335,9 +450,16 @@ export default ({ getService }: FtrProviderContext) => {
expect(resultDoc.test_field).to.eql('latest');
});
it('should return history value if both latest and history values', async () => {
const op: FieldRetentionOperator = {
operation: 'prefer_oldest_value',
field: 'test_field',
const op: FieldDescription = {
retention: { operation: 'prefer_oldest_value' },
destination: 'test_field',
source: 'test_field',
aggregation: {
type: 'terms',
limit: 10,
lookbackPeriod: undefined,
},
mapping: { type: 'keyword' },
};
const doc = {

2
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality_csv_upload.ts
View file

@ -109,7 +109,7 @@ export default ({ getService }: FtrProviderContext) => {
{
index: 2,
message:
'Invalid entity type "invalid_entity", expected to be one of: user, host, service',
'Invalid entity type "invalid_entity", expected to be one of: user, host, service, universal',
},
{
index: 3,