[SecuritySolution] Service Entity Store (#202344)

## Summary


### Service Definition:

https://github.com/elastic/kibana/pull/202344/files#diff-42c7dd345e0500c97f85824904a70a11162827ea8f8df6982082a9047ca04ff1


### Acceptance Criteria
- [x] Upon installation of the entity store, the Service entity
definition should be created by default
- [x] The Service definition will be installed in the exact same way as
the User and Host definitions
- [x] The unique identifier for service entities will be `service.name`
- [x] The fields captured for service entities should match the field
mapping spreadsheet (see Implementation Notes below)


### Stored Entity
```json
{
          "@timestamp": "2024-12-02T10:43:13.856Z",
          "event": {
            "ingested": "2024-12-02T10:51:28.987428Z"
          },
          "entity": {
            "name": "test123 name",
            "id": "test123 name",
            "source": "logs-blito",
            "type": "service"
          },
          "service": {
            "node": {
              "roles": [
                "test123 node roles"
              ],
              "name": [
                "test123 node name"
              ]
            },
            "environment": [
              "test123 environment"
            ],
            "address": [
              "test123 address"
            ],
            "name": "test123 name",
            "id": [
              "test123 id"
            ],
            "state": [
              "test123 state"
            ],
            "ephemeral_id": [
              "test123 ephemeral_id"
            ],
            "type": [
              "test123 type"
            ],
            "version": [
              "test123 version"
            ]
          }
}
```

### How to test it?

* Start Kibana
<details>
  <summary>Create mappings</summary>
  
```
PUT /logs-test
{
  "mappings": {
    "properties": {      
      "service.name": {
        "type": "keyword"
      },
      "service.address": {
        "type": "keyword"
      },
      "service.environment": {
        "type": "keyword"
      },
      "service.ephemeral_id": {
        "type": "keyword"
      },
      "service.id": {
        "type": "keyword"
      },
      "service.node.name": {
        "type": "keyword"
      },
      "service.node.roles": {
        "type": "keyword"
      },
      "service.state": {
        "type": "keyword"
      },
      "service.type": {
        "type": "keyword"
      },
      "service.version": {
        "type": "keyword"
      },
      "@timestamp": {
        "type": "date"
      }
    }
  }
}
```` 
</details>


<details>
  <summary>Create document</summary>
  
```
PUT /logs-test
POST logs-test/_doc
{
  "service": {
    "name": "test123 name",
    "address": "test123 address",
    "environment": "test123 environment",
    "ephemeral_id": "test123 ephemeral_id",
    "id": "test123 id",
    "node.roles": "test123 node roles",
    "node.name": "test123 node name",    
    "state": "test123 state",
    "type": "test123 type",
    "version": "test123 version"
  },
  "@timestamp": "2024-12-02T10:43:13.856Z"
}

```` 
</details>

* Init the entity store
* Wait...
* Query the service index `GET
.entities.v1.latest.security_service_default/_search`


### Open Questions
* Can we merge this PR without first updating all other features that
will use service entities?
* If we merge it, the service engine will be installed together with
other entities, but it won't provide any functionality
* Do we need an experimental flag?

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Pablo Machado 2024-12-09 18:12:51 +01:00 committed by GitHub
parent 58b8b47928
commit fdedae07b8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 417 additions and 20 deletions

View file

@ -47102,6 +47102,7 @@ components:
enum:
- user
- host
- service
type: string
Security_Entity_Analytics_API_HostEntity:
type: object

View file

@ -54783,6 +54783,7 @@ components:
enum:
- user
- host
- service
type: string
Security_Entity_Analytics_API_HostEntity:
type: object

View file

@ -17,7 +17,7 @@
import { z } from '@kbn/zod';
export type EntityType = z.infer<typeof EntityType>;
export const EntityType = z.enum(['user', 'host']);
export const EntityType = z.enum(['user', 'host', 'service']);
export type EntityTypeEnum = typeof EntityType.enum;
export const EntityTypeEnum = EntityType.enum;

View file

@ -11,6 +11,7 @@ components:
enum:
- user
- host
- service
EngineDescriptor:
type: object

View file

@ -243,6 +243,11 @@ export const allowedExperimentalValues = Object.freeze({
*/
entityStoreDisabled: false,
/**
* Enables the Service Entity Store. The Entity Store feature will install the service engine by default.
*/
serviceEntityStoreEnabled: true,
/**
* Enables the siem migrations feature
*/

View file

@ -987,6 +987,7 @@ components:
enum:
- user
- host
- service
type: string
HostEntity:
type: object

View file

@ -987,6 +987,7 @@ components:
enum:
- user
- host
- service
type: string
HostEntity:
type: object

View file

@ -12,10 +12,11 @@ import type { IHttpFetchError } from '@kbn/core-http-browser';
import type { GetEntityStoreStatusResponse } from '../../../../../common/api/entity_analytics/entity_store/status.gen';
import type { InitEntityStoreResponse } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
import { useKibana } from '../../../../common/lib/kibana/kibana_react';
import type {
DeleteEntityEngineResponse,
InitEntityEngineResponse,
StopEntityEngineResponse,
import type { EntityType } from '../../../../../common/api/entity_analytics';
import {
type DeleteEntityEngineResponse,
type InitEntityEngineResponse,
type StopEntityEngineResponse,
} from '../../../../../common/api/entity_analytics';
import { useEntityStoreRoutes } from '../../../api/entity_store';
import { EntityEventTypes } from '../../../../common/lib/telemetry';
@ -68,13 +69,16 @@ export const useEnableEntityStoreMutation = (options?: UseMutationOptions<{}>) =
};
export const INIT_ENTITY_ENGINE_STATUS_KEY = ['POST', 'INIT_ENTITY_ENGINE'];
/**
* @deprecated
* It will be deleted on a follow-up PR
*/
export const useInitEntityEngineMutation = (options?: UseMutationOptions<{}>) => {
const queryClient = useQueryClient();
const { initEntityEngine } = useEntityStoreRoutes();
return useMutation<InitEntityEngineResponse[]>(
() => Promise.all([initEntityEngine('user'), initEntityEngine('host')]),
{
mutationKey: INIT_ENTITY_ENGINE_STATUS_KEY,
onSuccess: () => queryClient.refetchQueries({ queryKey: ENTITY_STORE_STATUS }),
@ -84,7 +88,7 @@ export const useInitEntityEngineMutation = (options?: UseMutationOptions<{}>) =>
};
export const STOP_ENTITY_ENGINE_STATUS_KEY = ['POST', 'STOP_ENTITY_ENGINE'];
export const useStopEntityEngineMutation = (options?: UseMutationOptions<{}>) => {
export const useStopEntityEngineMutation = (entityTypes: EntityType[]) => {
const { telemetry } = useKibana().services;
const queryClient = useQueryClient();
@ -95,23 +99,28 @@ export const useStopEntityEngineMutation = (options?: UseMutationOptions<{}>) =>
timestamp: new Date().toISOString(),
action: 'stop',
});
return Promise.all([stopEntityEngine('user'), stopEntityEngine('host')]);
return Promise.all(entityTypes.map((entityType) => stopEntityEngine(entityType)));
},
{
mutationKey: STOP_ENTITY_ENGINE_STATUS_KEY,
onSuccess: () => queryClient.refetchQueries({ queryKey: ENTITY_STORE_STATUS }),
...options,
}
);
};
export const DELETE_ENTITY_ENGINE_STATUS_KEY = ['POST', 'STOP_ENTITY_ENGINE'];
export const useDeleteEntityEngineMutation = ({ onSuccess }: { onSuccess?: () => void }) => {
export const useDeleteEntityEngineMutation = ({
onSuccess,
entityTypes,
}: {
onSuccess?: () => void;
entityTypes: EntityType[];
}) => {
const queryClient = useQueryClient();
const { deleteEntityEngine } = useEntityStoreRoutes();
return useMutation<DeleteEntityEngineResponse[]>(
() => Promise.all([deleteEntityEngine('user', true), deleteEntityEngine('host', true)]),
() => Promise.all(entityTypes.map((entityType) => deleteEntityEngine(entityType, true))),
{
mutationKey: DELETE_ENTITY_ENGINE_STATUS_KEY,
onSuccess: () => {

View file

@ -33,7 +33,7 @@ import React, { useCallback, useEffect, useState } from 'react';
import { FormattedMessage } from '@kbn/i18n-react';
import type { SecurityAppError } from '@kbn/securitysolution-t-grid';
import type { StoreStatus } from '../../../common/api/entity_analytics';
import { EntityType, EntityTypeEnum, type StoreStatus } from '../../../common/api/entity_analytics';
import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features';
import { ASSET_CRITICALITY_INDEX_PATTERN } from '../../../common/entity_analytics/asset_criticality';
import { useKibana } from '../../common/lib/kibana';
@ -73,13 +73,20 @@ export const EntityStoreManagementPage = () => {
const hasAssetCriticalityWritePermissions = assetCriticalityPrivileges?.has_write_permissions;
const [selectedTabId, setSelectedTabId] = useState(TabId.Import);
const entityStoreStatus = useEntityStoreStatus({});
const isServiceEntityStoreEnabled = useIsExperimentalFeatureEnabled('serviceEntityStoreEnabled');
const allEntityTypes = Object.values(EntityType.Values);
const entityTypes = isServiceEntityStoreEnabled
? allEntityTypes
: allEntityTypes.filter((value) => value !== EntityTypeEnum.service);
const enableStoreMutation = useEnableEntityStoreMutation();
const stopEntityEngineMutation = useStopEntityEngineMutation();
const stopEntityEngineMutation = useStopEntityEngineMutation(entityTypes);
const deleteEntityEngineMutation = useDeleteEntityEngineMutation({
onSuccess: () => {
closeClearModal();
},
entityTypes,
});
const [isClearModalVisible, setIsClearModalVisible] = useState(false);

View file

@ -16,6 +16,7 @@ import type { EntityType } from '../../../../common/api/entity_analytics/entity_
import type { DataViewsService } from '@kbn/data-views-plugin/common';
import type { AppClient } from '../../..';
import type { EntityStoreConfig } from './types';
import { mockGlobalState } from '../../../../public/common/mock';
describe('EntityStoreDataClient', () => {
const mockSavedObjectClient = savedObjectsClientMock.create();
@ -31,6 +32,7 @@ describe('EntityStoreDataClient', () => {
dataViewsService: {} as DataViewsService,
appClient: {} as AppClient,
config: {} as EntityStoreConfig,
experimentalFeatures: mockGlobalState.app.enableExperimental,
});
const defaultSearchParams = {

View file

@ -23,6 +23,7 @@ import moment from 'moment';
import type { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types';
import type { EntityDefinition } from '@kbn/entities-schema';
import type { estypes } from '@elastic/elasticsearch';
import type { ExperimentalFeatures } from '../../../../common';
import type {
GetEntityStoreStatusRequestQuery,
GetEntityStoreStatusResponse,
@ -32,7 +33,10 @@ import type {
InitEntityStoreResponse,
} from '../../../../common/api/entity_analytics/entity_store/enable.gen';
import type { AppClient } from '../../..';
import { EngineComponentResourceEnum, EntityType } from '../../../../common/api/entity_analytics';
import {
EngineComponentResourceEnum,
EntityTypeEnum,
} from '../../../../common/api/entity_analytics';
import type {
Entity,
EngineDataviewUpdateResult,
@ -42,6 +46,7 @@ import type {
ListEntityEnginesResponse,
EngineComponentStatus,
EngineComponentResource,
EntityType,
} from '../../../../common/api/entity_analytics';
import { EngineDescriptorClient } from './saved_object/engine_descriptor';
import { ENGINE_STATUS, ENTITY_STORE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
@ -108,6 +113,7 @@ interface EntityStoreClientOpts {
dataViewsService: DataViewsService;
appClient: AppClient;
config: EntityStoreConfig;
experimentalFeatures: ExperimentalFeatures;
telemetry?: AnalyticsServiceSetup;
}
@ -204,7 +210,13 @@ export class EntityStoreDataClient {
// Immediately defer the initialization to the next tick. This way we don't block on the init preflight checks
const run = <T>(fn: () => Promise<T>) =>
new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
const promises = Object.values(EntityType.Values).map((entity) =>
const { experimentalFeatures } = this.options;
const enginesTypes = experimentalFeatures.serviceEntityStoreEnabled
? [EntityTypeEnum.host, EntityTypeEnum.user, EntityTypeEnum.service]
: [EntityTypeEnum.host, EntityTypeEnum.user];
const promises = enginesTypes.map((entity) =>
run(() =>
this.init(entity, { indexPattern, filter, fieldHistoryLength }, { pipelineDebugMode })
)

View file

@ -7,4 +7,5 @@
export * from './host';
export * from './user';
export * from './service';
export { getCommonUnitedFieldDefinitions } from './common';

View file

@ -0,0 +1,29 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { collectValuesWithLength, newestValue } from '../definition_utils';
import type { UnitedDefinitionBuilder } from '../types';
export const SERVICE_DEFINITION_VERSION = '1.0.0';
export const getServiceUnitedDefinition: UnitedDefinitionBuilder = (fieldHistoryLength: number) => {
const collect = collectValuesWithLength(fieldHistoryLength);
return {
entityType: 'service',
version: SERVICE_DEFINITION_VERSION,
fields: [
collect({ field: 'service.address' }),
collect({ field: 'service.environment' }),
collect({ field: 'service.ephemeral_id' }),
collect({ field: 'service.id' }),
collect({ field: 'service.node.name' }),
collect({ field: 'service.node.roles' }),
newestValue({ field: 'service.state' }),
collect({ field: 'service.type' }),
newestValue({ field: 'service.version' }),
],
};
};

View file

@ -587,4 +587,315 @@ describe('getUnitedEntityDefinition', () => {
`);
});
});
describe('service', () => {
const unitedDefinition = getUnitedEntityDefinition({
entityType: 'service',
namespace: 'test',
fieldHistoryLength: 10,
indexPatterns,
syncDelay: '1m',
frequency: '1m',
});
it('mapping', () => {
expect(unitedDefinition.indexMappings).toMatchInlineSnapshot(`
Object {
"properties": Object {
"@timestamp": Object {
"type": "date",
},
"asset.criticality": Object {
"type": "keyword",
},
"entity.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"entity.source": Object {
"type": "keyword",
},
"service.address": Object {
"type": "keyword",
},
"service.environment": Object {
"type": "keyword",
},
"service.ephemeral_id": Object {
"type": "keyword",
},
"service.id": Object {
"type": "keyword",
},
"service.name": Object {
"fields": Object {
"text": Object {
"type": "match_only_text",
},
},
"type": "keyword",
},
"service.node.name": Object {
"type": "keyword",
},
"service.node.roles": Object {
"type": "keyword",
},
"service.risk.calculated_level": Object {
"type": "keyword",
},
"service.risk.calculated_score": Object {
"type": "float",
},
"service.risk.calculated_score_norm": Object {
"type": "float",
},
"service.state": Object {
"type": "keyword",
},
"service.type": Object {
"type": "keyword",
},
"service.version": Object {
"type": "keyword",
},
},
}
`);
});
it('fieldRetentionDefinition', () => {
expect(unitedDefinition.fieldRetentionDefinition).toMatchInlineSnapshot(`
Object {
"entityType": "service",
"fields": Array [
Object {
"field": "service.address",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.environment",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.ephemeral_id",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.id",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.node.name",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.node.roles",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.state",
"operation": "prefer_newest_value",
},
Object {
"field": "service.type",
"maxLength": 10,
"operation": "collect_values",
},
Object {
"field": "service.version",
"operation": "prefer_newest_value",
},
Object {
"field": "entity.source",
"operation": "prefer_oldest_value",
},
Object {
"field": "asset.criticality",
"operation": "prefer_newest_value",
},
Object {
"field": "service.risk.calculated_level",
"operation": "prefer_newest_value",
},
Object {
"field": "service.risk.calculated_score",
"operation": "prefer_newest_value",
},
Object {
"field": "service.risk.calculated_score_norm",
"operation": "prefer_newest_value",
},
],
"matchField": "service.name",
}
`);
});
it('entityManagerDefinition', () => {
expect(unitedDefinition.entityManagerDefinition).toMatchInlineSnapshot(`
Object {
"displayNameTemplate": "{{service.name}}",
"id": "security_service_test",
"identityFields": Array [
Object {
"field": "service.name",
"optional": false,
},
],
"indexPatterns": Array [
"test*",
],
"latest": Object {
"lookbackPeriod": "24h",
"settings": Object {
"frequency": "1m",
"syncDelay": "1m",
},
"timestampField": "@timestamp",
},
"managed": true,
"metadata": Array [
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.address",
"source": "service.address",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.environment",
"source": "service.environment",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.ephemeral_id",
"source": "service.ephemeral_id",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.id",
"source": "service.id",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.node.name",
"source": "service.node.name",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.node.roles",
"source": "service.node.roles",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.state",
"source": "service.state",
},
Object {
"aggregation": Object {
"limit": 10,
"type": "terms",
},
"destination": "service.type",
"source": "service.type",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.version",
"source": "service.version",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "asc",
},
"type": "top_value",
},
"destination": "entity.source",
"source": "_index",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "asset.criticality",
"source": "asset.criticality",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_level",
"source": "service.risk.calculated_level",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_score",
"source": "service.risk.calculated_score",
},
Object {
"aggregation": Object {
"sort": Object {
"@timestamp": "desc",
},
"type": "top_value",
},
"destination": "service.risk.calculated_score_norm",
"source": "service.risk.calculated_score_norm",
},
],
"name": "Security 'service' Entity Store Definition",
"type": "service",
"version": "1.0.0",
}
`);
});
});
});

View file

@ -12,12 +12,14 @@ import {
getCommonUnitedFieldDefinitions,
USER_DEFINITION_VERSION,
HOST_DEFINITION_VERSION,
getServiceUnitedDefinition,
} from './entity_types';
import type { UnitedDefinitionBuilder } from './types';
import { UnitedEntityDefinition } from './united_entity_definition';
const unitedDefinitionBuilders: Record<EntityType, UnitedDefinitionBuilder> = {
host: getHostUnitedDefinition,
user: getUserUnitedDefinition,
service: getServiceUnitedDefinition,
};
interface Options {
@ -57,8 +59,14 @@ export const getUnitedEntityDefinition = memoize(
}
);
const versionByEntityType: Record<EntityType, string> = {
host: HOST_DEFINITION_VERSION,
user: USER_DEFINITION_VERSION,
service: USER_DEFINITION_VERSION,
};
export const getUnitedEntityDefinitionVersion = (entityType: EntityType): string =>
entityType === 'host' ? HOST_DEFINITION_VERSION : USER_DEFINITION_VERSION;
versionByEntityType[entityType];
export const getAvailableEntityTypes = (): EntityType[] =>
Object.keys(unitedDefinitionBuilders) as EntityType[];

View file

@ -14,13 +14,20 @@ import type { DataViewsService, DataView } from '@kbn/data-views-plugin/common';
import type { AppClient } from '../../../../types';
import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine';
import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality';
import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
import {
EntityTypeEnum,
type EntityType,
} from '../../../../../common/api/entity_analytics/entity_store/common.gen';
import { entityEngineDescriptorTypeName } from '../saved_object';
export const getIdentityFieldForEntityType = (entityType: EntityType) => {
if (entityType === 'host') return 'host.name';
const identityFieldMap: Record<EntityType, string> = {
[EntityTypeEnum.host]: 'host.name',
[EntityTypeEnum.user]: 'user.name',
[EntityTypeEnum.service]: 'service.name',
};
return 'user.name';
export const getIdentityFieldForEntityType = (entityType: EntityType) => {
return identityFieldMap[entityType];
};
export const buildIndexPatterns = async (

View file

@ -237,6 +237,7 @@ export class RequestContextFactory implements IRequestContextFactory {
auditLogger: getAuditLogger(),
kibanaVersion: options.kibanaVersion,
config: config.entityAnalytics.entityStore,
experimentalFeatures: config.experimentalFeatures,
telemetry: core.analytics,
});
}),