[8.x] [SecuritySolution] Service Entity Store (#202344) (#203521)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[SecuritySolution] Service Entity Store
(#202344)](https://github.com/elastic/kibana/pull/202344)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Pablo
Machado","email":"pablo.nevesmachado@elastic.co"},"sourceCommit":{"committedDate":"2024-12-09T17:12:51Z","message":"[SecuritySolution]
Service Entity Store (#202344)\n\n## Summary\n\n\n### Service
Definition:\n\nhttps://github.com/elastic/kibana/pull/202344/files#diff-42c7dd345e0500c97f85824904a70a11162827ea8f8df6982082a9047ca04ff1\n\n\n###
Acceptance Criteria\n- [x] Upon installation of the entity store, the
Service entity\ndefinition should be created by default\n- [x] The
Service definition will be installed in the exact same way as\nthe User
and Host definitions\n- [x] The unique identifier for service entities
will be `service.name`\n- [x] The fields captured for service entities
should match the field\nmapping spreadsheet (see Implementation Notes
below)\n\n\n### Stored Entity\n```json\n{\n \"@timestamp\":
\"2024-12-02T10:43:13.856Z\",\n \"event\": {\n \"ingested\":
\"2024-12-02T10:51:28.987428Z\"\n },\n \"entity\": {\n \"name\":
\"test123 name\",\n \"id\": \"test123 name\",\n \"source\":
\"logs-blito\",\n \"type\": \"service\"\n },\n \"service\": {\n
\"node\": {\n \"roles\": [\n \"test123 node roles\"\n ],\n \"name\": [\n
\"test123 node name\"\n ]\n },\n \"environment\": [\n \"test123
environment\"\n ],\n \"address\": [\n \"test123 address\"\n ],\n
\"name\": \"test123 name\",\n \"id\": [\n \"test123 id\"\n ],\n
\"state\": [\n \"test123 state\"\n ],\n \"ephemeral_id\": [\n \"test123
ephemeral_id\"\n ],\n \"type\": [\n \"test123 type\"\n ],\n \"version\":
[\n \"test123 version\"\n ]\n }\n}\n```\n\n### How to test it?\n\n*
Start Kibana\n<details>\n <summary>Create mappings</summary>\n
\n```\nPUT /logs-test\n{\n \"mappings\": {\n \"properties\": { \n
\"service.name\": {\n \"type\": \"keyword\"\n },\n \"service.address\":
{\n \"type\": \"keyword\"\n },\n \"service.environment\": {\n \"type\":
\"keyword\"\n },\n \"service.ephemeral_id\": {\n \"type\": \"keyword\"\n
},\n \"service.id\": {\n \"type\": \"keyword\"\n },\n
\"service.node.name\": {\n \"type\": \"keyword\"\n },\n
\"service.node.roles\": {\n \"type\": \"keyword\"\n },\n
\"service.state\": {\n \"type\": \"keyword\"\n },\n \"service.type\":
{\n \"type\": \"keyword\"\n },\n \"service.version\": {\n \"type\":
\"keyword\"\n },\n \"@timestamp\": {\n \"type\": \"date\"\n }\n }\n
}\n}\n```` \n</details>\n\n\n<details>\n <summary>Create
document</summary>\n \n```\nPUT /logs-test\nPOST logs-test/_doc\n{\n
\"service\": {\n \"name\": \"test123 name\",\n \"address\": \"test123
address\",\n \"environment\": \"test123 environment\",\n
\"ephemeral_id\": \"test123 ephemeral_id\",\n \"id\": \"test123 id\",\n
\"node.roles\": \"test123 node roles\",\n \"node.name\": \"test123 node
name\", \n \"state\": \"test123 state\",\n \"type\": \"test123 type\",\n
\"version\": \"test123 version\"\n },\n \"@timestamp\":
\"2024-12-02T10:43:13.856Z\"\n}\n\n```` \n</details>\n\n* Init the
entity store\n* Wait...\n* Query the service index
`GET\n.entities.v1.latest.security_service_default/_search`\n\n\n###
Open Questions\n* Can we merge this PR without first updating all other
features that\nwill use service entities?\n* If we merge it, the service
engine will be installed together with\nother entities, but it won't
provide any functionality\n* Do we need an experimental
flag?\n\n---------\n\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"fdedae07b854280b37f142b652892f1b5ee44018","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["v9.0.0","Team:
SecuritySolution","release_note:feature","Theme:
entity_analytics","Feature:Entity Analytics","Team:Entity
Analytics","backport:version","v8.18.0"],"number":202344,"url":"https://github.com/elastic/kibana/pull/202344","mergeCommit":{"message":"[SecuritySolution]
Service Entity Store (#202344)\n\n## Summary\n\n\n### Service
Definition:\n\nhttps://github.com/elastic/kibana/pull/202344/files#diff-42c7dd345e0500c97f85824904a70a11162827ea8f8df6982082a9047ca04ff1\n\n\n###
Acceptance Criteria\n- [x] Upon installation of the entity store, the
Service entity\ndefinition should be created by default\n- [x] The
Service definition will be installed in the exact same way as\nthe User
and Host definitions\n- [x] The unique identifier for service entities
will be `service.name`\n- [x] The fields captured for service entities
should match the field\nmapping spreadsheet (see Implementation Notes
below)\n\n\n### Stored Entity\n```json\n{\n \"@timestamp\":
\"2024-12-02T10:43:13.856Z\",\n \"event\": {\n \"ingested\":
\"2024-12-02T10:51:28.987428Z\"\n },\n \"entity\": {\n \"name\":
\"test123 name\",\n \"id\": \"test123 name\",\n \"source\":
\"logs-blito\",\n \"type\": \"service\"\n },\n \"service\": {\n
\"node\": {\n \"roles\": [\n \"test123 node roles\"\n ],\n \"name\": [\n
\"test123 node name\"\n ]\n },\n \"environment\": [\n \"test123
environment\"\n ],\n \"address\": [\n \"test123 address\"\n ],\n
\"name\": \"test123 name\",\n \"id\": [\n \"test123 id\"\n ],\n
\"state\": [\n \"test123 state\"\n ],\n \"ephemeral_id\": [\n \"test123
ephemeral_id\"\n ],\n \"type\": [\n \"test123 type\"\n ],\n \"version\":
[\n \"test123 version\"\n ]\n }\n}\n```\n\n### How to test it?\n\n*
Start Kibana\n<details>\n <summary>Create mappings</summary>\n
\n```\nPUT /logs-test\n{\n \"mappings\": {\n \"properties\": { \n
\"service.name\": {\n \"type\": \"keyword\"\n },\n \"service.address\":
{\n \"type\": \"keyword\"\n },\n \"service.environment\": {\n \"type\":
\"keyword\"\n },\n \"service.ephemeral_id\": {\n \"type\": \"keyword\"\n
},\n \"service.id\": {\n \"type\": \"keyword\"\n },\n
\"service.node.name\": {\n \"type\": \"keyword\"\n },\n
\"service.node.roles\": {\n \"type\": \"keyword\"\n },\n
\"service.state\": {\n \"type\": \"keyword\"\n },\n \"service.type\":
{\n \"type\": \"keyword\"\n },\n \"service.version\": {\n \"type\":
\"keyword\"\n },\n \"@timestamp\": {\n \"type\": \"date\"\n }\n }\n
}\n}\n```` \n</details>\n\n\n<details>\n <summary>Create
document</summary>\n \n```\nPUT /logs-test\nPOST logs-test/_doc\n{\n
\"service\": {\n \"name\": \"test123 name\",\n \"address\": \"test123
address\",\n \"environment\": \"test123 environment\",\n
\"ephemeral_id\": \"test123 ephemeral_id\",\n \"id\": \"test123 id\",\n
\"node.roles\": \"test123 node roles\",\n \"node.name\": \"test123 node
name\", \n \"state\": \"test123 state\",\n \"type\": \"test123 type\",\n
\"version\": \"test123 version\"\n },\n \"@timestamp\":
\"2024-12-02T10:43:13.856Z\"\n}\n\n```` \n</details>\n\n* Init the
entity store\n* Wait...\n* Query the service index
`GET\n.entities.v1.latest.security_service_default/_search`\n\n\n###
Open Questions\n* Can we merge this PR without first updating all other
features that\nwill use service entities?\n* If we merge it, the service
engine will be installed together with\nother entities, but it won't
provide any functionality\n* Do we need an experimental
flag?\n\n---------\n\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"fdedae07b854280b37f142b652892f1b5ee44018"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/202344","number":202344,"mergeCommit":{"message":"[SecuritySolution]
Service Entity Store (#202344)\n\n## Summary\n\n\n### Service
Definition:\n\nhttps://github.com/elastic/kibana/pull/202344/files#diff-42c7dd345e0500c97f85824904a70a11162827ea8f8df6982082a9047ca04ff1\n\n\n###
Acceptance Criteria\n- [x] Upon installation of the entity store, the
Service entity\ndefinition should be created by default\n- [x] The
Service definition will be installed in the exact same way as\nthe User
and Host definitions\n- [x] The unique identifier for service entities
will be `service.name`\n- [x] The fields captured for service entities
should match the field\nmapping spreadsheet (see Implementation Notes
below)\n\n\n### Stored Entity\n```json\n{\n \"@timestamp\":
\"2024-12-02T10:43:13.856Z\",\n \"event\": {\n \"ingested\":
\"2024-12-02T10:51:28.987428Z\"\n },\n \"entity\": {\n \"name\":
\"test123 name\",\n \"id\": \"test123 name\",\n \"source\":
\"logs-blito\",\n \"type\": \"service\"\n },\n \"service\": {\n
\"node\": {\n \"roles\": [\n \"test123 node roles\"\n ],\n \"name\": [\n
\"test123 node name\"\n ]\n },\n \"environment\": [\n \"test123
environment\"\n ],\n \"address\": [\n \"test123 address\"\n ],\n
\"name\": \"test123 name\",\n \"id\": [\n \"test123 id\"\n ],\n
\"state\": [\n \"test123 state\"\n ],\n \"ephemeral_id\": [\n \"test123
ephemeral_id\"\n ],\n \"type\": [\n \"test123 type\"\n ],\n \"version\":
[\n \"test123 version\"\n ]\n }\n}\n```\n\n### How to test it?\n\n*
Start Kibana\n<details>\n <summary>Create mappings</summary>\n
\n```\nPUT /logs-test\n{\n \"mappings\": {\n \"properties\": { \n
\"service.name\": {\n \"type\": \"keyword\"\n },\n \"service.address\":
{\n \"type\": \"keyword\"\n },\n \"service.environment\": {\n \"type\":
\"keyword\"\n },\n \"service.ephemeral_id\": {\n \"type\": \"keyword\"\n
},\n \"service.id\": {\n \"type\": \"keyword\"\n },\n
\"service.node.name\": {\n \"type\": \"keyword\"\n },\n
\"service.node.roles\": {\n \"type\": \"keyword\"\n },\n
\"service.state\": {\n \"type\": \"keyword\"\n },\n \"service.type\":
{\n \"type\": \"keyword\"\n },\n \"service.version\": {\n \"type\":
\"keyword\"\n },\n \"@timestamp\": {\n \"type\": \"date\"\n }\n }\n
}\n}\n```` \n</details>\n\n\n<details>\n <summary>Create
document</summary>\n \n```\nPUT /logs-test\nPOST logs-test/_doc\n{\n
\"service\": {\n \"name\": \"test123 name\",\n \"address\": \"test123
address\",\n \"environment\": \"test123 environment\",\n
\"ephemeral_id\": \"test123 ephemeral_id\",\n \"id\": \"test123 id\",\n
\"node.roles\": \"test123 node roles\",\n \"node.name\": \"test123 node
name\", \n \"state\": \"test123 state\",\n \"type\": \"test123 type\",\n
\"version\": \"test123 version\"\n },\n \"@timestamp\":
\"2024-12-02T10:43:13.856Z\"\n}\n\n```` \n</details>\n\n* Init the
entity store\n* Wait...\n* Query the service index
`GET\n.entities.v1.latest.security_service_default/_search`\n\n\n###
Open Questions\n* Can we merge this PR without first updating all other
features that\nwill use service entities?\n* If we merge it, the service
engine will be installed together with\nother entities, but it won't
provide any functionality\n* Do we need an experimental
flag?\n\n---------\n\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"fdedae07b854280b37f142b652892f1b5ee44018"}},{"branch":"8.x","label":"v8.18.0","labelRegex":"^v8.18.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

oas_docs/output/kibana.yaml

@@ -41033,6 +41033,7 @@ components:
       enum:
         - user
         - host
+        - service
       type: string
     Security_Entity_Analytics_API_HostEntity:
       type: object

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts

@@ -17,7 +17,7 @@
 import { z } from '@kbn/zod';
 
 export type EntityType = z.infer<typeof EntityType>;
-export const EntityType = z.enum(['user', 'host']);
+export const EntityType = z.enum(['user', 'host', 'service']);
 export type EntityTypeEnum = typeof EntityType.enum;
 export const EntityTypeEnum = EntityType.enum;
 

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml

@@ -11,6 +11,7 @@ components:
       enum:
         - user
         - host
+        - service
 
     EngineDescriptor:
       type: object

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -243,6 +243,11 @@ export const allowedExperimentalValues = Object.freeze({
    */
   entityStoreDisabled: false,
 
+  /**
+   * Enables the Service Entity Store. The Entity Store feature will install the service engine by default.
+   */
+  serviceEntityStoreEnabled: true,
+
   /**
    * Enables the siem migrations feature
    */

x-pack/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -972,6 +972,7 @@ components:
       enum:
         - user
         - host
+        - service
       type: string
     HostEntity:
       type: object

x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -972,6 +972,7 @@ components:
       enum:
         - user
         - host
+        - service
       type: string
     HostEntity:
       type: object

x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/hooks/use_entity_store.ts

@@ -12,10 +12,11 @@ import type { IHttpFetchError } from '@kbn/core-http-browser';
 import type { GetEntityStoreStatusResponse } from '../../../../../common/api/entity_analytics/entity_store/status.gen';
 import type { InitEntityStoreResponse } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
 import { useKibana } from '../../../../common/lib/kibana/kibana_react';
-import type {
+import type { EntityType } from '../../../../../common/api/entity_analytics';
-  DeleteEntityEngineResponse,
+import {
-  InitEntityEngineResponse,
+  type DeleteEntityEngineResponse,
-  StopEntityEngineResponse,
+  type InitEntityEngineResponse,
+  type StopEntityEngineResponse,
 } from '../../../../../common/api/entity_analytics';
 import { useEntityStoreRoutes } from '../../../api/entity_store';
 import { EntityEventTypes } from '../../../../common/lib/telemetry';
@@ -68,13 +69,16 @@ export const useEnableEntityStoreMutation = (options?: UseMutationOptions<{}>) =
 };
 
 export const INIT_ENTITY_ENGINE_STATUS_KEY = ['POST', 'INIT_ENTITY_ENGINE'];
+/**
+ * @deprecated
+ * It will be deleted on a follow-up PR
+ */
 export const useInitEntityEngineMutation = (options?: UseMutationOptions<{}>) => {
   const queryClient = useQueryClient();
 
   const { initEntityEngine } = useEntityStoreRoutes();
   return useMutation<InitEntityEngineResponse[]>(
     () => Promise.all([initEntityEngine('user'), initEntityEngine('host')]),
-
     {
       mutationKey: INIT_ENTITY_ENGINE_STATUS_KEY,
       onSuccess: () => queryClient.refetchQueries({ queryKey: ENTITY_STORE_STATUS }),
@@ -84,7 +88,7 @@ export const useInitEntityEngineMutation = (options?: UseMutationOptions<{}>) =>
 };
 
 export const STOP_ENTITY_ENGINE_STATUS_KEY = ['POST', 'STOP_ENTITY_ENGINE'];
-export const useStopEntityEngineMutation = (options?: UseMutationOptions<{}>) => {
+export const useStopEntityEngineMutation = (entityTypes: EntityType[]) => {
   const { telemetry } = useKibana().services;
   const queryClient = useQueryClient();
 
@@ -95,23 +99,28 @@ export const useStopEntityEngineMutation = (options?: UseMutationOptions<{}>) =>
         timestamp: new Date().toISOString(),
         action: 'stop',
       });
-      return Promise.all([stopEntityEngine('user'), stopEntityEngine('host')]);
+      return Promise.all(entityTypes.map((entityType) => stopEntityEngine(entityType)));
     },
     {
       mutationKey: STOP_ENTITY_ENGINE_STATUS_KEY,
       onSuccess: () => queryClient.refetchQueries({ queryKey: ENTITY_STORE_STATUS }),
-      ...options,
     }
   );
 };
 
 export const DELETE_ENTITY_ENGINE_STATUS_KEY = ['POST', 'STOP_ENTITY_ENGINE'];
-export const useDeleteEntityEngineMutation = ({ onSuccess }: { onSuccess?: () => void }) => {
+export const useDeleteEntityEngineMutation = ({
+  onSuccess,
+  entityTypes,
+}: {
+  onSuccess?: () => void;
+  entityTypes: EntityType[];
+}) => {
   const queryClient = useQueryClient();
   const { deleteEntityEngine } = useEntityStoreRoutes();
 
   return useMutation<DeleteEntityEngineResponse[]>(
-    () => Promise.all([deleteEntityEngine('user', true), deleteEntityEngine('host', true)]),
+    () => Promise.all(entityTypes.map((entityType) => deleteEntityEngine(entityType, true))),
     {
       mutationKey: DELETE_ENTITY_ENGINE_STATUS_KEY,
       onSuccess: () => {

x-pack/plugins/security_solution/public/entity_analytics/pages/entity_store_management_page.tsx

@@ -33,7 +33,7 @@ import React, { useCallback, useEffect, useState } from 'react';
 import { FormattedMessage } from '@kbn/i18n-react';
 
 import type { SecurityAppError } from '@kbn/securitysolution-t-grid';
-import type { StoreStatus } from '../../../common/api/entity_analytics';
+import { EntityType, EntityTypeEnum, type StoreStatus } from '../../../common/api/entity_analytics';
 import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features';
 import { ASSET_CRITICALITY_INDEX_PATTERN } from '../../../common/entity_analytics/asset_criticality';
 import { useKibana } from '../../common/lib/kibana';
@@ -73,13 +73,20 @@ export const EntityStoreManagementPage = () => {
   const hasAssetCriticalityWritePermissions = assetCriticalityPrivileges?.has_write_permissions;
   const [selectedTabId, setSelectedTabId] = useState(TabId.Import);
   const entityStoreStatus = useEntityStoreStatus({});
+  const isServiceEntityStoreEnabled = useIsExperimentalFeatureEnabled('serviceEntityStoreEnabled');
+  const allEntityTypes = Object.values(EntityType.Values);
+
+  const entityTypes = isServiceEntityStoreEnabled
+    ? allEntityTypes
+    : allEntityTypes.filter((value) => value !== EntityTypeEnum.service);
 
   const enableStoreMutation = useEnableEntityStoreMutation();
-  const stopEntityEngineMutation = useStopEntityEngineMutation();
+  const stopEntityEngineMutation = useStopEntityEngineMutation(entityTypes);
   const deleteEntityEngineMutation = useDeleteEntityEngineMutation({
     onSuccess: () => {
       closeClearModal();
     },
+    entityTypes,
   });
 
   const [isClearModalVisible, setIsClearModalVisible] = useState(false);

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts

@@ -16,6 +16,7 @@ import type { EntityType } from '../../../../common/api/entity_analytics/entity_
 import type { DataViewsService } from '@kbn/data-views-plugin/common';
 import type { AppClient } from '../../..';
 import type { EntityStoreConfig } from './types';
+import { mockGlobalState } from '../../../../public/common/mock';
 
 describe('EntityStoreDataClient', () => {
   const mockSavedObjectClient = savedObjectsClientMock.create();
@@ -31,6 +32,7 @@ describe('EntityStoreDataClient', () => {
     dataViewsService: {} as DataViewsService,
     appClient: {} as AppClient,
     config: {} as EntityStoreConfig,
+    experimentalFeatures: mockGlobalState.app.enableExperimental,
   });
 
   const defaultSearchParams = {

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -23,6 +23,7 @@ import moment from 'moment';
 import type { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types';
 import type { EntityDefinition } from '@kbn/entities-schema';
 import type { estypes } from '@elastic/elasticsearch';
+import type { ExperimentalFeatures } from '../../../../common';
 import type {
   GetEntityStoreStatusRequestQuery,
   GetEntityStoreStatusResponse,
@@ -32,7 +33,10 @@ import type {
   InitEntityStoreResponse,
 } from '../../../../common/api/entity_analytics/entity_store/enable.gen';
 import type { AppClient } from '../../..';
-import { EngineComponentResourceEnum, EntityType } from '../../../../common/api/entity_analytics';
+import {
+  EngineComponentResourceEnum,
+  EntityTypeEnum,
+} from '../../../../common/api/entity_analytics';
 import type {
   Entity,
   EngineDataviewUpdateResult,
@@ -42,6 +46,7 @@ import type {
   ListEntityEnginesResponse,
   EngineComponentStatus,
   EngineComponentResource,
+  EntityType,
 } from '../../../../common/api/entity_analytics';
 import { EngineDescriptorClient } from './saved_object/engine_descriptor';
 import { ENGINE_STATUS, ENTITY_STORE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
@@ -108,6 +113,7 @@ interface EntityStoreClientOpts {
   kibanaVersion: string;
   dataViewsService: DataViewsService;
   appClient: AppClient;
+  experimentalFeatures: ExperimentalFeatures;
   telemetry?: AnalyticsServiceSetup;
   config: EntityStoreConfig;
 }
@@ -205,7 +211,13 @@ export class EntityStoreDataClient {
     // Immediately defer the initialization to the next tick. This way we don't block on the init preflight checks
     const run = <T>(fn: () => Promise<T>) =>
       new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
-    const promises = Object.values(EntityType.Values).map((entity) =>
+
+    const { experimentalFeatures } = this.options;
+    const enginesTypes = experimentalFeatures.serviceEntityStoreEnabled
+      ? [EntityTypeEnum.host, EntityTypeEnum.user, EntityTypeEnum.service]
+      : [EntityTypeEnum.host, EntityTypeEnum.user];
+
+    const promises = enginesTypes.map((entity) =>
       run(() =>
         this.init(entity, { indexPattern, filter, fieldHistoryLength }, { pipelineDebugMode })
       )

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/index.ts

@@ -7,4 +7,5 @@
 
 export * from './host';
 export * from './user';
+export * from './service';
 export { getCommonUnitedFieldDefinitions } from './common';

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/service.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { collectValuesWithLength, newestValue } from '../definition_utils';
+import type { UnitedDefinitionBuilder } from '../types';
+
+export const SERVICE_DEFINITION_VERSION = '1.0.0';
+export const getServiceUnitedDefinition: UnitedDefinitionBuilder = (fieldHistoryLength: number) => {
+  const collect = collectValuesWithLength(fieldHistoryLength);
+  return {
+    entityType: 'service',
+    version: SERVICE_DEFINITION_VERSION,
+    fields: [
+      collect({ field: 'service.address' }),
+      collect({ field: 'service.environment' }),
+      collect({ field: 'service.ephemeral_id' }),
+      collect({ field: 'service.id' }),
+      collect({ field: 'service.node.name' }),
+      collect({ field: 'service.node.roles' }),
+      newestValue({ field: 'service.state' }),
+      collect({ field: 'service.type' }),
+      newestValue({ field: 'service.version' }),
+    ],
+  };
+};

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts

@@ -587,4 +587,315 @@ describe('getUnitedEntityDefinition', () => {
       `);
     });
   });
+
+  describe('service', () => {
+    const unitedDefinition = getUnitedEntityDefinition({
+      entityType: 'service',
+      namespace: 'test',
+      fieldHistoryLength: 10,
+      indexPatterns,
+      syncDelay: '1m',
+      frequency: '1m',
+    });
+
+    it('mapping', () => {
+      expect(unitedDefinition.indexMappings).toMatchInlineSnapshot(`
+        Object {
+          "properties": Object {
+            "@timestamp": Object {
+              "type": "date",
+            },
+            "asset.criticality": Object {
+              "type": "keyword",
+            },
+            "entity.name": Object {
+              "fields": Object {
+                "text": Object {
+                  "type": "match_only_text",
+                },
+              },
+              "type": "keyword",
+            },
+            "entity.source": Object {
+              "type": "keyword",
+            },
+            "service.address": Object {
+              "type": "keyword",
+            },
+            "service.environment": Object {
+              "type": "keyword",
+            },
+            "service.ephemeral_id": Object {
+              "type": "keyword",
+            },
+            "service.id": Object {
+              "type": "keyword",
+            },
+            "service.name": Object {
+              "fields": Object {
+                "text": Object {
+                  "type": "match_only_text",
+                },
+              },
+              "type": "keyword",
+            },
+            "service.node.name": Object {
+              "type": "keyword",
+            },
+            "service.node.roles": Object {
+              "type": "keyword",
+            },
+            "service.risk.calculated_level": Object {
+              "type": "keyword",
+            },
+            "service.risk.calculated_score": Object {
+              "type": "float",
+            },
+            "service.risk.calculated_score_norm": Object {
+              "type": "float",
+            },
+            "service.state": Object {
+              "type": "keyword",
+            },
+            "service.type": Object {
+              "type": "keyword",
+            },
+            "service.version": Object {
+              "type": "keyword",
+            },
+          },
+        }
+      `);
+    });
+    it('fieldRetentionDefinition', () => {
+      expect(unitedDefinition.fieldRetentionDefinition).toMatchInlineSnapshot(`
+        Object {
+          "entityType": "service",
+          "fields": Array [
+            Object {
+              "field": "service.address",
+              "maxLength": 10,
+              "operation": "collect_values",
+            },
+            Object {
+              "field": "service.environment",
+              "maxLength": 10,
+              "operation": "collect_values",
+            },
+            Object {
+              "field": "service.ephemeral_id",
+              "maxLength": 10,
+              "operation": "collect_values",
+            },
+            Object {
+              "field": "service.id",
+              "maxLength": 10,
+              "operation": "collect_values",
+            },
+            Object {
+              "field": "service.node.name",
+              "maxLength": 10,
+              "operation": "collect_values",
+            },
+            Object {
+              "field": "service.node.roles",
+              "maxLength": 10,
+              "operation": "collect_values",
+            },
+            Object {
+              "field": "service.state",
+              "operation": "prefer_newest_value",
+            },
+            Object {
+              "field": "service.type",
+              "maxLength": 10,
+              "operation": "collect_values",
+            },
+            Object {
+              "field": "service.version",
+              "operation": "prefer_newest_value",
+            },
+            Object {
+              "field": "entity.source",
+              "operation": "prefer_oldest_value",
+            },
+            Object {
+              "field": "asset.criticality",
+              "operation": "prefer_newest_value",
+            },
+            Object {
+              "field": "service.risk.calculated_level",
+              "operation": "prefer_newest_value",
+            },
+            Object {
+              "field": "service.risk.calculated_score",
+              "operation": "prefer_newest_value",
+            },
+            Object {
+              "field": "service.risk.calculated_score_norm",
+              "operation": "prefer_newest_value",
+            },
+          ],
+          "matchField": "service.name",
+        }
+      `);
+    });
+    it('entityManagerDefinition', () => {
+      expect(unitedDefinition.entityManagerDefinition).toMatchInlineSnapshot(`
+        Object {
+          "displayNameTemplate": "{{service.name}}",
+          "id": "security_service_test",
+          "identityFields": Array [
+            Object {
+              "field": "service.name",
+              "optional": false,
+            },
+          ],
+          "indexPatterns": Array [
+            "test*",
+          ],
+          "latest": Object {
+            "lookbackPeriod": "24h",
+            "settings": Object {
+              "frequency": "1m",
+              "syncDelay": "1m",
+            },
+            "timestampField": "@timestamp",
+          },
+          "managed": true,
+          "metadata": Array [
+            Object {
+              "aggregation": Object {
+                "limit": 10,
+                "type": "terms",
+              },
+              "destination": "service.address",
+              "source": "service.address",
+            },
+            Object {
+              "aggregation": Object {
+                "limit": 10,
+                "type": "terms",
+              },
+              "destination": "service.environment",
+              "source": "service.environment",
+            },
+            Object {
+              "aggregation": Object {
+                "limit": 10,
+                "type": "terms",
+              },
+              "destination": "service.ephemeral_id",
+              "source": "service.ephemeral_id",
+            },
+            Object {
+              "aggregation": Object {
+                "limit": 10,
+                "type": "terms",
+              },
+              "destination": "service.id",
+              "source": "service.id",
+            },
+            Object {
+              "aggregation": Object {
+                "limit": 10,
+                "type": "terms",
+              },
+              "destination": "service.node.name",
+              "source": "service.node.name",
+            },
+            Object {
+              "aggregation": Object {
+                "limit": 10,
+                "type": "terms",
+              },
+              "destination": "service.node.roles",
+              "source": "service.node.roles",
+            },
+            Object {
+              "aggregation": Object {
+                "sort": Object {
+                  "@timestamp": "desc",
+                },
+                "type": "top_value",
+              },
+              "destination": "service.state",
+              "source": "service.state",
+            },
+            Object {
+              "aggregation": Object {
+                "limit": 10,
+                "type": "terms",
+              },
+              "destination": "service.type",
+              "source": "service.type",
+            },
+            Object {
+              "aggregation": Object {
+                "sort": Object {
+                  "@timestamp": "desc",
+                },
+                "type": "top_value",
+              },
+              "destination": "service.version",
+              "source": "service.version",
+            },
+            Object {
+              "aggregation": Object {
+                "sort": Object {
+                  "@timestamp": "asc",
+                },
+                "type": "top_value",
+              },
+              "destination": "entity.source",
+              "source": "_index",
+            },
+            Object {
+              "aggregation": Object {
+                "sort": Object {
+                  "@timestamp": "desc",
+                },
+                "type": "top_value",
+              },
+              "destination": "asset.criticality",
+              "source": "asset.criticality",
+            },
+            Object {
+              "aggregation": Object {
+                "sort": Object {
+                  "@timestamp": "desc",
+                },
+                "type": "top_value",
+              },
+              "destination": "service.risk.calculated_level",
+              "source": "service.risk.calculated_level",
+            },
+            Object {
+              "aggregation": Object {
+                "sort": Object {
+                  "@timestamp": "desc",
+                },
+                "type": "top_value",
+              },
+              "destination": "service.risk.calculated_score",
+              "source": "service.risk.calculated_score",
+            },
+            Object {
+              "aggregation": Object {
+                "sort": Object {
+                  "@timestamp": "desc",
+                },
+                "type": "top_value",
+              },
+              "destination": "service.risk.calculated_score_norm",
+              "source": "service.risk.calculated_score_norm",
+            },
+          ],
+          "name": "Security 'service' Entity Store Definition",
+          "type": "service",
+          "version": "1.0.0",
+        }
+      `);
+    });
+  });
 });

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts

@@ -12,12 +12,14 @@ import {
   getCommonUnitedFieldDefinitions,
   USER_DEFINITION_VERSION,
   HOST_DEFINITION_VERSION,
+  getServiceUnitedDefinition,
 } from './entity_types';
 import type { UnitedDefinitionBuilder } from './types';
 import { UnitedEntityDefinition } from './united_entity_definition';
 const unitedDefinitionBuilders: Record<EntityType, UnitedDefinitionBuilder> = {
   host: getHostUnitedDefinition,
   user: getUserUnitedDefinition,
+  service: getServiceUnitedDefinition,
 };
 
 interface Options {
@@ -57,8 +59,14 @@ export const getUnitedEntityDefinition = memoize(
   }
 );
 
+const versionByEntityType: Record<EntityType, string> = {
+  host: HOST_DEFINITION_VERSION,
+  user: USER_DEFINITION_VERSION,
+  service: USER_DEFINITION_VERSION,
+};
+
 export const getUnitedEntityDefinitionVersion = (entityType: EntityType): string =>
-  entityType === 'host' ? HOST_DEFINITION_VERSION : USER_DEFINITION_VERSION;
+  versionByEntityType[entityType];
 
 export const getAvailableEntityTypes = (): EntityType[] =>
   Object.keys(unitedDefinitionBuilders) as EntityType[];

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts

@@ -14,13 +14,20 @@ import type { DataViewsService, DataView } from '@kbn/data-views-plugin/common';
 import type { AppClient } from '../../../../types';
 import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine';
 import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality';
-import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
+import {
+  EntityTypeEnum,
+  type EntityType,
+} from '../../../../../common/api/entity_analytics/entity_store/common.gen';
 import { entityEngineDescriptorTypeName } from '../saved_object';
 
-export const getIdentityFieldForEntityType = (entityType: EntityType) => {
+const identityFieldMap: Record<EntityType, string> = {
-  if (entityType === 'host') return 'host.name';
+  [EntityTypeEnum.host]: 'host.name',
+  [EntityTypeEnum.user]: 'user.name',
+  [EntityTypeEnum.service]: 'service.name',
+};
 
-  return 'user.name';
+export const getIdentityFieldForEntityType = (entityType: EntityType) => {
+  return identityFieldMap[entityType];
 };
 
 export const buildIndexPatterns = async (

x-pack/plugins/security_solution/server/request_context_factory.ts

@@ -236,6 +236,7 @@ export class RequestContextFactory implements IRequestContextFactory {
           taskManager: startPlugins.taskManager,
           auditLogger: getAuditLogger(),
           kibanaVersion: options.kibanaVersion,
+          experimentalFeatures: config.experimentalFeatures,
           telemetry: core.analytics,
           config: config.entityAnalytics.entityStore,
         });