[EEM] Update the builtin service entity definition (#187021)

## Summary

This change includes updates to the builtin service definition:
- removal of the high cardinality metadata fields until we have a
solution in place
- fetching of the metrics from the preaggregated apm metrics
- fixed metrics aggregations
- increased history transform frequency and delay to cover for delayed
ingestion

---------

Co-authored-by: Milton Hultgren <milton.hultgren@elastic.co>
Co-authored-by: Søren Louv-Jansen <sorenlouv@gmail.com>

x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts

@@ -35,7 +35,6 @@ export const entityDefinitionSchema = z.object({
     interval: durationSchema.refine((val) => val.asMinutes() >= 1, {
       message: 'The history.interval can not be less than 1m',
     }),
-    lookbackPeriod: z.optional(durationSchema),
     settings: z.optional(
       z.object({
         syncField: z.optional(z.string()),

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/constants.ts

@@ -7,9 +7,9 @@
 
 export const BUILT_IN_ID_PREFIX = 'builtin_';
 export const BUILT_IN_ALLOWED_INDICES = [
-  'apm-*',
   'logs-*',
-  'metrics-*',
   'filebeat*',
+  'metrics-*',
   'metricbeat*',
+  'traces-*',
 ];

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/index.ts

@@ -6,8 +6,8 @@
  */
 
 import { EntityDefinition } from '@kbn/entities-schema';
-import { builtInServicesEntityDefinition } from './services';
+import { builtInServicesFromLogsEntityDefinition } from './services';
 
 export { BUILT_IN_ID_PREFIX } from './constants';
 
-export const builtInDefinitions: EntityDefinition[] = [builtInServicesEntityDefinition];
+export const builtInDefinitions: EntityDefinition[] = [builtInServicesFromLogsEntityDefinition];

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts

@@ -8,63 +8,121 @@
 import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
 import { BUILT_IN_ID_PREFIX } from './constants';
 
-export const builtInServicesEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({
-  id: `${BUILT_IN_ID_PREFIX}services`,
-  version: '0.1.0',
-  name: 'Services from logs',
-  type: 'service',
-  managed: true,
-  indexPatterns: ['logs-*', 'filebeat*'],
-  history: {
-    timestampField: '@timestamp',
-    interval: '1m',
-  },
-  latest: {
-    lookback: '5m',
-  },
-  identityFields: ['service.name', { field: 'service.environment', optional: true }],
-  displayNameTemplate: '{{service.name}}{{#service.environment}}:{{.}}{{/service.environment}}',
-  metadata: [
-    { source: '_index', destination: 'sourceIndex' },
-    'data_stream.type',
-    'service.instance.id',
-    'service.namespace',
-    'service.version',
-    'service.runtime.name',
-    'service.runtime.version',
-    'service.node.name',
-    'service.language.name',
-    'agent.name',
-    'cloud.provider',
-    'cloud.instance.id',
-    'cloud.availability_zone',
-    'cloud.instance.name',
-    'cloud.machine.type',
-    'host.name',
-    'container.id',
-  ],
-  metrics: [
-    {
-      name: 'logRate',
-      equation: 'A / 5',
-      metrics: [
-        {
-          name: 'A',
-          aggregation: 'doc_count',
-          filter: 'log.level: *',
-        },
-      ],
+const serviceTransactionFilter = (additionalFilters: string[] = []) => {
+  const baseFilters = [
+    'processor.event: "metric"',
+    'metricset.name: "service_transaction"',
+    'metricset.interval: "1m"',
+  ];
+
+  return [...baseFilters, ...additionalFilters].join(' AND ');
+};
+
+export const builtInServicesFromLogsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    version: '0.1.0',
+    id: `${BUILT_IN_ID_PREFIX}services_from_ecs_data`,
+    name: 'Services from ECS data',
+    description:
+      'This definition extracts service entities from common data streams by looking for the ECS field service.name',
+    type: 'service',
+    managed: true,
+    filter: '@timestamp >= now-10m',
+    indexPatterns: ['logs-*', 'filebeat*', 'metrics-apm.service_transaction.1m*'],
+    history: {
+      timestampField: '@timestamp',
+      interval: '1m',
+      settings: {
+        frequency: '2m',
+        syncDelay: '2m',
+      },
     },
-    {
-      name: 'errorRate',
-      equation: 'A / 5',
-      metrics: [
-        {
-          name: 'A',
-          aggregation: 'doc_count',
-          filter: 'log.level: "ERROR"',
-        },
-      ],
+    latest: {
+      lookback: '5m',
     },
-  ],
-});
+    identityFields: ['service.name', { field: 'service.environment', optional: true }],
+    displayNameTemplate: '{{service.name}}{{#service.environment}}:{{.}}{{/service.environment}}',
+    metadata: [
+      { source: '_index', destination: 'sourceIndex' },
+      { source: 'agent.name', limit: 100 },
+      'data_stream.type',
+      'service.environment',
+      'service.name',
+      'service.namespace',
+      'service.version',
+      'service.runtime.name',
+      'service.runtime.version',
+      'service.language.name',
+      'cloud.provider',
+      'cloud.availability_zone',
+      'cloud.machine.type',
+    ],
+    metrics: [
+      {
+        name: 'latency',
+        equation: 'A',
+        metrics: [
+          {
+            name: 'A',
+            aggregation: 'avg',
+            filter: serviceTransactionFilter(),
+            field: 'transaction.duration.histogram',
+          },
+        ],
+      },
+      {
+        name: 'throughput',
+        equation: 'A',
+        metrics: [
+          {
+            name: 'A',
+            aggregation: 'doc_count',
+            filter: serviceTransactionFilter(),
+          },
+        ],
+      },
+      {
+        name: 'failedTransactionRate',
+        equation: 'A / B',
+        metrics: [
+          {
+            name: 'A',
+            aggregation: 'doc_count',
+            filter: serviceTransactionFilter(['event.outcome: "failure"']),
+          },
+          {
+            name: 'B',
+            aggregation: 'doc_count',
+            filter: serviceTransactionFilter(['event.outcome: *']),
+          },
+        ],
+      },
+      {
+        name: 'logErrorRate',
+        equation: 'A / B',
+        metrics: [
+          {
+            name: 'A',
+            aggregation: 'doc_count',
+            filter: 'log.level: "error" OR error.log.level: "error"',
+          },
+          {
+            name: 'B',
+            aggregation: 'doc_count',
+            filter: 'log.level: * OR error.log.level: *',
+          },
+        ],
+      },
+      {
+        name: 'logRate',
+        equation: 'A',
+        metrics: [
+          {
+            name: 'A',
+            aggregation: 'doc_count',
+            filter: 'log.level: * OR error.log.level: *',
+          },
+        ],
+      },
+    ],
+  });

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts

@@ -12,7 +12,7 @@ import { EntityDefinition } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { installBuiltInEntityDefinitions } from './install_entity_definition';
-import { builtInServicesEntityDefinition } from './built_in/services';
+import { builtInServicesFromLogsEntityDefinition } from './built_in/services';
 import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
 import {
   generateHistoryIngestPipelineId,
@@ -36,7 +36,7 @@ const assertHasCreatedDefinition = (
 
   expect(esClient.ingest.putPipeline).toBeCalledTimes(2);
   expect(esClient.ingest.putPipeline).toBeCalledWith({
-    id: generateHistoryIngestPipelineId(builtInServicesEntityDefinition),
+    id: generateHistoryIngestPipelineId(builtInServicesFromLogsEntityDefinition),
     processors: expect.anything(),
     _meta: {
       definitionVersion: '0.1.0',
@@ -44,7 +44,7 @@ const assertHasCreatedDefinition = (
     },
   });
   expect(esClient.ingest.putPipeline).toBeCalledWith({
-    id: generateLatestIngestPipelineId(builtInServicesEntityDefinition),
+    id: generateLatestIngestPipelineId(builtInServicesFromLogsEntityDefinition),
     processors: expect.anything(),
     _meta: {
       definitionVersion: '0.1.0',
@@ -54,10 +54,10 @@ const assertHasCreatedDefinition = (
 
   expect(esClient.transform.putTransform).toBeCalledTimes(2);
   expect(esClient.transform.putTransform).toBeCalledWith(
-    generateHistoryTransform(builtInServicesEntityDefinition)
+    generateHistoryTransform(builtInServicesFromLogsEntityDefinition)
   );
   expect(esClient.transform.putTransform).toBeCalledWith(
-    generateLatestTransform(builtInServicesEntityDefinition)
+    generateLatestTransform(builtInServicesFromLogsEntityDefinition)
   );
 };
 
@@ -65,13 +65,13 @@ const assertHasStartedTransform = (definition: EntityDefinition, esClient: Elast
   expect(esClient.transform.startTransform).toBeCalledTimes(2);
   expect(esClient.transform.startTransform).toBeCalledWith(
     {
-      transform_id: generateHistoryTransformId(builtInServicesEntityDefinition),
+      transform_id: generateHistoryTransformId(builtInServicesFromLogsEntityDefinition),
     },
     expect.anything()
   );
   expect(esClient.transform.startTransform).toBeCalledWith(
     {
-      transform_id: generateLatestTransformId(builtInServicesEntityDefinition),
+      transform_id: generateLatestTransformId(builtInServicesFromLogsEntityDefinition),
     },
     expect.anything()
   );
@@ -116,7 +116,7 @@ const assertHasUninstalledDefinition = (
 describe('install_entity_definition', () => {
   describe('installBuiltInEntityDefinitions', () => {
     it('should install and start definition when not found', async () => {
-      const builtInDefinitions = [builtInServicesEntityDefinition];
+      const builtInDefinitions = [builtInServicesFromLogsEntityDefinition];
       const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
       const soClient = savedObjectsClientMock.create();
       soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 });
@@ -128,12 +128,12 @@ describe('install_entity_definition', () => {
         logger: loggerMock.create(),
       });
 
-      assertHasCreatedDefinition(builtInServicesEntityDefinition, soClient, esClient);
-      assertHasStartedTransform(builtInServicesEntityDefinition, esClient);
+      assertHasCreatedDefinition(builtInServicesFromLogsEntityDefinition, soClient, esClient);
+      assertHasStartedTransform(builtInServicesFromLogsEntityDefinition, esClient);
     });
 
     it('should reinstall when partial state found', async () => {
-      const builtInDefinitions = [builtInServicesEntityDefinition];
+      const builtInDefinitions = [builtInServicesFromLogsEntityDefinition];
       const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
       // mock partially installed definition
       esClient.ingest.getPipeline.mockResolvedValue({});
@@ -142,11 +142,11 @@ describe('install_entity_definition', () => {
       const definitionSOResult = {
         saved_objects: [
           {
-            id: builtInServicesEntityDefinition.id,
+            id: builtInServicesFromLogsEntityDefinition.id,
             type: 'entity-definition',
             references: [],
             score: 0,
-            attributes: builtInServicesEntityDefinition,
+            attributes: builtInServicesFromLogsEntityDefinition,
           },
         ],
         total: 1,
@@ -170,18 +170,18 @@ describe('install_entity_definition', () => {
         logger: loggerMock.create(),
       });
 
-      assertHasUninstalledDefinition(builtInServicesEntityDefinition, soClient, esClient);
-      assertHasCreatedDefinition(builtInServicesEntityDefinition, soClient, esClient);
-      assertHasStartedTransform(builtInServicesEntityDefinition, esClient);
+      assertHasUninstalledDefinition(builtInServicesFromLogsEntityDefinition, soClient, esClient);
+      assertHasCreatedDefinition(builtInServicesFromLogsEntityDefinition, soClient, esClient);
+      assertHasStartedTransform(builtInServicesFromLogsEntityDefinition, esClient);
     });
 
     it('should start a stopped definition', async () => {
-      const builtInDefinitions = [builtInServicesEntityDefinition];
+      const builtInDefinitions = [builtInServicesFromLogsEntityDefinition];
       const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
       // mock installed but stopped definition
       esClient.ingest.getPipeline.mockResolvedValue({
-        [generateHistoryIngestPipelineId(builtInServicesEntityDefinition)]: {},
-        [generateLatestIngestPipelineId(builtInServicesEntityDefinition)]: {},
+        [generateHistoryIngestPipelineId(builtInServicesFromLogsEntityDefinition)]: {},
+        [generateLatestIngestPipelineId(builtInServicesFromLogsEntityDefinition)]: {},
       });
       esClient.transform.getTransformStats.mockResolvedValue({
         // @ts-expect-error
@@ -192,11 +192,11 @@ describe('install_entity_definition', () => {
       soClient.find.mockResolvedValue({
         saved_objects: [
           {
-            id: builtInServicesEntityDefinition.id,
+            id: builtInServicesFromLogsEntityDefinition.id,
             type: 'entity-definition',
             references: [],
             score: 0,
-            attributes: builtInServicesEntityDefinition,
+            attributes: builtInServicesFromLogsEntityDefinition,
           },
         ],
         total: 1,
@@ -211,7 +211,7 @@ describe('install_entity_definition', () => {
       });
 
       expect(soClient.create).toHaveBeenCalledTimes(0);
-      assertHasStartedTransform(builtInServicesEntityDefinition, esClient);
+      assertHasStartedTransform(builtInServicesFromLogsEntityDefinition, esClient);
     });
   });
 });