[APM] Generate breakdown metrics (#114390)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/elastic-apm-generator/src/index.ts

@@ -12,3 +12,4 @@ export { getTransactionMetrics } from './lib/utils/get_transaction_metrics';
 export { getSpanDestinationMetrics } from './lib/utils/get_span_destination_metrics';
 export { getObserverDefaults } from './lib/defaults/get_observer_defaults';
 export { toElasticsearchOutput } from './lib/output/to_elasticsearch_output';
+export { getBreakdownMetrics } from './lib/utils/get_breakdown_metrics';

packages/elastic-apm-generator/src/lib/base_span.ts

@@ -8,10 +8,12 @@
 
 import { Fields } from './entity';
 import { Serializable } from './serializable';
+import { Span } from './span';
+import { Transaction } from './transaction';
 import { generateTraceId } from './utils/generate_id';
 
 export class BaseSpan extends Serializable {
-  private _children: BaseSpan[] = [];
+  private readonly _children: BaseSpan[] = [];
 
   constructor(fields: Fields) {
     super({
@@ -22,20 +24,29 @@ export class BaseSpan extends Serializable {
     });
   }
 
-  traceId(traceId: string) {
-    this.fields['trace.id'] = traceId;
+  parent(span: BaseSpan) {
+    this.fields['trace.id'] = span.fields['trace.id'];
+    this.fields['parent.id'] = span.isSpan()
+      ? span.fields['span.id']
+      : span.fields['transaction.id'];
+
+    if (this.isSpan()) {
+      this.fields['transaction.id'] = span.fields['transaction.id'];
+    }
     this._children.forEach((child) => {
-      child.fields['trace.id'] = traceId;
+      child.parent(this);
     });
+
     return this;
   }
 
   children(...children: BaseSpan[]) {
-    this._children.push(...children);
     children.forEach((child) => {
-      child.traceId(this.fields['trace.id']!);
+      child.parent(this);
     });
 
+    this._children.push(...children);
+
     return this;
   }
 
@@ -52,4 +63,12 @@ export class BaseSpan extends Serializable {
   serialize(): Fields[] {
     return [this.fields, ...this._children.flatMap((child) => child.serialize())];
   }
+
+  isSpan(): this is Span {
+    return this.fields['processor.event'] === 'span';
+  }
+
+  isTransaction(): this is Transaction {
+    return this.fields['processor.event'] === 'transaction';
+  }
 }

packages/elastic-apm-generator/src/lib/entity.ts

@@ -10,9 +10,11 @@ export type Fields = Partial<{
   '@timestamp': number;
   'agent.name': string;
   'agent.version': string;
+  'container.id': string;
   'ecs.version': string;
   'event.outcome': string;
   'event.ingested': number;
+  'host.name': string;
   'metricset.name': string;
   'observer.version': string;
   'observer.version_major': number;
@@ -42,6 +44,8 @@ export type Fields = Partial<{
   'span.destination.service.type': string;
   'span.destination.service.response_time.sum.us': number;
   'span.destination.service.response_time.count': number;
+  'span.self_time.count': number;
+  'span.self_time.sum.us': number;
 }>;
 
 export class Entity {

packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts

@@ -14,10 +14,12 @@ export function toElasticsearchOutput(events: Fields[], versionOverride?: string
   return events.map((event) => {
     const values = {
       ...event,
+      ...getObserverDefaults(),
       '@timestamp': new Date(event['@timestamp']!).toISOString(),
       'timestamp.us': event['@timestamp']! * 1000,
       'ecs.version': '1.4',
-      ...getObserverDefaults(),
+      'service.node.name':
+        event['service.node.name'] || event['container.id'] || event['host.name'],
     };
 
     const document = {};

packages/elastic-apm-generator/src/lib/service.ts

@@ -14,6 +14,7 @@ export class Service extends Entity {
     return new Instance({
       ...this.fields,
       ['service.node.name']: instanceName,
+      'container.id': instanceName,
     });
   }
 }

packages/elastic-apm-generator/src/lib/span.ts

@@ -19,18 +19,6 @@ export class Span extends BaseSpan {
     });
   }
 
-  children(...children: BaseSpan[]) {
-    super.children(...children);
-
-    children.forEach((child) =>
-      child.defaults({
-        'parent.id': this.fields['span.id'],
-      })
-    );
-
-    return this;
-  }
-
   duration(duration: number) {
     this.fields['span.duration.us'] = duration * 1000;
     return this;

packages/elastic-apm-generator/src/lib/transaction.ts

@@ -11,6 +11,8 @@ import { Fields } from './entity';
 import { generateEventId } from './utils/generate_id';
 
 export class Transaction extends BaseSpan {
+  private _sampled: boolean = true;
+
   constructor(fields: Fields) {
     super({
       ...fields,
@@ -19,19 +21,25 @@ export class Transaction extends BaseSpan {
       'transaction.sampled': true,
     });
   }
-  children(...children: BaseSpan[]) {
-    super.children(...children);
-    children.forEach((child) =>
-      child.defaults({
-        'transaction.id': this.fields['transaction.id'],
-        'parent.id': this.fields['transaction.id'],
-      })
-    );
-    return this;
-  }
 
   duration(duration: number) {
     this.fields['transaction.duration.us'] = duration * 1000;
     return this;
   }
+
+  sample(sampled: boolean = true) {
+    this._sampled = sampled;
+    return this;
+  }
+
+  serialize() {
+    const [transaction, ...spans] = super.serialize();
+
+    const events = [transaction];
+    if (this._sampled) {
+      events.push(...spans);
+    }
+
+    return events;
+  }
 }

packages/elastic-apm-generator/src/lib/utils/aggregate.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import moment from 'moment';
+import { pickBy } from 'lodash';
+import objectHash from 'object-hash';
+import { Fields } from '../entity';
+import { createPicker } from './create_picker';
+
+export function aggregate(events: Fields[], fields: string[]) {
+  const picker = createPicker(fields);
+
+  const metricsets = new Map<string, { key: Fields; events: Fields[] }>();
+
+  function getMetricsetKey(span: Fields) {
+    const timestamp = moment(span['@timestamp']).valueOf();
+    return {
+      '@timestamp': timestamp - (timestamp % (60 * 1000)),
+      ...pickBy(span, picker),
+    };
+  }
+
+  for (const event of events) {
+    const key = getMetricsetKey(event);
+    const id = objectHash(key);
+
+    let metricset = metricsets.get(id);
+
+    if (!metricset) {
+      metricset = {
+        key: { ...key, 'processor.event': 'metric', 'processor.name': 'metric' },
+        events: [],
+      };
+      metricsets.set(id, metricset);
+    }
+
+    metricset.events.push(event);
+  }
+
+  return Array.from(metricsets.values());
+}

packages/elastic-apm-generator/src/lib/utils/create_picker.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+export function createPicker(fields: string[]) {
+  const wildcards = fields
+    .filter((field) => field.endsWith('.*'))
+    .map((field) => field.replace('*', ''));
+
+  return (value: unknown, key: string) => {
+    return fields.includes(key) || wildcards.some((field) => key.startsWith(field));
+  };
+}

packages/elastic-apm-generator/src/lib/utils/get_breakdown_metrics.ts

@@ -0,0 +1,145 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import objectHash from 'object-hash';
+import { groupBy, pickBy } from 'lodash';
+import { Fields } from '../entity';
+import { createPicker } from './create_picker';
+
+const instanceFields = [
+  'container.*',
+  'kubernetes.*',
+  'agent.*',
+  'process.*',
+  'cloud.*',
+  'service.*',
+  'host.*',
+];
+
+const instancePicker = createPicker(instanceFields);
+
+const metricsetPicker = createPicker([
+  'transaction.type',
+  'transaction.name',
+  'span.type',
+  'span.subtype',
+]);
+
+export function getBreakdownMetrics(events: Fields[]) {
+  const txWithSpans = groupBy(
+    events.filter(
+      (event) => event['processor.event'] === 'span' || event['processor.event'] === 'transaction'
+    ),
+    (event) => event['transaction.id']
+  );
+
+  const metricsets: Map<string, Fields> = new Map();
+
+  Object.keys(txWithSpans).forEach((transactionId) => {
+    const txEvents = txWithSpans[transactionId];
+    const transaction = txEvents.find((event) => event['processor.event'] === 'transaction')!;
+
+    const eventsById: Record<string, Fields> = {};
+    const activityByParentId: Record<string, Array<{ from: number; to: number }>> = {};
+    for (const event of txEvents) {
+      const id =
+        event['processor.event'] === 'transaction' ? event['transaction.id'] : event['span.id'];
+      eventsById[id!] = event;
+
+      const parentId = event['parent.id'];
+
+      if (!parentId) {
+        continue;
+      }
+
+      if (!activityByParentId[parentId]) {
+        activityByParentId[parentId] = [];
+      }
+
+      const from = event['@timestamp']! * 1000;
+      const to =
+        from +
+        (event['processor.event'] === 'transaction'
+          ? event['transaction.duration.us']!
+          : event['span.duration.us']!);
+
+      activityByParentId[parentId].push({ from, to });
+    }
+
+    // eslint-disable-next-line guard-for-in
+    for (const id in eventsById) {
+      const event = eventsById[id];
+      const activities = activityByParentId[id] || [];
+
+      const timeStart = event['@timestamp']! * 1000;
+
+      let selfTime = 0;
+      let lastMeasurement = timeStart;
+      const changeTimestamps = [
+        ...new Set([
+          timeStart,
+          ...activities.flatMap((activity) => [activity.from, activity.to]),
+          timeStart +
+            (event['processor.event'] === 'transaction'
+              ? event['transaction.duration.us']!
+              : event['span.duration.us']!),
+        ]),
+      ];
+
+      for (const timestamp of changeTimestamps) {
+        const hasActiveChildren = activities.some(
+          (activity) => activity.from < timestamp && activity.to >= timestamp
+        );
+
+        if (!hasActiveChildren) {
+          selfTime += timestamp - lastMeasurement;
+        }
+
+        lastMeasurement = timestamp;
+      }
+
+      const key = {
+        '@timestamp': event['@timestamp']! - (event['@timestamp']! % (30 * 1000)),
+        'transaction.type': transaction['transaction.type'],
+        'transaction.name': transaction['transaction.name'],
+        ...pickBy(event, metricsetPicker),
+      };
+
+      const instance = pickBy(event, instancePicker);
+
+      const metricsetId = objectHash(key);
+
+      let metricset = metricsets.get(metricsetId);
+
+      if (!metricset) {
+        metricset = {
+          ...key,
+          ...instance,
+          'processor.event': 'metric',
+          'processor.name': 'metric',
+          'metricset.name': `span_breakdown`,
+          'span.self_time.count': 0,
+          'span.self_time.sum.us': 0,
+        };
+
+        if (event['processor.event'] === 'transaction') {
+          metricset['span.type'] = 'app';
+        } else {
+          metricset['span.type'] = event['span.type'];
+          metricset['span.subtype'] = event['span.subtype'];
+        }
+
+        metricsets.set(metricsetId, metricset);
+      }
+
+      metricset['span.self_time.count']!++;
+      metricset['span.self_time.sum.us']! += selfTime;
+    }
+  });
+
+  return Array.from(metricsets.values());
+}

packages/elastic-apm-generator/src/lib/utils/get_span_destination_metrics.ts

@@ -6,46 +6,34 @@
  * Side Public License, v 1.
  */
 
-import { pick } from 'lodash';
-import moment from 'moment';
-import objectHash from 'object-hash';
 import { Fields } from '../entity';
+import { aggregate } from './aggregate';
 
 export function getSpanDestinationMetrics(events: Fields[]) {
   const exitSpans = events.filter((event) => !!event['span.destination.service.resource']);
 
-  const metricsets = new Map<string, Fields>();
+  const metricsets = aggregate(exitSpans, [
+    'event.outcome',
+    'agent.name',
+    'service.environment',
+    'service.name',
+    'span.destination.service.resource',
+  ]);
 
-  function getSpanBucketKey(span: Fields) {
-    return {
-      '@timestamp': moment(span['@timestamp']).startOf('minute').valueOf(),
-      ...pick(span, [
-        'event.outcome',
-        'agent.name',
-        'service.environment',
-        'service.name',
-        'span.destination.service.resource',
-      ]),
-    };
-  }
+  return metricsets.map((metricset) => {
+    let count = 0;
+    let sum = 0;
 
-  for (const span of exitSpans) {
-    const key = getSpanBucketKey(span);
-    const id = objectHash(key);
-
-    let metricset = metricsets.get(id);
-    if (!metricset) {
-      metricset = {
-        ['processor.event']: 'metric',
-        ...key,
-        'span.destination.service.response_time.sum.us': 0,
-        'span.destination.service.response_time.count': 0,
-      };
-      metricsets.set(id, metricset);
+    for (const event of metricset.events) {
+      count++;
+      sum += event['span.duration.us']!;
     }
-    metricset['span.destination.service.response_time.count']! += 1;
-    metricset['span.destination.service.response_time.sum.us']! += span['span.duration.us']!;
-  }
 
-  return [...Array.from(metricsets.values())];
+    return {
+      ...metricset.key,
+      ['metricset.name']: 'span_destination',
+      'span.destination.service.response_time.sum.us': sum,
+      'span.destination.service.response_time.count': count,
+    };
+  });
 }

packages/elastic-apm-generator/src/lib/utils/get_transaction_metrics.ts

@@ -6,10 +6,9 @@
  * Side Public License, v 1.
  */
 
-import { pick, sortBy } from 'lodash';
-import moment from 'moment';
-import objectHash from 'object-hash';
+import { sortBy } from 'lodash';
 import { Fields } from '../entity';
+import { aggregate } from './aggregate';
 
 function sortAndCompressHistogram(histogram?: { values: number[]; counts: number[] }) {
   return sortBy(histogram?.values).reduce(
@@ -30,60 +29,45 @@ function sortAndCompressHistogram(histogram?: { values: number[]; counts: number
 }
 
 export function getTransactionMetrics(events: Fields[]) {
-  const transactions = events.filter((event) => event['processor.event'] === 'transaction');
-
-  const metricsets = new Map<string, Fields>();
-
-  function getTransactionBucketKey(transaction: Fields) {
-    return {
-      '@timestamp': moment(transaction['@timestamp']).startOf('minute').valueOf(),
-      'trace.root': transaction['parent.id'] === undefined,
-      ...pick(transaction, [
-        'transaction.name',
-        'transaction.type',
-        'event.outcome',
-        'transaction.result',
-        'agent.name',
-        'service.environment',
-        'service.name',
-        'service.version',
-        'host.name',
-        'container.id',
-        'kubernetes.pod.name',
-      ]),
-    };
-  }
-
-  for (const transaction of transactions) {
-    const key = getTransactionBucketKey(transaction);
-    const id = objectHash(key);
-    let metricset = metricsets.get(id);
-    if (!metricset) {
-      metricset = {
-        ...key,
-        ['processor.event']: 'metric',
-        'transaction.duration.histogram': {
-          values: [],
-          counts: [],
-        },
-      };
-      metricsets.set(id, metricset);
-    }
-    metricset['transaction.duration.histogram']?.counts.push(1);
-    metricset['transaction.duration.histogram']?.values.push(
-      Number(transaction['transaction.duration.us'])
-    );
-  }
-
-  return [
-    ...Array.from(metricsets.values()).map((metricset) => {
+  const transactions = events
+    .filter((event) => event['processor.event'] === 'transaction')
+    .map((transaction) => {
       return {
-        ...metricset,
-        ['transaction.duration.histogram']: sortAndCompressHistogram(
-          metricset['transaction.duration.histogram']
-        ),
-        _doc_count: metricset['transaction.duration.histogram']!.values.length,
+        ...transaction,
+        ['trace.root']: transaction['parent.id'] === undefined,
       };
-    }),
-  ];
+    });
+
+  const metricsets = aggregate(transactions, [
+    'trace.root',
+    'transaction.name',
+    'transaction.type',
+    'event.outcome',
+    'transaction.result',
+    'agent.name',
+    'service.environment',
+    'service.name',
+    'service.version',
+    'host.name',
+    'container.id',
+    'kubernetes.pod.name',
+  ]);
+
+  return metricsets.map((metricset) => {
+    const histogram = {
+      values: [] as number[],
+      counts: [] as number[],
+    };
+
+    for (const transaction of metricset.events) {
+      histogram.counts.push(1);
+      histogram.values.push(Number(transaction['transaction.duration.us']));
+    }
+
+    return {
+      ...metricset.key,
+      'transaction.duration.histogram': sortAndCompressHistogram(histogram),
+      _doc_count: metricset.events.length,
+    };
+  });
 }

packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts

@@ -7,17 +7,18 @@
  */
 
 import { service, timerange, getTransactionMetrics, getSpanDestinationMetrics } from '../..';
+import { getBreakdownMetrics } from '../../lib/utils/get_breakdown_metrics';
 
 export function simpleTrace(from: number, to: number) {
   const instance = service('opbeans-go', 'production', 'go').instance('instance');
 
   const range = timerange(from, to);
 
-  const transactionName = '100rpm (75% success) failed 1000ms';
+  const transactionName = '100rpm (80% success) failed 1000ms';
 
   const successfulTraceEvents = range
-    .interval('1m')
-    .rate(75)
+    .interval('30s')
+    .rate(40)
     .flatMap((timestamp) =>
       instance
         .transaction(transactionName)
@@ -31,14 +32,14 @@ export function simpleTrace(from: number, to: number) {
             .success()
             .destination('elasticsearch')
             .timestamp(timestamp),
-          instance.span('custom_operation', 'app').duration(50).success().timestamp(timestamp)
+          instance.span('custom_operation', 'custom').duration(100).success().timestamp(timestamp)
         )
         .serialize()
     );
 
   const failedTraceEvents = range
-    .interval('1m')
-    .rate(25)
+    .interval('30s')
+    .rate(10)
     .flatMap((timestamp) =>
       instance
         .transaction(transactionName)
@@ -50,5 +51,10 @@ export function simpleTrace(from: number, to: number) {
 
   const events = successfulTraceEvents.concat(failedTraceEvents);
 
-  return events.concat(getTransactionMetrics(events)).concat(getSpanDestinationMetrics(events));
+  return [
+    ...events,
+    ...getTransactionMetrics(events),
+    ...getSpanDestinationMetrics(events),
+    ...getBreakdownMetrics(events),
+  ];
 }

packages/elastic-apm-generator/src/test/scenarios/01_simple_trace.test.ts

@@ -68,6 +68,7 @@ describe('simple trace', () => {
     expect(transaction).toEqual({
       '@timestamp': 1609459200000,
       'agent.name': 'java',
+      'container.id': 'instance-1',
       'event.outcome': 'success',
       'processor.event': 'transaction',
       'processor.name': 'transaction',
@@ -89,6 +90,7 @@ describe('simple trace', () => {
     expect(span).toEqual({
       '@timestamp': 1609459200050,
       'agent.name': 'java',
+      'container.id': 'instance-1',
       'event.outcome': 'success',
       'parent.id': 'e7433020f2745625',
       'processor.event': 'span',

packages/elastic-apm-generator/src/test/scenarios/04_breakdown_metrics.test.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import { sumBy } from 'lodash';
+import { Fields } from '../../lib/entity';
+import { service } from '../../lib/service';
+import { timerange } from '../../lib/timerange';
+import { getBreakdownMetrics } from '../../lib/utils/get_breakdown_metrics';
+
+describe('breakdown metrics', () => {
+  let events: Fields[];
+
+  const LIST_RATE = 2;
+  const LIST_SPANS = 2;
+  const ID_RATE = 4;
+  const ID_SPANS = 2;
+  const INTERVALS = 6;
+
+  beforeEach(() => {
+    const javaService = service('opbeans-java', 'production', 'java');
+    const javaInstance = javaService.instance('instance-1');
+
+    const start = new Date('2021-01-01T00:00:00.000Z').getTime();
+
+    const range = timerange(start, start + INTERVALS * 30 * 1000 - 1);
+
+    events = getBreakdownMetrics([
+      ...range
+        .interval('30s')
+        .rate(LIST_RATE)
+        .flatMap((timestamp) =>
+          javaInstance
+            .transaction('GET /api/product/list')
+            .timestamp(timestamp)
+            .duration(1000)
+            .children(
+              javaInstance
+                .span('GET apm-*/_search', 'db', 'elasticsearch')
+                .timestamp(timestamp + 150)
+                .duration(500),
+              javaInstance.span('GET foo', 'db', 'redis').timestamp(timestamp).duration(100)
+            )
+            .serialize()
+        ),
+      ...range
+        .interval('30s')
+        .rate(ID_RATE)
+        .flatMap((timestamp) =>
+          javaInstance
+            .transaction('GET /api/product/:id')
+            .timestamp(timestamp)
+            .duration(1000)
+            .children(
+              javaInstance
+                .span('GET apm-*/_search', 'db', 'elasticsearch')
+                .duration(500)
+                .timestamp(timestamp + 100)
+                .children(
+                  javaInstance
+                    .span('bar', 'external', 'http')
+                    .timestamp(timestamp + 200)
+                    .duration(100)
+                )
+            )
+            .serialize()
+        ),
+    ]).filter((event) => event['processor.event'] === 'metric');
+  });
+
+  it('generates the right amount of breakdown metrics', () => {
+    expect(events.length).toBe(INTERVALS * (LIST_SPANS + 1 + ID_SPANS + 1));
+  });
+
+  it('calculates breakdown metrics for the right amount of transactions and spans', () => {
+    expect(sumBy(events, (event) => event['span.self_time.count']!)).toBe(
+      INTERVALS * LIST_RATE * (LIST_SPANS + 1) + INTERVALS * ID_RATE * (ID_SPANS + 1)
+    );
+  });
+
+  it('generates app metricsets for transaction self time', () => {
+    expect(events.some((event) => event['span.type'] === 'app' && !event['span.subtype'])).toBe(
+      true
+    );
+  });
+
+  it('generates the right statistic', () => {
+    const elasticsearchSets = events.filter((event) => event['span.subtype'] === 'elasticsearch');
+
+    const expectedCountFromListTransaction = INTERVALS * LIST_RATE;
+
+    const expectedCountFromIdTransaction = INTERVALS * ID_RATE;
+
+    const expectedCount = expectedCountFromIdTransaction + expectedCountFromListTransaction;
+
+    expect(sumBy(elasticsearchSets, (set) => set['span.self_time.count']!)).toBe(expectedCount);
+
+    expect(sumBy(elasticsearchSets, (set) => set['span.self_time.sum.us']!)).toBe(
+      expectedCountFromListTransaction * 500 * 1000 + expectedCountFromIdTransaction * 400 * 1000
+    );
+  });
+});

packages/elastic-apm-generator/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap

@@ -5,6 +5,7 @@ Array [
   Object {
     "@timestamp": 1609459200000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -21,6 +22,7 @@ Array [
   Object {
     "@timestamp": 1609459200050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "36c16f18e75058f8",
     "processor.event": "span",
@@ -39,6 +41,7 @@ Array [
   Object {
     "@timestamp": 1609459260000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -55,6 +58,7 @@ Array [
   Object {
     "@timestamp": 1609459260050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "65ce74106eb050be",
     "processor.event": "span",
@@ -73,6 +77,7 @@ Array [
   Object {
     "@timestamp": 1609459320000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -89,6 +94,7 @@ Array [
   Object {
     "@timestamp": 1609459320050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "91fa709d90625fff",
     "processor.event": "span",
@@ -107,6 +113,7 @@ Array [
   Object {
     "@timestamp": 1609459380000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -123,6 +130,7 @@ Array [
   Object {
     "@timestamp": 1609459380050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "6c500d1d19835e68",
     "processor.event": "span",
@@ -141,6 +149,7 @@ Array [
   Object {
     "@timestamp": 1609459440000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -157,6 +166,7 @@ Array [
   Object {
     "@timestamp": 1609459440050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "1b3246cc83595869",
     "processor.event": "span",
@@ -175,6 +185,7 @@ Array [
   Object {
     "@timestamp": 1609459500000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -191,6 +202,7 @@ Array [
   Object {
     "@timestamp": 1609459500050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "12b49e3c83fe58d5",
     "processor.event": "span",
@@ -209,6 +221,7 @@ Array [
   Object {
     "@timestamp": 1609459560000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -225,6 +238,7 @@ Array [
   Object {
     "@timestamp": 1609459560050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "d9272009dd4354a1",
     "processor.event": "span",
@@ -243,6 +257,7 @@ Array [
   Object {
     "@timestamp": 1609459620000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -259,6 +274,7 @@ Array [
   Object {
     "@timestamp": 1609459620050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "bc52ca08063c505b",
     "processor.event": "span",
@@ -277,6 +293,7 @@ Array [
   Object {
     "@timestamp": 1609459680000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -293,6 +310,7 @@ Array [
   Object {
     "@timestamp": 1609459680050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "186858dd88b75d59",
     "processor.event": "span",
@@ -311,6 +329,7 @@ Array [
   Object {
     "@timestamp": 1609459740000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -327,6 +346,7 @@ Array [
   Object {
     "@timestamp": 1609459740050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "0d5f44d48189546c",
     "processor.event": "span",
@@ -345,6 +365,7 @@ Array [
   Object {
     "@timestamp": 1609459800000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -361,6 +382,7 @@ Array [
   Object {
     "@timestamp": 1609459800050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "7483e0606e435c83",
     "processor.event": "span",
@@ -379,6 +401,7 @@ Array [
   Object {
     "@timestamp": 1609459860000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -395,6 +418,7 @@ Array [
   Object {
     "@timestamp": 1609459860050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "f142c4cbc7f3568e",
     "processor.event": "span",
@@ -413,6 +437,7 @@ Array [
   Object {
     "@timestamp": 1609459920000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -429,6 +454,7 @@ Array [
   Object {
     "@timestamp": 1609459920050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "2e3a47fa2d905519",
     "processor.event": "span",
@@ -447,6 +473,7 @@ Array [
   Object {
     "@timestamp": 1609459980000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -463,6 +490,7 @@ Array [
   Object {
     "@timestamp": 1609459980050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "de5eaa1e47dc56b1",
     "processor.event": "span",
@@ -481,6 +509,7 @@ Array [
   Object {
     "@timestamp": 1609460040000,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "processor.event": "transaction",
     "processor.name": "transaction",
@@ -497,6 +526,7 @@ Array [
   Object {
     "@timestamp": 1609460040050,
     "agent.name": "java",
+    "container.id": "instance-1",
     "event.outcome": "success",
     "parent.id": "af7eac7ae61e576a",
     "processor.event": "span",

x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts

@@ -109,6 +109,9 @@ export async function getServiceInstancesTransactionStatistics<
       filter: [
         { term: { [SERVICE_NAME]: serviceName } },
         { term: { [TRANSACTION_TYPE]: transactionType } },
+        ...getDocumentTypeFilterForAggregatedTransactions(
+          searchAggregatedTransactions
+        ),
         ...rangeQuery(start, end),
         ...environmentQuery(environment),
         ...kqlQuery(kuery),

x-pack/test/apm_api_integration/common/trace_data.ts

@@ -6,6 +6,7 @@
  */
 
 import {
+  getBreakdownMetrics,
   getSpanDestinationMetrics,
   getTransactionMetrics,
   toElasticsearchOutput,
@@ -20,7 +21,12 @@ export async function traceData(context: InheritedFtrProviderContext) {
   return {
     index: (events: any[]) => {
       const esEvents = toElasticsearchOutput(
-        events.concat(getTransactionMetrics(events)).concat(getSpanDestinationMetrics(events)),
+        [
+          ...events,
+          ...getTransactionMetrics(events),
+          ...getSpanDestinationMetrics(events),
+          ...getBreakdownMetrics(events),
+        ],
         '7.14.0'
       );
 

x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.ts

@@ -8,6 +8,7 @@
 import expect from '@kbn/expect';
 import { pick, sortBy } from 'lodash';
 import moment from 'moment';
+import { service, timerange } from '@elastic/apm-generator';
 import { APIReturnType } from '../../../../plugins/apm/public/services/rest/createCallApmApi';
 import { isFiniteNumber } from '../../../../plugins/apm/common/utils/is_finite_number';
 import { FtrProviderContext } from '../../common/ftr_provider_context';
@@ -15,9 +16,12 @@ import archives from '../../common/fixtures/es_archiver/archives_metadata';
 import { registry } from '../../common/registry';
 
 import { LatencyAggregationType } from '../../../../plugins/apm/common/latency_aggregation_types';
+import { ENVIRONMENT_ALL } from '../../../../plugins/apm/common/environment_filter_values';
+import { SERVICE_NODE_NAME_MISSING } from '../../../../plugins/apm/common/service_nodes';
 
 export default function ApiTest({ getService }: FtrProviderContext) {
   const apmApiClient = getService('apmApiClient');
+  const traceData = getService('traceData');
 
   const archiveName = 'apm_8.0.0';
   const { start, end } = archives[archiveName];
@@ -278,4 +282,145 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       });
     }
   );
+
+  registry.when(
+    'Service overview instances main statistics when data is generated',
+    { config: 'basic', archives: ['apm_8.0.0_empty'] },
+    () => {
+      describe('for two go instances and one java instance', () => {
+        const GO_A_INSTANCE_RATE_SUCCESS = 10;
+        const GO_A_INSTANCE_RATE_FAILURE = 5;
+        const GO_B_INSTANCE_RATE_SUCCESS = 15;
+
+        const JAVA_INSTANCE_RATE = 20;
+
+        const rangeStart = new Date('2021-01-01T12:00:00.000Z').getTime();
+        const rangeEnd = new Date('2021-01-01T12:15:00.000Z').getTime() - 1;
+
+        before(async () => {
+          const goService = service('opbeans-go', 'production', 'go');
+          const javaService = service('opbeans-java', 'production', 'java');
+
+          const goInstanceA = goService.instance('go-instance-a');
+          const goInstanceB = goService.instance('go-instance-b');
+          const javaInstance = javaService.instance('java-instance');
+
+          const interval = timerange(rangeStart, rangeEnd).interval('1m');
+
+          // include exit spans to generate span_destination metrics
+          // that should not be included
+          function withSpans(timestamp: number) {
+            return new Array(3).fill(undefined).map(() =>
+              goInstanceA
+                .span('GET apm-*/_search', 'db', 'elasticsearch')
+                .timestamp(timestamp + 100)
+                .duration(300)
+                .destination('elasticsearch')
+                .success()
+            );
+          }
+
+          return traceData.index([
+            ...interval.rate(GO_A_INSTANCE_RATE_SUCCESS).flatMap((timestamp) =>
+              goInstanceA
+                .transaction('GET /api/product/list')
+                .success()
+                .duration(500)
+                .timestamp(timestamp)
+                .children(...withSpans(timestamp))
+                .serialize()
+            ),
+            ...interval.rate(GO_A_INSTANCE_RATE_FAILURE).flatMap((timestamp) =>
+              goInstanceA
+                .transaction('GET /api/product/list')
+                .failure()
+                .duration(500)
+                .timestamp(timestamp)
+                .children(...withSpans(timestamp))
+                .serialize()
+            ),
+            ...interval.rate(GO_B_INSTANCE_RATE_SUCCESS).flatMap((timestamp) =>
+              goInstanceB
+                .transaction('GET /api/product/list')
+                .success()
+                .duration(500)
+                .timestamp(timestamp)
+                .children(...withSpans(timestamp))
+                .serialize()
+            ),
+            ...interval.rate(JAVA_INSTANCE_RATE).flatMap((timestamp) =>
+              javaInstance
+                .transaction('GET /api/product/list')
+                .success()
+                .duration(500)
+                .timestamp(timestamp)
+                .children(...withSpans(timestamp))
+                .serialize()
+            ),
+          ]);
+        });
+
+        after(async () => {
+          return traceData.clean();
+        });
+
+        describe('for the go service', () => {
+          let body: APIReturnType<'GET /internal/apm/services/{serviceName}/service_overview_instances/main_statistics'>;
+
+          before(async () => {
+            body = (
+              await apmApiClient.readUser({
+                endpoint:
+                  'GET /internal/apm/services/{serviceName}/service_overview_instances/main_statistics',
+                params: {
+                  path: {
+                    serviceName: 'opbeans-go',
+                  },
+                  query: {
+                    start: new Date(rangeStart).toISOString(),
+                    end: new Date(rangeEnd + 1).toISOString(),
+                    environment: ENVIRONMENT_ALL.value,
+                    kuery: '',
+                    latencyAggregationType: LatencyAggregationType.avg,
+                    transactionType: 'request',
+                  },
+                },
+              })
+            ).body;
+          });
+
+          it('returns statistics for the go instances', () => {
+            const goAStats = body.currentPeriod.find(
+              (stat) => stat.serviceNodeName === 'go-instance-a'
+            );
+            const goBStats = body.currentPeriod.find(
+              (stat) => stat.serviceNodeName === 'go-instance-b'
+            );
+
+            expect(goAStats?.throughput).to.eql(
+              GO_A_INSTANCE_RATE_SUCCESS + GO_A_INSTANCE_RATE_FAILURE
+            );
+
+            expect(goBStats?.throughput).to.eql(GO_B_INSTANCE_RATE_SUCCESS);
+          });
+
+          it('does not return data for the java service', () => {
+            const javaStats = body.currentPeriod.find(
+              (stat) => stat.serviceNodeName === 'java-instance'
+            );
+
+            expect(javaStats).to.be(undefined);
+          });
+
+          it('does not return data for missing service node name', () => {
+            const missingNameStats = body.currentPeriod.find(
+              (stat) => stat.serviceNodeName === SERVICE_NODE_NAME_MISSING
+            );
+
+            expect(missingNameStats).to.be(undefined);
+          });
+        });
+      });
+    }
+  );
 }