[APM] Trace generation library (#113764)

package.json

@@ -92,10 +92,11 @@
     "yarn": "^1.21.1"
   },
   "dependencies": {
+    "@babel/runtime": "^7.15.4",
     "@dnd-kit/core": "^3.1.1",
     "@dnd-kit/sortable": "^4.0.0",
     "@dnd-kit/utilities": "^2.0.0",
-    "@babel/runtime": "^7.15.4",
+    "@elastic/apm-generator": "link:bazel-bin/packages/elastic-apm-generator",
     "@elastic/apm-rum": "^5.9.1",
     "@elastic/apm-rum-react": "^1.3.1",
     "@elastic/charts": "34.2.1",

packages/BUILD.bazel

@@ -3,7 +3,8 @@
 filegroup(
   name = "build",
   srcs = [
-     "//packages/elastic-datemath:build",
+      "//packages/elastic-apm-generator:build",
+      "//packages/elastic-datemath:build",
       "//packages/elastic-eslint-config-kibana:build",
       "//packages/elastic-safer-lodash-set:build",
       "//packages/kbn-ace:build",

packages/elastic-apm-generator/BUILD.bazel

@@ -0,0 +1,99 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler")
+
+PKG_BASE_NAME = "elastic-apm-generator"
+PKG_REQUIRE_NAME = "@elastic/apm-generator"
+
+SOURCE_FILES = glob(
+  [
+    "src/**/*.ts",
+  ],
+  exclude = ["**/*.test.*"],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+  "README.md"
+]
+
+RUNTIME_DEPS = [
+  "@npm//@elastic/elasticsearch",
+  "@npm//lodash",
+  "@npm//moment",
+  "@npm//object-hash",
+  "@npm//p-limit",
+  "@npm//utility-types",
+  "@npm//uuid",
+  "@npm//yargs",
+]
+
+TYPES_DEPS = [
+  "@npm//@elastic/elasticsearch",
+  "@npm//moment",
+  "@npm//p-limit",
+  "@npm//@types/jest",
+  "@npm//@types/lodash",
+  "@npm//@types/node",
+  "@npm//@types/uuid",
+  "@npm//@types/object-hash",
+]
+
+jsts_transpiler(
+  name = "target_node",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+)
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+    "//:tsconfig.base.json",
+    "//:tsconfig.bazel.json",
+  ],
+)
+
+ts_project(
+  name = "tsc_types",
+  args = ['--pretty'],
+  srcs = SRCS,
+  deps = TYPES_DEPS,
+  declaration = True,
+  declaration_map = True,
+  emit_declaration_only = True,
+  out_dir = "target_types",
+  root_dir = "src",
+  source_map = True,
+  tsconfig = ":tsconfig",
+)
+
+js_library(
+  name = PKG_BASE_NAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  deps = RUNTIME_DEPS + [":target_node", ":tsc_types"],
+  package_name = PKG_REQUIRE_NAME,
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [
+    ":%s" % PKG_BASE_NAME,
+  ]
+)
+
+filegroup(
+  name = "build",
+  srcs = [
+    ":npm_module",
+  ],
+  visibility = ["//visibility:public"],
+)

packages/elastic-apm-generator/jest.config.js

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/elastic-apm-generator'],
+  setupFiles: [],
+  setupFilesAfterEnv: [],
+};

packages/elastic-apm-generator/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "@elastic/apm-generator",
+  "version": "0.1.0",
+  "description": "Elastic APM trace data generator",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "main": "./target_node/index.js",
+  "types": "./target_types/index.d.ts",
+  "private": true
+}

packages/elastic-apm-generator/src/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { service } from './lib/service';
+export { timerange } from './lib/timerange';
+export { getTransactionMetrics } from './lib/utils/get_transaction_metrics';
+export { getSpanDestinationMetrics } from './lib/utils/get_span_destination_metrics';
+export { getObserverDefaults } from './lib/defaults/get_observer_defaults';
+export { toElasticsearchOutput } from './lib/output/to_elasticsearch_output';

packages/elastic-apm-generator/src/lib/base_span.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Fields } from './entity';
+import { Serializable } from './serializable';
+import { generateTraceId } from './utils/generate_id';
+
+export class BaseSpan extends Serializable {
+  private _children: BaseSpan[] = [];
+
+  constructor(fields: Fields) {
+    super({
+      ...fields,
+      'event.outcome': 'unknown',
+      'trace.id': generateTraceId(),
+      'processor.name': 'transaction',
+    });
+  }
+
+  traceId(traceId: string) {
+    this.fields['trace.id'] = traceId;
+    this._children.forEach((child) => {
+      child.fields['trace.id'] = traceId;
+    });
+    return this;
+  }
+
+  children(...children: BaseSpan[]) {
+    this._children.push(...children);
+    children.forEach((child) => {
+      child.traceId(this.fields['trace.id']!);
+    });
+
+    return this;
+  }
+
+  success() {
+    this.fields['event.outcome'] = 'success';
+    return this;
+  }
+
+  failure() {
+    this.fields['event.outcome'] = 'failure';
+    return this;
+  }
+
+  serialize(): Fields[] {
+    return [this.fields, ...this._children.flatMap((child) => child.serialize())];
+  }
+}

packages/elastic-apm-generator/src/lib/defaults/get_observer_defaults.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Fields } from '../entity';
+
+export function getObserverDefaults(): Fields {
+  return {
+    'observer.version': '7.16.0',
+    'observer.version_major': 7,
+  };
+}

packages/elastic-apm-generator/src/lib/entity.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export type Fields = Partial<{
+  '@timestamp': number;
+  'agent.name': string;
+  'agent.version': string;
+  'ecs.version': string;
+  'event.outcome': string;
+  'event.ingested': number;
+  'metricset.name': string;
+  'observer.version': string;
+  'observer.version_major': number;
+  'parent.id': string;
+  'processor.event': string;
+  'processor.name': string;
+  'trace.id': string;
+  'transaction.name': string;
+  'transaction.type': string;
+  'transaction.id': string;
+  'transaction.duration.us': number;
+  'transaction.duration.histogram': {
+    values: number[];
+    counts: number[];
+  };
+  'transaction.sampled': true;
+  'service.name': string;
+  'service.environment': string;
+  'service.node.name': string;
+  'span.id': string;
+  'span.name': string;
+  'span.type': string;
+  'span.subtype': string;
+  'span.duration.us': number;
+  'span.destination.service.name': string;
+  'span.destination.service.resource': string;
+  'span.destination.service.type': string;
+  'span.destination.service.response_time.sum.us': number;
+  'span.destination.service.response_time.count': number;
+}>;
+
+export class Entity {
+  constructor(public readonly fields: Fields) {
+    this.fields = fields;
+  }
+
+  defaults(defaults: Fields) {
+    Object.keys(defaults).forEach((key) => {
+      const fieldName: keyof Fields = key as any;
+
+      if (!(fieldName in this.fields)) {
+        this.fields[fieldName] = defaults[fieldName] as any;
+      }
+    });
+
+    return this;
+  }
+}

packages/elastic-apm-generator/src/lib/instance.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Entity } from './entity';
+import { Span } from './span';
+import { Transaction } from './transaction';
+
+export class Instance extends Entity {
+  transaction(transactionName: string, transactionType = 'request') {
+    return new Transaction({
+      ...this.fields,
+      'transaction.name': transactionName,
+      'transaction.type': transactionType,
+    });
+  }
+
+  span(spanName: string, spanType: string, spanSubtype?: string) {
+    return new Span({
+      ...this.fields,
+      'span.name': spanName,
+      'span.type': spanType,
+      'span.subtype': spanSubtype,
+    });
+  }
+}

packages/elastic-apm-generator/src/lib/interval.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import moment from 'moment';
+
+export class Interval {
+  constructor(
+    private readonly from: number,
+    private readonly to: number,
+    private readonly interval: string
+  ) {}
+
+  rate(rate: number) {
+    let now = this.from;
+    const args = this.interval.match(/(.*)(s|m|h|d)/);
+    if (!args) {
+      throw new Error('Failed to parse interval');
+    }
+    const timestamps: number[] = [];
+    while (now <= this.to) {
+      timestamps.push(...new Array<number>(rate).fill(now));
+      now = moment(now)
+        .add(Number(args[1]), args[2] as any)
+        .valueOf();
+    }
+    return timestamps;
+  }
+}

packages/elastic-apm-generator/src/lib/metricset.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Serializable } from './serializable';
+
+export class Metricset extends Serializable {}
+
+export function metricset(name: string) {
+  return new Metricset({
+    'metricset.name': name,
+  });
+}

packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { set } from 'lodash';
+import { getObserverDefaults } from '../..';
+import { Fields } from '../entity';
+
+export function toElasticsearchOutput(events: Fields[], versionOverride?: string) {
+  return events.map((event) => {
+    const values = {
+      ...event,
+      '@timestamp': new Date(event['@timestamp']!).toISOString(),
+      'timestamp.us': event['@timestamp']! * 1000,
+      'ecs.version': '1.4',
+      ...getObserverDefaults(),
+    };
+
+    const document = {};
+    // eslint-disable-next-line guard-for-in
+    for (const key in values) {
+      set(document, key, values[key as keyof typeof values]);
+    }
+    return {
+      _index: `apm-${versionOverride || values['observer.version']}-${values['processor.event']}`,
+      _source: document,
+    };
+  });
+}

packages/elastic-apm-generator/src/lib/serializable.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Entity, Fields } from './entity';
+
+export class Serializable extends Entity {
+  constructor(fields: Fields) {
+    super({
+      ...fields,
+    });
+  }
+
+  timestamp(time: number) {
+    this.fields['@timestamp'] = time;
+    return this;
+  }
+  serialize(): Fields[] {
+    return [this.fields];
+  }
+}

packages/elastic-apm-generator/src/lib/service.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Entity } from './entity';
+import { Instance } from './instance';
+
+export class Service extends Entity {
+  instance(instanceName: string) {
+    return new Instance({
+      ...this.fields,
+      ['service.node.name']: instanceName,
+    });
+  }
+}
+
+export function service(name: string, environment: string, agentName: string) {
+  return new Service({
+    'service.name': name,
+    'service.environment': environment,
+    'agent.name': agentName,
+  });
+}

packages/elastic-apm-generator/src/lib/span.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { BaseSpan } from './base_span';
+import { Fields } from './entity';
+import { generateEventId } from './utils/generate_id';
+
+export class Span extends BaseSpan {
+  constructor(fields: Fields) {
+    super({
+      ...fields,
+      'processor.event': 'span',
+      'span.id': generateEventId(),
+    });
+  }
+
+  children(...children: BaseSpan[]) {
+    super.children(...children);
+
+    children.forEach((child) =>
+      child.defaults({
+        'parent.id': this.fields['span.id'],
+      })
+    );
+
+    return this;
+  }
+
+  duration(duration: number) {
+    this.fields['span.duration.us'] = duration * 1000;
+    return this;
+  }
+
+  destination(resource: string, type?: string, name?: string) {
+    if (!type) {
+      type = this.fields['span.type'];
+    }
+
+    if (!name) {
+      name = resource;
+    }
+    this.fields['span.destination.service.resource'] = resource;
+    this.fields['span.destination.service.name'] = name;
+    this.fields['span.destination.service.type'] = type;
+
+    return this;
+  }
+}

packages/elastic-apm-generator/src/lib/timerange.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Interval } from './interval';
+
+export class Timerange {
+  constructor(private from: number, private to: number) {}
+
+  interval(interval: string) {
+    return new Interval(this.from, this.to, interval);
+  }
+}
+
+export function timerange(from: number, to: number) {
+  return new Timerange(from, to);
+}

packages/elastic-apm-generator/src/lib/transaction.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { BaseSpan } from './base_span';
+import { Fields } from './entity';
+import { generateEventId } from './utils/generate_id';
+
+export class Transaction extends BaseSpan {
+  constructor(fields: Fields) {
+    super({
+      ...fields,
+      'processor.event': 'transaction',
+      'transaction.id': generateEventId(),
+      'transaction.sampled': true,
+    });
+  }
+  children(...children: BaseSpan[]) {
+    super.children(...children);
+    children.forEach((child) =>
+      child.defaults({
+        'transaction.id': this.fields['transaction.id'],
+        'parent.id': this.fields['transaction.id'],
+      })
+    );
+    return this;
+  }
+
+  duration(duration: number) {
+    this.fields['transaction.duration.us'] = duration * 1000;
+    return this;
+  }
+}

packages/elastic-apm-generator/src/lib/utils/generate_id.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import uuidv5 from 'uuid/v5';
+
+let seq = 0;
+
+const namespace = 'f38d5b83-8eee-4f5b-9aa6-2107e15a71e3';
+
+function generateId() {
+  return uuidv5(String(seq++), namespace).replace(/-/g, '');
+}
+
+export function generateEventId() {
+  return generateId().substr(0, 16);
+}
+
+export function generateTraceId() {
+  return generateId().substr(0, 32);
+}

packages/elastic-apm-generator/src/lib/utils/get_span_destination_metrics.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { pick } from 'lodash';
+import moment from 'moment';
+import objectHash from 'object-hash';
+import { Fields } from '../entity';
+
+export function getSpanDestinationMetrics(events: Fields[]) {
+  const exitSpans = events.filter((event) => !!event['span.destination.service.resource']);
+
+  const metricsets = new Map<string, Fields>();
+
+  function getSpanBucketKey(span: Fields) {
+    return {
+      '@timestamp': moment(span['@timestamp']).startOf('minute').valueOf(),
+      ...pick(span, [
+        'event.outcome',
+        'agent.name',
+        'service.environment',
+        'service.name',
+        'span.destination.service.resource',
+      ]),
+    };
+  }
+
+  for (const span of exitSpans) {
+    const key = getSpanBucketKey(span);
+    const id = objectHash(key);
+
+    let metricset = metricsets.get(id);
+    if (!metricset) {
+      metricset = {
+        ['processor.event']: 'metric',
+        ...key,
+        'span.destination.service.response_time.sum.us': 0,
+        'span.destination.service.response_time.count': 0,
+      };
+      metricsets.set(id, metricset);
+    }
+    metricset['span.destination.service.response_time.count']! += 1;
+    metricset['span.destination.service.response_time.sum.us']! += span['span.duration.us']!;
+  }
+
+  return [...Array.from(metricsets.values())];
+}

packages/elastic-apm-generator/src/lib/utils/get_transaction_metrics.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { pick, sortBy } from 'lodash';
+import moment from 'moment';
+import objectHash from 'object-hash';
+import { Fields } from '../entity';
+
+function sortAndCompressHistogram(histogram?: { values: number[]; counts: number[] }) {
+  return sortBy(histogram?.values).reduce(
+    (prev, current) => {
+      const lastValue = prev.values[prev.values.length - 1];
+      if (lastValue === current) {
+        prev.counts[prev.counts.length - 1]++;
+        return prev;
+      }
+
+      prev.counts.push(1);
+      prev.values.push(current);
+
+      return prev;
+    },
+    { values: [] as number[], counts: [] as number[] }
+  );
+}
+
+export function getTransactionMetrics(events: Fields[]) {
+  const transactions = events.filter((event) => event['processor.event'] === 'transaction');
+
+  const metricsets = new Map<string, Fields>();
+
+  function getTransactionBucketKey(transaction: Fields) {
+    return {
+      '@timestamp': moment(transaction['@timestamp']).startOf('minute').valueOf(),
+      'trace.root': transaction['parent.id'] === undefined,
+      ...pick(transaction, [
+        'transaction.name',
+        'transaction.type',
+        'event.outcome',
+        'transaction.result',
+        'agent.name',
+        'service.environment',
+        'service.name',
+        'service.version',
+        'host.name',
+        'container.id',
+        'kubernetes.pod.name',
+      ]),
+    };
+  }
+
+  for (const transaction of transactions) {
+    const key = getTransactionBucketKey(transaction);
+    const id = objectHash(key);
+    let metricset = metricsets.get(id);
+    if (!metricset) {
+      metricset = {
+        ...key,
+        ['processor.event']: 'metric',
+        'transaction.duration.histogram': {
+          values: [],
+          counts: [],
+        },
+      };
+      metricsets.set(id, metricset);
+    }
+    metricset['transaction.duration.histogram']?.counts.push(1);
+    metricset['transaction.duration.histogram']?.values.push(
+      Number(transaction['transaction.duration.us'])
+    );
+  }
+
+  return [
+    ...Array.from(metricsets.values()).map((metricset) => {
+      return {
+        ...metricset,
+        ['transaction.duration.histogram']: sortAndCompressHistogram(
+          metricset['transaction.duration.histogram']
+        ),
+        _doc_count: metricset['transaction.duration.histogram']!.values.length,
+      };
+    }),
+  ];
+}

packages/elastic-apm-generator/src/scripts/es.js

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/* eslint-disable @typescript-eslint/no-var-requires*/
+require('@babel/register')({
+  extensions: ['.ts', '.js'],
+  presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'],
+});
+
+require('./es.ts');

packages/elastic-apm-generator/src/scripts/es.ts

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { inspect } from 'util';
+import { Client } from '@elastic/elasticsearch';
+import { chunk } from 'lodash';
+import pLimit from 'p-limit';
+import yargs from 'yargs/yargs';
+import { toElasticsearchOutput } from '..';
+import { simpleTrace } from './examples/01_simple_trace';
+
+yargs(process.argv.slice(2))
+  .command(
+    'example',
+    'run an example scenario',
+    (y) => {
+      return y
+        .positional('scenario', {
+          describe: 'scenario to run',
+          choices: ['simple-trace'],
+          demandOption: true,
+        })
+        .option('target', {
+          describe: 'elasticsearch target, including username/password',
+        })
+        .option('from', { describe: 'start of timerange' })
+        .option('to', { describe: 'end of timerange' })
+        .option('workers', {
+          default: 1,
+          describe: 'number of concurrently connected ES clients',
+        })
+        .option('apm-server-version', {
+          describe: 'APM Server version override',
+        })
+        .demandOption('target');
+    },
+    (argv) => {
+      let events: any[] = [];
+      const toDateString = (argv.to as string | undefined) || new Date().toISOString();
+      const fromDateString =
+        (argv.from as string | undefined) ||
+        new Date(new Date(toDateString).getTime() - 15 * 60 * 1000).toISOString();
+
+      const to = new Date(toDateString).getTime();
+      const from = new Date(fromDateString).getTime();
+
+      switch (argv._[1]) {
+        case 'simple-trace':
+          events = simpleTrace(from, to);
+          break;
+      }
+
+      const docs = toElasticsearchOutput(events, argv['apm-server-version'] as string);
+
+      const client = new Client({
+        node: argv.target as string,
+      });
+
+      const fn = pLimit(argv.workers);
+
+      const batches = chunk(docs, 1000);
+
+      // eslint-disable-next-line no-console
+      console.log(
+        'Uploading',
+        docs.length,
+        'docs in',
+        batches.length,
+        'batches',
+        'from',
+        fromDateString,
+        'to',
+        toDateString
+      );
+
+      Promise.all(
+        batches.map((batch) =>
+          fn(() => {
+            return client.bulk({
+              require_alias: true,
+              body: batch.flatMap((doc) => {
+                return [{ index: { _index: doc._index } }, doc._source];
+              }),
+            });
+          })
+        )
+      )
+        .then((results) => {
+          const errors = results
+            .flatMap((result) => result.body.items)
+            .filter((item) => !!item.index?.error)
+            .map((item) => item.index?.error);
+
+          if (errors.length) {
+            // eslint-disable-next-line no-console
+            console.error(inspect(errors.slice(0, 10), { depth: null }));
+            throw new Error('Failed to upload some items');
+          }
+          process.exit();
+        })
+        .catch((err) => {
+          // eslint-disable-next-line no-console
+          console.error(err);
+          process.exit(1);
+        });
+    }
+  )
+  .parse();

packages/elastic-apm-generator/src/scripts/examples/01_simple_trace.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { service, timerange, getTransactionMetrics, getSpanDestinationMetrics } from '../..';
+
+export function simpleTrace(from: number, to: number) {
+  const instance = service('opbeans-go', 'production', 'go').instance('instance');
+
+  const range = timerange(from, to);
+
+  const transactionName = '100rpm (75% success) failed 1000ms';
+
+  const successfulTraceEvents = range
+    .interval('1m')
+    .rate(75)
+    .flatMap((timestamp) =>
+      instance
+        .transaction(transactionName)
+        .timestamp(timestamp)
+        .duration(1000)
+        .success()
+        .children(
+          instance
+            .span('GET apm-*/_search', 'db', 'elasticsearch')
+            .duration(1000)
+            .success()
+            .destination('elasticsearch')
+            .timestamp(timestamp),
+          instance.span('custom_operation', 'app').duration(50).success().timestamp(timestamp)
+        )
+        .serialize()
+    );
+
+  const failedTraceEvents = range
+    .interval('1m')
+    .rate(25)
+    .flatMap((timestamp) =>
+      instance
+        .transaction(transactionName)
+        .timestamp(timestamp)
+        .duration(1000)
+        .failure()
+        .serialize()
+    );
+
+  const events = successfulTraceEvents.concat(failedTraceEvents);
+
+  return events.concat(getTransactionMetrics(events)).concat(getSpanDestinationMetrics(events));
+}

packages/elastic-apm-generator/src/test/scenarios/01_simple_trace.test.ts

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { service } from '../../lib/service';
+import { timerange } from '../../lib/timerange';
+
+describe('simple trace', () => {
+  let events: Array<Record<string, any>>;
+
+  beforeEach(() => {
+    const javaService = service('opbeans-java', 'production', 'java');
+    const javaInstance = javaService.instance('instance-1');
+
+    const range = timerange(
+      new Date('2021-01-01T00:00:00.000Z').getTime(),
+      new Date('2021-01-01T00:15:00.000Z').getTime() - 1
+    );
+
+    events = range
+      .interval('1m')
+      .rate(1)
+      .flatMap((timestamp) =>
+        javaInstance
+          .transaction('GET /api/product/list')
+          .duration(1000)
+          .success()
+          .timestamp(timestamp)
+          .children(
+            javaInstance
+              .span('GET apm-*/_search', 'db', 'elasticsearch')
+              .success()
+              .duration(900)
+              .timestamp(timestamp + 50)
+          )
+          .serialize()
+      );
+  });
+
+  it('generates the same data every time', () => {
+    expect(events).toMatchSnapshot();
+  });
+
+  it('generates 15 transaction events', () => {
+    expect(events.filter((event) => event['processor.event'] === 'transaction').length).toEqual(15);
+  });
+
+  it('generates 15 span events', () => {
+    expect(events.filter((event) => event['processor.event'] === 'span').length).toEqual(15);
+  });
+
+  it('correctly sets the trace/transaction id of children', () => {
+    const [transaction, span] = events;
+
+    expect(span['transaction.id']).toEqual(transaction['transaction.id']);
+    expect(span['parent.id']).toEqual(transaction['transaction.id']);
+
+    expect(span['trace.id']).toEqual(transaction['trace.id']);
+  });
+
+  it('outputs transaction events', () => {
+    const [transaction] = events;
+
+    expect(transaction).toEqual({
+      '@timestamp': 1609459200000,
+      'agent.name': 'java',
+      'event.outcome': 'success',
+      'processor.event': 'transaction',
+      'processor.name': 'transaction',
+      'service.environment': 'production',
+      'service.name': 'opbeans-java',
+      'service.node.name': 'instance-1',
+      'trace.id': 'f6eb2f1cbba2597e89d2a63771c4344d',
+      'transaction.duration.us': 1000000,
+      'transaction.id': 'e9ece67cbacb52bf',
+      'transaction.name': 'GET /api/product/list',
+      'transaction.type': 'request',
+      'transaction.sampled': true,
+    });
+  });
+
+  it('outputs span events', () => {
+    const [, span] = events;
+
+    expect(span).toEqual({
+      '@timestamp': 1609459200050,
+      'agent.name': 'java',
+      'event.outcome': 'success',
+      'parent.id': 'e7433020f2745625',
+      'processor.event': 'span',
+      'processor.name': 'transaction',
+      'service.environment': 'production',
+      'service.name': 'opbeans-java',
+      'service.node.name': 'instance-1',
+      'span.duration.us': 900000,
+      'span.id': '21a776b44b9853dd',
+      'span.name': 'GET apm-*/_search',
+      'span.subtype': 'elasticsearch',
+      'span.type': 'db',
+      'trace.id': '048a0647263853abb94649ec0b92bdb4',
+      'transaction.id': 'e7433020f2745625',
+    });
+  });
+});

packages/elastic-apm-generator/src/test/scenarios/02_transaction_metrics.test.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { service } from '../../lib/service';
+import { timerange } from '../../lib/timerange';
+import { getTransactionMetrics } from '../../lib/utils/get_transaction_metrics';
+
+describe('transaction metrics', () => {
+  let events: Array<Record<string, any>>;
+
+  beforeEach(() => {
+    const javaService = service('opbeans-java', 'production', 'java');
+    const javaInstance = javaService.instance('instance-1');
+
+    const range = timerange(
+      new Date('2021-01-01T00:00:00.000Z').getTime(),
+      new Date('2021-01-01T00:15:00.000Z').getTime() - 1
+    );
+
+    events = getTransactionMetrics(
+      range
+        .interval('1m')
+        .rate(25)
+        .flatMap((timestamp) =>
+          javaInstance
+            .transaction('GET /api/product/list')
+            .duration(1000)
+            .success()
+            .timestamp(timestamp)
+            .serialize()
+        )
+        .concat(
+          range
+            .interval('1m')
+            .rate(50)
+            .flatMap((timestamp) =>
+              javaInstance
+                .transaction('GET /api/product/list')
+                .duration(1000)
+                .failure()
+                .timestamp(timestamp)
+                .serialize()
+            )
+        )
+    );
+  });
+
+  it('generates the right amount of transaction metrics', () => {
+    expect(events.length).toBe(30);
+  });
+
+  it('generates a metricset per interval', () => {
+    const metricsSetsForSuccessfulTransactions = events.filter(
+      (event) => event['event.outcome'] === 'success'
+    );
+
+    const [first, second] = metricsSetsForSuccessfulTransactions.map((event) =>
+      new Date(event['@timestamp']).toISOString()
+    );
+
+    expect([first, second]).toEqual(['2021-01-01T00:00:00.000Z', '2021-01-01T00:01:00.000Z']);
+  });
+
+  it('generates a metricset per value of event.outcome', () => {
+    const metricsSetsForSuccessfulTransactions = events.filter(
+      (event) => event['event.outcome'] === 'success'
+    );
+
+    const metricsSetsForFailedTransactions = events.filter(
+      (event) => event['event.outcome'] === 'failure'
+    );
+
+    expect(metricsSetsForSuccessfulTransactions.length).toBe(15);
+    expect(metricsSetsForFailedTransactions.length).toBe(15);
+  });
+
+  it('captures all the values from aggregated transactions', () => {
+    const metricsSetsForSuccessfulTransactions = events.filter(
+      (event) => event['event.outcome'] === 'success'
+    );
+
+    const metricsSetsForFailedTransactions = events.filter(
+      (event) => event['event.outcome'] === 'failure'
+    );
+
+    expect(metricsSetsForSuccessfulTransactions.length).toBe(15);
+
+    metricsSetsForSuccessfulTransactions.forEach((event) => {
+      expect(event['transaction.duration.histogram']).toEqual({
+        values: [1000000],
+        counts: [25],
+      });
+    });
+
+    metricsSetsForFailedTransactions.forEach((event) => {
+      expect(event['transaction.duration.histogram']).toEqual({
+        values: [1000000],
+        counts: [50],
+      });
+    });
+  });
+});

packages/elastic-apm-generator/src/test/scenarios/03_span_destination_metrics.test.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { service } from '../../lib/service';
+import { timerange } from '../../lib/timerange';
+import { getSpanDestinationMetrics } from '../../lib/utils/get_span_destination_metrics';
+
+describe('span destination metrics', () => {
+  let events: Array<Record<string, any>>;
+
+  beforeEach(() => {
+    const javaService = service('opbeans-java', 'production', 'java');
+    const javaInstance = javaService.instance('instance-1');
+
+    const range = timerange(
+      new Date('2021-01-01T00:00:00.000Z').getTime(),
+      new Date('2021-01-01T00:15:00.000Z').getTime() - 1
+    );
+
+    events = getSpanDestinationMetrics(
+      range
+        .interval('1m')
+        .rate(25)
+        .flatMap((timestamp) =>
+          javaInstance
+            .transaction('GET /api/product/list')
+            .duration(1000)
+            .success()
+            .timestamp(timestamp)
+            .children(
+              javaInstance
+                .span('GET apm-*/_search', 'db', 'elasticsearch')
+                .timestamp(timestamp)
+                .duration(1000)
+                .destination('elasticsearch')
+                .success()
+            )
+            .serialize()
+        )
+        .concat(
+          range
+            .interval('1m')
+            .rate(50)
+            .flatMap((timestamp) =>
+              javaInstance
+                .transaction('GET /api/product/list')
+                .duration(1000)
+                .failure()
+                .timestamp(timestamp)
+                .children(
+                  javaInstance
+                    .span('GET apm-*/_search', 'db', 'elasticsearch')
+                    .timestamp(timestamp)
+                    .duration(1000)
+                    .destination('elasticsearch')
+                    .failure(),
+                  javaInstance
+                    .span('custom_operation', 'app')
+                    .timestamp(timestamp)
+                    .duration(500)
+                    .success()
+                )
+                .serialize()
+            )
+        )
+    );
+  });
+
+  it('generates the right amount of span metrics', () => {
+    expect(events.length).toBe(30);
+  });
+
+  it('does not generate metricsets for non-exit spans', () => {
+    expect(
+      events.every((event) => event['span.destination.service.resource'] === 'elasticsearch')
+    ).toBe(true);
+  });
+
+  it('captures all the values from aggregated exit spans', () => {
+    const metricsSetsForSuccessfulExitSpans = events.filter(
+      (event) => event['event.outcome'] === 'success'
+    );
+
+    const metricsSetsForFailedExitSpans = events.filter(
+      (event) => event['event.outcome'] === 'failure'
+    );
+
+    expect(metricsSetsForSuccessfulExitSpans.length).toBe(15);
+
+    metricsSetsForSuccessfulExitSpans.forEach((event) => {
+      expect(event['span.destination.service.response_time.count']).toEqual(25);
+      expect(event['span.destination.service.response_time.sum.us']).toEqual(25000000);
+    });
+
+    metricsSetsForFailedExitSpans.forEach((event) => {
+      expect(event['span.destination.service.response_time.count']).toEqual(50);
+      expect(event['span.destination.service.response_time.sum.us']).toEqual(50000000);
+    });
+  });
+});

packages/elastic-apm-generator/src/test/scenarios/__snapshots__/01_simple_trace.test.ts.snap

@@ -0,0 +1,516 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`simple trace generates the same data every time 1`] = `
+Array [
+  Object {
+    "@timestamp": 1609459200000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "b1c6c04a9ac15b138f716d383cc85e6b",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "36c16f18e75058f8",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459200050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "36c16f18e75058f8",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "fe778a305e6d57dd",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "b1c6c04a9ac15b138f716d383cc85e6b",
+    "transaction.id": "36c16f18e75058f8",
+  },
+  Object {
+    "@timestamp": 1609459260000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "53c6c37bd4c85f4fbc880cd80704a9cd",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "65ce74106eb050be",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459260050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "65ce74106eb050be",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "ad8c5e249a8658ec",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "53c6c37bd4c85f4fbc880cd80704a9cd",
+    "transaction.id": "65ce74106eb050be",
+  },
+  Object {
+    "@timestamp": 1609459320000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "5eebf2e8d8cc5f85be8c573a1b501c7d",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "91fa709d90625fff",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459320050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "91fa709d90625fff",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "228b569c530c52ac",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "5eebf2e8d8cc5f85be8c573a1b501c7d",
+    "transaction.id": "91fa709d90625fff",
+  },
+  Object {
+    "@timestamp": 1609459380000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "6e8da4beb752589a86d53287c9d902de",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "6c500d1d19835e68",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459380050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "6c500d1d19835e68",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "5eb13f140bde5334",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "6e8da4beb752589a86d53287c9d902de",
+    "transaction.id": "6c500d1d19835e68",
+  },
+  Object {
+    "@timestamp": 1609459440000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "0aaa92bd91df543c8fd10b662051d9e8",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "1b3246cc83595869",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459440050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "1b3246cc83595869",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "582221c79fd75a76",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "0aaa92bd91df543c8fd10b662051d9e8",
+    "transaction.id": "1b3246cc83595869",
+  },
+  Object {
+    "@timestamp": 1609459500000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "26be5f0e2c16576ebf5f39c505eb1ff2",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "12b49e3c83fe58d5",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459500050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "12b49e3c83fe58d5",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "526d186996835c09",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "26be5f0e2c16576ebf5f39c505eb1ff2",
+    "transaction.id": "12b49e3c83fe58d5",
+  },
+  Object {
+    "@timestamp": 1609459560000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "c17c414c0b51564ca30e2ad839393180",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "d9272009dd4354a1",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459560050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "d9272009dd4354a1",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "7582541fcbfc5dc6",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "c17c414c0b51564ca30e2ad839393180",
+    "transaction.id": "d9272009dd4354a1",
+  },
+  Object {
+    "@timestamp": 1609459620000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "0280b1ffaae75e7ab097c0b52c3b3e6a",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "bc52ca08063c505b",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459620050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "bc52ca08063c505b",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "37ab978487935abb",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "0280b1ffaae75e7ab097c0b52c3b3e6a",
+    "transaction.id": "bc52ca08063c505b",
+  },
+  Object {
+    "@timestamp": 1609459680000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "6fb5191297fb59cebdb6a0196e273676",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "186858dd88b75d59",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459680050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "186858dd88b75d59",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "5ab56f27d0ae569b",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "6fb5191297fb59cebdb6a0196e273676",
+    "transaction.id": "186858dd88b75d59",
+  },
+  Object {
+    "@timestamp": 1609459740000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "77b5ffe303ae59b49f9b0e5d5270c16a",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "0d5f44d48189546c",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459740050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "0d5f44d48189546c",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "80e94b0847cd5104",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "77b5ffe303ae59b49f9b0e5d5270c16a",
+    "transaction.id": "0d5f44d48189546c",
+  },
+  Object {
+    "@timestamp": 1609459800000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "51c6b70db4dc5cf89b690de45c0c7b71",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "7483e0606e435c83",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459800050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "7483e0606e435c83",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "2e99d193e0f954c1",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "51c6b70db4dc5cf89b690de45c0c7b71",
+    "transaction.id": "7483e0606e435c83",
+  },
+  Object {
+    "@timestamp": 1609459860000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "5d91a6cde6015897935e413bc500f211",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "f142c4cbc7f3568e",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459860050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "f142c4cbc7f3568e",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "1fc52f16e2f551ea",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "5d91a6cde6015897935e413bc500f211",
+    "transaction.id": "f142c4cbc7f3568e",
+  },
+  Object {
+    "@timestamp": 1609459920000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "c097c19d884d52579bb11a601b8a98b3",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "2e3a47fa2d905519",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459920050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "2e3a47fa2d905519",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "7c7828c850685337",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "c097c19d884d52579bb11a601b8a98b3",
+    "transaction.id": "2e3a47fa2d905519",
+  },
+  Object {
+    "@timestamp": 1609459980000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "4591e57f4d7f5986bdd7892561224e0f",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "de5eaa1e47dc56b1",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609459980050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "de5eaa1e47dc56b1",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "8f62257f4a41546a",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "4591e57f4d7f5986bdd7892561224e0f",
+    "transaction.id": "de5eaa1e47dc56b1",
+  },
+  Object {
+    "@timestamp": 1609460040000,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "processor.event": "transaction",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "trace.id": "85ee8e618433577b9316a1e14961aa89",
+    "transaction.duration.us": 1000000,
+    "transaction.id": "af7eac7ae61e576a",
+    "transaction.name": "GET /api/product/list",
+    "transaction.sampled": true,
+    "transaction.type": "request",
+  },
+  Object {
+    "@timestamp": 1609460040050,
+    "agent.name": "java",
+    "event.outcome": "success",
+    "parent.id": "af7eac7ae61e576a",
+    "processor.event": "span",
+    "processor.name": "transaction",
+    "service.environment": "production",
+    "service.name": "opbeans-java",
+    "service.node.name": "instance-1",
+    "span.duration.us": 900000,
+    "span.id": "cc88b4cd921e590e",
+    "span.name": "GET apm-*/_search",
+    "span.subtype": "elasticsearch",
+    "span.type": "db",
+    "trace.id": "85ee8e618433577b9316a1e14961aa89",
+    "transaction.id": "af7eac7ae61e576a",
+  },
+]
+`;

packages/elastic-apm-generator/src/test/to_elasticsearch_output.test.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Fields } from '../lib/entity';
+import { toElasticsearchOutput } from '../lib/output/to_elasticsearch_output';
+
+describe('output to elasticsearch', () => {
+  let event: Fields;
+
+  beforeEach(() => {
+    event = {
+      '@timestamp': new Date('2020-12-31T23:00:00.000Z').getTime(),
+      'processor.event': 'transaction',
+      'processor.name': 'transaction',
+    };
+  });
+
+  it('properly formats @timestamp', () => {
+    const doc = toElasticsearchOutput([event])[0] as any;
+
+    expect(doc._source['@timestamp']).toEqual('2020-12-31T23:00:00.000Z');
+  });
+
+  it('formats a nested object', () => {
+    const doc = toElasticsearchOutput([event])[0] as any;
+
+    expect(doc._source.processor).toEqual({
+      event: 'transaction',
+      name: 'transaction',
+    });
+  });
+});

packages/elastic-apm-generator/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../tsconfig.bazel.json",
+  "compilerOptions": {
+    "declaration": true,
+    "declarationMap": true,
+    "emitDeclarationOnly": true,
+    "outDir": "target_types",
+    "rootDir": "./src",
+    "sourceMap": true,
+    "sourceRoot": "../../../../packages/elastic-apm-generator/src",
+    "types": [
+      "node",
+      "jest"
+    ]
+  },
+  "include": [
+    "./src/**/*.ts"
+  ]
+}

x-pack/test/apm_api_integration/common/config.ts

@@ -15,6 +15,7 @@ import { createApmUser, APM_TEST_PASSWORD, ApmUser } from './authentication';
 import { APMFtrConfigName } from '../configs';
 import { createApmApiClient } from './apm_api_supertest';
 import { registry } from './registry';
+import { traceData } from './trace_data';
 
 interface Config {
   name: APMFtrConfigName;
@@ -76,7 +77,7 @@ export function createTestConfig(config: Config) {
       servers,
       services: {
         ...services,
-
+        traceData,
         apmApiClient: async (context: InheritedFtrProviderContext) => {
           const security = context.getService('security');
           await security.init();

x-pack/test/apm_api_integration/common/registry.ts

@@ -15,6 +15,7 @@ import { FtrProviderContext } from './ftr_provider_context';
 
 type ArchiveName =
   | 'apm_8.0.0'
+  | 'apm_8.0.0_empty'
   | '8.0.0'
   | 'metrics_8.0.0'
   | 'ml_8.0.0'

x-pack/test/apm_api_integration/common/trace_data.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  getSpanDestinationMetrics,
+  getTransactionMetrics,
+  toElasticsearchOutput,
+} from '@elastic/apm-generator';
+import { chunk } from 'lodash';
+import pLimit from 'p-limit';
+import { inspect } from 'util';
+import { InheritedFtrProviderContext } from './ftr_provider_context';
+
+export async function traceData(context: InheritedFtrProviderContext) {
+  const es = context.getService('es');
+  return {
+    index: (events: any[]) => {
+      const esEvents = toElasticsearchOutput(
+        events.concat(getTransactionMetrics(events)).concat(getSpanDestinationMetrics(events)),
+        '7.14.0'
+      );
+
+      const batches = chunk(esEvents, 1000);
+      const limiter = pLimit(1);
+
+      return Promise.all(
+        batches.map((batch) =>
+          limiter(() => {
+            return es.bulk({
+              body: batch.flatMap(({ _index, _source }) => [{ index: { _index } }, _source]),
+              require_alias: true,
+              refresh: true,
+            });
+          })
+        )
+      ).then((results) => {
+        const errors = results
+          .flatMap((result) => result.body.items)
+          .filter((item) => !!item.index?.error)
+          .map((item) => item.index?.error);
+
+        if (errors.length) {
+          // eslint-disable-next-line no-console
+          console.log(inspect(errors.slice(0, 10), { depth: null }));
+          throw new Error('Failed to upload some events');
+        }
+        return results;
+      });
+    },
+    clean: () => {
+      return es.deleteByQuery({
+        index: 'apm-*',
+        body: {
+          query: {
+            match_all: {},
+          },
+        },
+      });
+    },
+  };
+}

x-pack/test/apm_api_integration/tests/services/throughput.ts

@@ -5,11 +5,14 @@
  * 2.0.
  */
 
+import { service, timerange } from '@elastic/apm-generator';
 import expect from '@kbn/expect';
-import { first, last, mean } from 'lodash';
+import { first, last, mean, uniq } from 'lodash';
 import moment from 'moment';
+import { ENVIRONMENT_ALL } from '../../../../plugins/apm/common/environment_filter_values';
 import { isFiniteNumber } from '../../../../plugins/apm/common/utils/is_finite_number';
 import { APIReturnType } from '../../../../plugins/apm/public/services/rest/createCallApmApi';
+import { PromiseReturnType } from '../../../../plugins/observability/typings/common';
 import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata';
 import { FtrProviderContext } from '../../common/ftr_provider_context';
 import { registry } from '../../common/registry';
@@ -18,10 +21,148 @@ type ThroughputReturn = APIReturnType<'GET /api/apm/services/{serviceName}/throu
 
 export default function ApiTest({ getService }: FtrProviderContext) {
   const apmApiClient = getService('apmApiClient');
+  const traceData = getService('traceData');
 
   const archiveName = 'apm_8.0.0';
   const metadata = archives_metadata[archiveName];
 
+  registry.when(
+    'Throughput with statically generated data',
+    { config: 'basic', archives: ['apm_8.0.0_empty'] },
+    () => {
+      const start = new Date('2021-01-01T00:00:00.000Z').getTime();
+      const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
+
+      const GO_PROD_RATE = 10;
+      const GO_DEV_RATE = 5;
+      const JAVA_PROD_RATE = 20;
+
+      before(async () => {
+        const serviceGoProdInstance = service('synth-go', 'production', 'go').instance(
+          'instance-a'
+        );
+        const serviceGoDevInstance = service('synth-go', 'development', 'go').instance(
+          'instance-b'
+        );
+        const serviceJavaInstance = service('synth-java', 'production', 'java').instance(
+          'instance-c'
+        );
+
+        await traceData.index([
+          ...timerange(start, end)
+            .interval('1s')
+            .rate(GO_PROD_RATE)
+            .flatMap((timestamp) =>
+              serviceGoProdInstance
+                .transaction('GET /api/product/list')
+                .duration(1000)
+                .timestamp(timestamp)
+                .serialize()
+            ),
+          ...timerange(start, end)
+            .interval('1s')
+            .rate(GO_DEV_RATE)
+            .flatMap((timestamp) =>
+              serviceGoDevInstance
+                .transaction('GET /api/product/:id')
+                .duration(1000)
+                .timestamp(timestamp)
+                .serialize()
+            ),
+          ...timerange(start, end)
+            .interval('1s')
+            .rate(JAVA_PROD_RATE)
+            .flatMap((timestamp) =>
+              serviceJavaInstance
+                .transaction('POST /api/product/buy')
+                .duration(1000)
+                .timestamp(timestamp)
+                .serialize()
+            ),
+        ]);
+      });
+
+      after(() => traceData.clean());
+
+      async function callApi(overrides?: {
+        start?: string;
+        end?: string;
+        transactionType?: string;
+        environment?: string;
+        kuery?: string;
+      }) {
+        const response = await apmApiClient.readUser({
+          endpoint: 'GET /api/apm/services/{serviceName}/throughput',
+          params: {
+            path: {
+              serviceName: 'synth-go',
+            },
+            query: {
+              start: new Date(start).toISOString(),
+              end: new Date(end).toISOString(),
+              transactionType: 'request',
+              environment: 'production',
+              kuery: 'processor.event:transaction',
+              ...overrides,
+            },
+          },
+        });
+
+        return response.body;
+      }
+
+      describe('when calling it with the default parameters', () => {
+        let body: PromiseReturnType<typeof callApi>;
+
+        before(async () => {
+          body = await callApi();
+        });
+
+        it('returns the throughput in seconds', () => {
+          expect(body.throughputUnit).to.eql('second');
+        });
+
+        it('returns the expected throughput', () => {
+          const throughputValues = uniq(body.currentPeriod.map((coord) => coord.y));
+          expect(throughputValues).to.eql([GO_PROD_RATE]);
+        });
+      });
+
+      describe('when setting environment to all', () => {
+        let body: PromiseReturnType<typeof callApi>;
+
+        before(async () => {
+          body = await callApi({
+            environment: ENVIRONMENT_ALL.value,
+          });
+        });
+
+        it('returns data for all environments', () => {
+          const throughputValues = body.currentPeriod.map(({ y }) => y);
+          expect(uniq(throughputValues)).to.eql([GO_PROD_RATE + GO_DEV_RATE]);
+          expect(body.throughputUnit).to.eql('second');
+        });
+      });
+
+      describe('when defining a kuery', () => {
+        let body: PromiseReturnType<typeof callApi>;
+
+        before(async () => {
+          body = await callApi({
+            kuery: `processor.event:transaction and transaction.name:"GET /api/product/:id"`,
+            environment: ENVIRONMENT_ALL.value,
+          });
+        });
+
+        it('returns data that matches the kuery', () => {
+          const throughputValues = body.currentPeriod.map(({ y }) => y);
+          expect(uniq(throughputValues)).to.eql([GO_DEV_RATE]);
+          expect(body.throughputUnit).to.eql('second');
+        });
+      });
+    }
+  );
+
   registry.when('Throughput when data is not loaded', { config: 'basic', archives: [] }, () => {
     it('handles the empty state', async () => {
       const response = await apmApiClient.readUser({

yarn.lock

@@ -2267,6 +2267,10 @@
     is-absolute "^1.0.0"
     is-negated-glob "^1.0.0"
 
+"@elastic/apm-generator@link:bazel-bin/packages/elastic-apm-generator":
+  version "0.0.0"
+  uid ""
+
 "@elastic/apm-rum-core@^5.12.1":
   version "5.12.1"
   resolved "https://registry.yarnpkg.com/@elastic/apm-rum-core/-/apm-rum-core-5.12.1.tgz#ad78787876c68b9ce718d1c42b8e7b12b12eaa69"