[APM] Synthtrace: Add distributed trace helper (#142593)

This commit is contained in:
Søren Louv-Jansen 2022-10-12 09:11:13 +02:00 committed by GitHub
parent 8280532300
commit 005a6eabb5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 638 additions and 35 deletions

View file

@ -58,6 +58,8 @@ export type ApmFields = Fields &
'error.grouping_key': string;
'host.name': string;
'host.hostname': string;
'http.request.method': string;
'http.response.status_code': number;
'kubernetes.pod.uid': string;
'kubernetes.pod.name': string;
'metricset.name': string;
@ -84,14 +86,13 @@ export type ApmFields = Fields &
'service.framework.name': string;
'service.target.name': string;
'service.target.type': string;
'span.action': string;
'span.id': string;
'span.name': string;
'span.type': string;
'span.subtype': string;
'span.duration.us': number;
'span.destination.service.name': string;
'span.destination.service.resource': string;
'span.destination.service.type': string;
'span.destination.service.response_time.sum.us': number;
'span.destination.service.response_time.count': number;
'span.self_time.count': number;

View file

@ -40,6 +40,10 @@ export class BaseSpan extends Serializable<ApmFields> {
return this;
}
getChildren() {
return this._children;
}
children(...children: BaseSpan[]): this {
children.forEach((child) => {
child.parent(this);

View file

@ -148,9 +148,7 @@ export class ApmSynthtraceApmClient {
const destination = (e.context.destination = e.context.destination ?? {});
const destinationService = (destination.service = destination.service ?? { resource: '' });
set('span.destination.service.name', destinationService, (c, v) => (c.name = v));
set('span.destination.service.resource', destinationService, (c, v) => (c.resource = v));
set('span.destination.service.type', destinationService, (c, v) => (c.type = v));
}
if (e.kind === 'transaction') {
set('transaction.name', e, (c, v) => (c.name = v));

View file

@ -13,6 +13,12 @@ import { Span } from './span';
import { Transaction } from './transaction';
import { ApmApplicationMetricFields, ApmFields } from './apm_fields';
export type SpanParams = {
spanName: string;
spanType: string;
spanSubtype?: string;
} & ApmFields;
export class Instance extends Entity<ApmFields> {
transaction({
transactionName,
@ -28,16 +34,7 @@ export class Instance extends Entity<ApmFields> {
});
}
span({
spanName,
spanType,
spanSubtype,
...apmFields
}: {
spanName: string;
spanType: string;
spanSubtype?: string;
} & ApmFields) {
span({ spanName, spanType, spanSubtype, ...apmFields }: SpanParams) {
return new Span({
...this.fields,
...apmFields,

View file

@ -10,6 +10,7 @@ import url from 'url';
import { BaseSpan } from './base_span';
import { generateShortId } from '../utils/generate_id';
import { ApmFields } from './apm_fields';
import { SpanParams } from './instance';
export class Span extends BaseSpan {
constructor(fields: ApmFields) {
@ -25,29 +26,26 @@ export class Span extends BaseSpan {
return this;
}
destination(resource: string, type?: string, name?: string) {
if (!type) {
type = this.fields['span.type'];
}
if (!name) {
name = resource;
}
destination(resource: string) {
this.fields['span.destination.service.resource'] = resource;
this.fields['span.destination.service.name'] = name;
this.fields['span.destination.service.type'] = type;
return this;
}
}
export type HttpMethod = 'GET' | 'POST' | 'DELETE' | 'PUT';
export function httpExitSpan({
spanName,
destinationUrl,
method = 'GET',
statusCode = 200,
}: {
spanName: string;
destinationUrl: string;
}) {
method?: HttpMethod;
statusCode?: number;
}): SpanParams {
// origin: 'http://opbeans-go:3000',
// host: 'opbeans-go:3000',
// hostname: 'opbeans-go',
@ -55,31 +53,98 @@ export function httpExitSpan({
const destination = new url.URL(destinationUrl);
const spanType = 'external';
const spanSubType = 'http';
const spanSubtype = 'http';
return {
spanName,
spanType,
spanSubType,
spanSubtype,
// http
'span.action': method,
'http.request.method': method,
'http.response.status_code': statusCode,
// destination
'destination.address': destination.hostname,
'destination.port': parseInt(destination.port, 10),
'service.target.name': destination.host,
'span.destination.service.name': destination.origin,
'span.destination.service.resource': destination.host,
'span.destination.service.type': 'external',
};
}
export function dbExitSpan({ spanName, spanSubType }: { spanName: string; spanSubType?: string }) {
export function dbExitSpan({ spanName, spanSubtype }: { spanName: string; spanSubtype?: string }) {
const spanType = 'db';
return {
spanName,
spanType,
spanSubType,
'service.target.type': spanSubType,
'span.destination.service.name': spanSubType,
'span.destination.service.resource': spanSubType,
'span.destination.service.type': spanType,
spanSubtype,
'service.target.type': spanSubtype,
'span.destination.service.resource': spanSubtype,
};
}
export function elasticsearchSpan(spanName: string, statement?: string): SpanParams {
const spanType = 'db';
const spanSubtype = 'elasticsearch';
return {
spanName,
spanType,
spanSubtype,
...(statement
? {
'span.db.statement': statement,
'span.db.type': 'elasticsearch',
}
: {}),
'service.target.type': spanSubtype,
'destination.address': 'qwerty.us-west2.gcp.elastic-cloud.com',
'destination.port': 443,
'span.destination.service.resource': spanSubtype,
};
}
export function sqliteSpan(spanName: string, statement?: string): SpanParams {
const spanType = 'db';
const spanSubtype = 'sqlite';
return {
spanName,
spanType,
spanSubtype,
...(statement
? {
'span.db.statement': statement,
'span.db.type': 'sql',
}
: {}),
// destination
'service.target.type': spanSubtype,
'destination.address': 'qwerty.us-west2.gcp.elastic-cloud.com',
'destination.port': 443,
'span.destination.service.resource': spanSubtype,
};
}
export function redisSpan(spanName: string): SpanParams {
const spanType = 'db';
const spanSubtype = 'redis';
return {
spanName,
spanType,
spanSubtype,
// destination
'service.target.type': spanSubtype,
'destination.address': 'qwerty.us-west2.gcp.elastic-cloud.com',
'destination.port': 443,
'span.destination.service.resource': spanSubtype,
};
}

View file

@ -0,0 +1,221 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { apm } from '../apm';
import { ApmFields } from '../apm/apm_fields';
import { BaseSpan } from '../apm/base_span';
import { DistributedTrace } from './distributed_trace_client';
const opbeansRum = apm
.service({ name: 'opbeans-rum', environment: 'prod', agentName: 'rum-js' })
.instance('my-instance');
const opbeansNode = apm
.service({ name: 'opbeans-node', environment: 'prod', agentName: 'nodejs' })
.instance('my-instance');
const opbeansGo = apm
.service({ name: 'opbeans-go', environment: 'prod', agentName: 'go' })
.instance('my-instance');
describe('DistributedTrace', () => {
describe('basic scenario', () => {
it('should add latency', () => {
const dt = new DistributedTrace({
serviceInstance: opbeansRum,
transactionName: 'Dashboard',
timestamp: 0,
children: (s) => {
s.service({
serviceInstance: opbeansNode,
transactionName: 'GET /nodejs/products',
children: (_) => {
_.service({ serviceInstance: opbeansGo, transactionName: 'GET /gogo' });
_.db({ name: 'GET apm-*/_search', type: 'elasticsearch', duration: 400 });
},
});
},
}).getTransaction();
const traceDocs = getTraceDocs(dt);
const formattedDocs = traceDocs.map((f) => {
return {
processorEvent: f['processor.event'],
timestamp: f['@timestamp'],
duration: (f['transaction.duration.us'] ?? f['span.duration.us'])! / 1000,
name: f['transaction.name'] ?? f['span.name'],
};
});
expect(formattedDocs).toMatchInlineSnapshot(`
Array [
Object {
"duration": 400,
"name": "Dashboard",
"processorEvent": "transaction",
"timestamp": 0,
},
Object {
"duration": 400,
"name": "GET /nodejs/products",
"processorEvent": "span",
"timestamp": 0,
},
Object {
"duration": 400,
"name": "GET /nodejs/products",
"processorEvent": "transaction",
"timestamp": 0,
},
Object {
"duration": 0,
"name": "GET /gogo",
"processorEvent": "span",
"timestamp": 0,
},
Object {
"duration": 0,
"name": "GET /gogo",
"processorEvent": "transaction",
"timestamp": 0,
},
Object {
"duration": 400,
"name": "GET apm-*/_search",
"processorEvent": "span",
"timestamp": 0,
},
]
`);
});
});
describe('latency', () => {
it('should add latency', () => {
const traceDocs = getSimpleScenario({ latency: 500 });
const timestamps = traceDocs.map((f) => f['@timestamp']);
expect(timestamps).toMatchInlineSnapshot(`
Array [
0,
0,
250,
250,
250,
250,
]
`);
});
it('should not add latency', () => {
const traceDocs = getSimpleScenario();
const timestamps = traceDocs.map((f) => f['@timestamp']);
expect(timestamps).toMatchInlineSnapshot(`
Array [
0,
0,
0,
0,
0,
0,
]
`);
});
});
describe('duration', () => {
it('should add duration', () => {
const traceDocs = getSimpleScenario({ duration: 3000 });
const durations = traceDocs.map(
(f) => (f['transaction.duration.us'] ?? f['span.duration.us'])! / 1000
);
expect(durations).toMatchInlineSnapshot(`
Array [
3000,
3000,
3000,
300,
400,
500,
]
`);
});
it('should not add duration', () => {
const traceDocs = getSimpleScenario();
const durations = traceDocs.map(
(f) => (f['transaction.duration.us'] ?? f['span.duration.us'])! / 1000
);
expect(durations).toMatchInlineSnapshot(`
Array [
500,
500,
500,
300,
400,
500,
]
`);
});
});
describe('repeat', () => {
it('produces few trace documents when "repeat" is disabled', () => {
const traceDocs = getSimpleScenario({ repeat: undefined });
expect(traceDocs.length).toBe(6);
});
it('produces more trace documents when "repeat" is enabled', () => {
const traceDocs = getSimpleScenario({ repeat: 20 });
expect(traceDocs.length).toBe(101);
});
});
});
function getTraceDocs(transaction: BaseSpan): ApmFields[] {
const children = transaction.getChildren();
if (children) {
const childFields = children.flatMap((child) => getTraceDocs(child));
return [transaction.fields, ...childFields];
}
return [transaction.fields];
}
function getSimpleScenario({
duration,
latency,
repeat,
}: {
duration?: number;
latency?: number;
repeat?: number;
} = {}) {
const dt = new DistributedTrace({
serviceInstance: opbeansRum,
transactionName: 'Dashboard',
timestamp: 0,
children: (s) => {
s.service({
serviceInstance: opbeansNode,
transactionName: 'GET /nodejs/products',
duration,
latency,
repeat,
children: (_) => {
_.db({ name: 'GET apm-*/_search', type: 'elasticsearch', duration: 300 });
_.db({ name: 'GET apm-*/_search', type: 'elasticsearch', duration: 400 });
_.db({ name: 'GET apm-*/_search', type: 'elasticsearch', duration: 500 });
},
});
},
}).getTransaction();
return getTraceDocs(dt);
}

View file

@ -0,0 +1,183 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { times } from 'lodash';
import { elasticsearchSpan, httpExitSpan, HttpMethod, redisSpan, sqliteSpan } from '../apm/span';
import { BaseSpan } from '../apm/base_span';
import { Instance, SpanParams } from '../apm/instance';
import { Transaction } from '../apm/transaction';
export class DistributedTrace {
timestamp: number;
serviceInstance: Instance;
spanEndTimes: number[] = [];
childSpans: BaseSpan[] = [];
transaction: Transaction;
constructor({
serviceInstance,
transactionName,
timestamp,
children,
}: {
serviceInstance: Instance;
transactionName: string;
timestamp: number;
children?: (dt: DistributedTrace) => void;
}) {
this.timestamp = timestamp;
this.serviceInstance = serviceInstance;
if (children) {
children(this);
}
const maxEndTime = Math.max(...this.spanEndTimes);
const duration = maxEndTime - this.timestamp;
this.transaction = serviceInstance
.transaction({ transactionName })
.timestamp(timestamp)
.duration(duration)
.children(...this.childSpans);
return this;
}
getTransaction() {
return this.transaction;
}
service({
serviceInstance,
transactionName,
latency = 0,
repeat = 1,
timestamp = this.timestamp,
duration,
children,
}: {
serviceInstance: Instance;
transactionName: string;
repeat?: number;
timestamp?: number;
latency?: number;
duration?: number;
children?: (dt: DistributedTrace) => unknown;
}) {
const originServiceInstance = this.serviceInstance;
times(repeat, () => {
const dt = new DistributedTrace({
serviceInstance,
transactionName,
timestamp: timestamp + latency / 2,
children,
});
const maxSpanEndTime = Math.max(...dt.spanEndTimes, timestamp + (duration ?? 0));
this.spanEndTimes.push(maxSpanEndTime + latency / 2);
// origin service
const exitSpanStart = timestamp;
const exitSpanDuration = (duration ?? maxSpanEndTime - exitSpanStart) + latency / 2;
// destination service
const transactionStart = timestamp + latency / 2;
const transactionDuration = duration ?? maxSpanEndTime - transactionStart;
const span = originServiceInstance
.span(
httpExitSpan({
spanName: transactionName,
destinationUrl: 'http://api-gateway:3000', // TODO: this should be derived from serviceInstance
})
)
.duration(exitSpanDuration)
.timestamp(exitSpanStart)
.children(
dt.serviceInstance
.transaction({ transactionName })
.timestamp(transactionStart)
.duration(transactionDuration)
.children(...(dt.childSpans ?? []))
);
this.childSpans.push(span);
});
}
external({
name,
url,
method,
statusCode,
duration,
timestamp = this.timestamp,
}: {
name: string;
url: string;
method?: HttpMethod;
statusCode?: number;
duration: number;
timestamp?: number;
}) {
const startTime = timestamp;
const endTime = startTime + duration;
this.spanEndTimes.push(endTime);
const span = this.serviceInstance
.span(httpExitSpan({ spanName: name, destinationUrl: url, method, statusCode }))
.timestamp(startTime)
.duration(duration)
.success();
this.childSpans.push(span);
}
db({
name,
duration,
type,
statement,
timestamp = this.timestamp,
}: {
name: string;
duration: number;
type: 'elasticsearch' | 'sqlite' | 'redis';
statement?: string;
timestamp?: number;
}) {
const startTime = timestamp;
const endTime = startTime + duration;
this.spanEndTimes.push(endTime);
let dbSpan: SpanParams;
switch (type) {
case 'elasticsearch':
dbSpan = elasticsearchSpan(name, statement);
break;
case 'sqlite':
dbSpan = sqliteSpan(name, statement);
break;
case 'redis':
dbSpan = redisSpan(name);
break;
}
const span = this.serviceInstance
.span(dbSpan)
.timestamp(startTime)
.duration(duration)
.success();
this.childSpans.push(span);
}
}

View file

@ -0,0 +1,134 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
/* eslint-disable @typescript-eslint/no-shadow */
import { apm, timerange } from '../..';
import { ApmFields } from '../lib/apm/apm_fields';
import { Scenario } from '../cli/scenario';
import { RunOptions } from '../cli/utils/parse_run_cli_flags';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { DistributedTrace } from '../lib/dsl/distributed_trace_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
return {
generate: ({ from, to }) => {
const ratePerMinute = 1;
const traceDuration = 1100;
const rootTransactionName = `${ratePerMinute}rpm / ${traceDuration}ms`;
const opbeansRum = apm
.service({ name: 'opbeans-rum', environment: ENVIRONMENT, agentName: 'rum-js' })
.instance('my-instance');
const opbeansNode = apm
.service({ name: 'opbeans-node', environment: ENVIRONMENT, agentName: 'nodejs' })
.instance('my-instance');
const opbeansGo = apm
.service({ name: 'opbeans-go', environment: ENVIRONMENT, agentName: 'go' })
.instance('my-instance');
const opbeansDotnet = apm
.service({ name: 'opbeans-dotnet', environment: ENVIRONMENT, agentName: 'dotnet' })
.instance('my-instance');
const opbeansJava = apm
.service({ name: 'opbeans-java', environment: ENVIRONMENT, agentName: 'java' })
.instance('my-instance');
const traces = timerange(from, to)
.ratePerMinute(ratePerMinute)
.generator((timestamp) => {
return new DistributedTrace({
serviceInstance: opbeansRum,
transactionName: rootTransactionName,
timestamp,
children: (_) => {
_.service({
repeat: 10,
serviceInstance: opbeansNode,
transactionName: 'GET /nodejs/products',
latency: 100,
children: (_) => {
_.service({
serviceInstance: opbeansGo,
transactionName: 'GET /go',
children: (_) => {
_.service({
repeat: 20,
serviceInstance: opbeansJava,
transactionName: 'GET /java',
children: (_) => {
_.external({
name: 'GET telemetry.elastic.co',
url: 'https://telemetry.elastic.co/ping',
duration: 50,
});
},
});
},
});
_.db({ name: 'GET apm-*/_search', type: 'elasticsearch', duration: 400 });
_.db({ name: 'GET', type: 'redis', duration: 500 });
_.db({ name: 'SELECT * FROM users', type: 'sqlite', duration: 600 });
},
});
_.service({
serviceInstance: opbeansNode,
transactionName: 'GET /nodejs/users',
latency: 100,
repeat: 10,
children: (_) => {
_.service({
serviceInstance: opbeansGo,
transactionName: 'GET /go/security',
latency: 50,
children: (_) => {
_.service({
repeat: 10,
serviceInstance: opbeansDotnet,
transactionName: 'GET /dotnet/cases/4',
latency: 50,
children: (_) =>
_.db({
name: 'GET apm-*/_search',
type: 'elasticsearch',
duration: 600,
statement: JSON.stringify(
{
query: {
query_string: {
query: '(new york city) OR (big apple)',
default_field: 'content',
},
},
},
null,
2
),
}),
});
},
});
},
});
},
}).getTransaction();
});
return traces;
},
};
};
export default scenario;