[APM] Adds service map dsl to synthtrace (#152526)

Adds `serviceMap` helper to generate transaction and spans in synthtrace
by defining a set of traces from which a service map can be rendered.
This can be considered a follow-up to
https://github.com/elastic/kibana/pull/149900 where synthtrace was used
to generate service maps for api integration tests.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Oliver Gupte 2023-03-02 09:49:14 -05:00 committed by GitHub
parent 5250d859f4
commit 35008c955e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 548 additions and 123 deletions

View file

@ -20,6 +20,7 @@ export type {
} from './src/lib/apm/mobile_device';
export { httpExitSpan } from './src/lib/apm/span';
export { DistributedTrace } from './src/lib/dsl/distributed_trace_client';
export { serviceMap } from './src/lib/dsl/service_map';
export type { Fields } from './src/lib/entity';
export type { Serializable } from './src/lib/serializable';
export { timerange } from './src/lib/timerange';

View file

@ -0,0 +1,278 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { pick } from 'lodash';
import { ApmFields } from '../apm/apm_fields';
import { BaseSpan } from '../apm/base_span';
import { serviceMap, ServiceMapOpts } from './service_map';
describe('serviceMap', () => {
const TIMESTAMP = 1677693600000;
describe('Basic definition', () => {
const BASIC_SERVICE_MAP_OPTS: ServiceMapOpts = {
services: [
'frontend-rum',
'frontend-node',
'advertService',
'checkoutService',
'cartService',
'paymentService',
'productCatalogService',
],
definePaths([rum, node, adv, chk, cart, pay, prod]) {
return [
[rum, node, adv, 'elasticsearch'],
[rum, node, cart, 'redis'],
[rum, node, chk, pay],
[chk, cart, 'redis'],
[rum, node, prod, 'elasticsearch'],
[chk, prod],
];
},
};
it('should create an accurate set of trace paths', () => {
const serviceMapGenerator = serviceMap(BASIC_SERVICE_MAP_OPTS);
const transactions = serviceMapGenerator(TIMESTAMP);
expect(transactions.map(getTracePathLabel)).toMatchInlineSnapshot(`
Array [
"frontend-rum → frontend-node → advertService → elasticsearch",
"frontend-rum → frontend-node → cartService → redis",
"frontend-rum → frontend-node → checkoutService → paymentService",
"checkoutService → cartService → redis",
"frontend-rum → frontend-node → productCatalogService → elasticsearch",
"checkoutService → productCatalogService",
]
`);
});
it('should use a default agent name if not defined', () => {
const serviceMapGenerator = serviceMap(BASIC_SERVICE_MAP_OPTS);
const transactions = serviceMapGenerator(TIMESTAMP);
const traceDocs = transactions.flatMap(getTraceDocsSubset);
for (const doc of traceDocs) {
expect(doc).toHaveProperty(['agent.name'], 'nodejs');
}
});
it('should use a default transaction/span names if not defined', () => {
const serviceMapGenerator = serviceMap(BASIC_SERVICE_MAP_OPTS);
const transactions = serviceMapGenerator(TIMESTAMP);
const traceDocs = transactions.map(getTraceDocsSubset);
for (let i = 0; i < traceDocs.length; i++) {
for (const doc of traceDocs[i]) {
const serviceName = doc['service.name'];
if (doc['processor.event'] === 'transaction') {
expect(doc).toHaveProperty(['transaction.name'], `GET /api/${serviceName}/${i}`);
}
if (doc['processor.event'] === 'span') {
if (doc['span.type'] === 'db') {
switch (doc['span.subtype']) {
case 'elasticsearch':
expect(doc).toHaveProperty(['span.name'], `GET ad-*/_search`);
break;
case 'redis':
expect(doc).toHaveProperty(['span.name'], `INCR item:i012345:count`);
break;
case 'sqlite':
expect(doc).toHaveProperty(['span.name'], `SELECT * FROM items`);
break;
}
} else {
expect(doc).toHaveProperty(['span.name'], `GET /api/${serviceName}/${i}`);
}
}
}
}
});
it('should create one parent transaction per trace', () => {
const serviceMapGenerator = serviceMap(BASIC_SERVICE_MAP_OPTS);
const transactions = serviceMapGenerator(TIMESTAMP);
const traces = transactions.map(getTraceDocsSubset);
for (const traceDocs of traces) {
const [transaction, ...spans] = traceDocs;
expect(transaction).toHaveProperty(['processor.event'], 'transaction');
expect(
spans.every(({ 'processor.event': processorEvent }) => processorEvent === 'span')
).toBe(true);
}
});
});
describe('Detailed definition', () => {
const DETAILED_SERVICE_MAP_OPTS: ServiceMapOpts = {
services: [
{ 'frontend-rum': 'rum-js' },
{ 'frontend-node': 'nodejs' },
{ advertService: 'java' },
{ checkoutService: 'go' },
{ cartService: 'dotnet' },
{ paymentService: 'nodejs' },
{ productCatalogService: 'go' },
],
definePaths([rum, node, adv, chk, cart, pay, prod]) {
return [
[
[rum, 'fetchAd'],
[node, 'GET /nodejs/adTag'],
[adv, 'APIRestController#getAd'],
['elasticsearch', 'GET ad-*/_search'],
],
[
[rum, 'AddToCart'],
[node, 'POST /nodejs/addToCart'],
[cart, 'POST /dotnet/reserveProduct'],
['redis', 'DECR inventory:i012345:stock'],
],
{
path: [
[rum, 'Checkout'],
[node, 'POST /nodejs/placeOrder'],
[chk, 'POST /go/placeOrder'],
[pay, 'POST /nodejs/processPayment'],
],
transaction: (t) => t.defaults({ 'labels.name': 'transaction hook test' }),
},
[
[chk, 'POST /go/clearCart'],
[cart, 'PUT /dotnet/cart/c12345/reset'],
['redis', 'INCR inventory:i012345:stock'],
],
[
[rum, 'ProductDashboard'],
[node, 'GET /nodejs/products'],
[prod, 'GET /go/product-catalog'],
['elasticsearch', 'GET product-*/_search'],
],
[
[chk, 'PUT /go/update-inventory'],
[prod, 'PUT /go/product/i012345'],
],
[pay],
];
},
};
const SERVICE_AGENT_MAP: Record<string, string> = {
'frontend-rum': 'rum-js',
'frontend-node': 'nodejs',
advertService: 'java',
checkoutService: 'go',
cartService: 'dotnet',
paymentService: 'nodejs',
productCatalogService: 'go',
};
it('should use the defined agent name for a given service', () => {
const serviceMapGenerator = serviceMap(DETAILED_SERVICE_MAP_OPTS);
const transactions = serviceMapGenerator(TIMESTAMP);
const traceDocs = transactions.flatMap(getTraceDocsSubset);
for (const doc of traceDocs) {
if (!(doc['service.name']! in SERVICE_AGENT_MAP)) {
throw new Error(`Unexpected service name '${doc['service.name']}' found`);
}
expect(doc).toHaveProperty(['agent.name'], SERVICE_AGENT_MAP[doc['service.name']!]);
}
});
it('should use the defined transaction/span names for each trace document', () => {
const serviceMapGenerator = serviceMap(DETAILED_SERVICE_MAP_OPTS);
const transactions = serviceMapGenerator(TIMESTAMP);
const traceDocs = transactions.map((transaction) => {
return getTraceDocsSubset(transaction).map(
({ 'span.name': spanName, 'transaction.name': transactionName }) =>
transactionName || spanName
);
});
expect(traceDocs).toMatchInlineSnapshot(`
Array [
Array [
"fetchAd",
"fetchAd",
"GET /nodejs/adTag",
"APIRestController#getAd",
"GET ad-*/_search",
],
Array [
"AddToCart",
"AddToCart",
"POST /nodejs/addToCart",
"POST /dotnet/reserveProduct",
"DECR inventory:i012345:stock",
],
Array [
"Checkout",
"Checkout",
"POST /nodejs/placeOrder",
"POST /go/placeOrder",
"POST /nodejs/processPayment",
],
Array [
"POST /go/clearCart",
"POST /go/clearCart",
"PUT /dotnet/cart/c12345/reset",
"INCR inventory:i012345:stock",
],
Array [
"ProductDashboard",
"ProductDashboard",
"GET /nodejs/products",
"GET /go/product-catalog",
"GET product-*/_search",
],
Array [
"PUT /go/update-inventory",
"PUT /go/update-inventory",
"PUT /go/product/i012345",
],
Array [
"GET /api/paymentService/6",
"GET /api/paymentService/6",
],
]
`);
});
it('should apply the transaction hook function if defined', () => {
const serviceMapGenerator = serviceMap(DETAILED_SERVICE_MAP_OPTS);
const transactions = serviceMapGenerator(TIMESTAMP);
expect(transactions[2].fields['labels.name']).toBe('transaction hook test');
});
});
});
function getTraceDocsSubset(transaction: BaseSpan): ApmFields[] {
const subsetFields = pick(transaction.fields, [
'processor.event',
'service.name',
'agent.name',
'transaction.name',
'span.name',
'span.type',
'span.subtype',
'span.destination.service.resource',
]);
const children = transaction.getChildren();
if (children) {
const childFields = children.flatMap((child) => getTraceDocsSubset(child));
return [subsetFields, ...childFields];
}
return [subsetFields];
}
function getTracePathLabel(transaction: BaseSpan) {
const traceDocs = getTraceDocsSubset(transaction);
const traceSpans = traceDocs.filter((doc) => doc['processor.event'] === 'span');
const spanLabels = traceSpans.map((span) =>
span['span.type'] === 'db' ? span['span.subtype'] : span['service.name']
);
return spanLabels.join(' → ');
}

View file

@ -0,0 +1,156 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { AgentName } from '../../types/agent_names';
import { apm } from '../apm';
import { Instance } from '../apm/instance';
import { elasticsearchSpan, redisSpan, sqliteSpan, Span } from '../apm/span';
import { Transaction } from '../apm/transaction';
const ENVIRONMENT = 'Synthtrace: service_map';
function service(serviceName: string, agentName: AgentName, environment?: string) {
return apm
.service({ name: serviceName, environment: environment || ENVIRONMENT, agentName })
.instance(serviceName);
}
type DbSpan = 'elasticsearch' | 'redis' | 'sqlite';
type ServiceMapNode = Instance | DbSpan;
type TransactionName = string;
type TraceItem = ServiceMapNode | [ServiceMapNode, TransactionName];
type TracePath = TraceItem[];
function getTraceItem(traceItem: TraceItem) {
if (Array.isArray(traceItem)) {
const transactionName = traceItem[1];
if (typeof traceItem[0] === 'string') {
const dbSpan = traceItem[0];
return { dbSpan, transactionName, serviceInstance: undefined };
} else {
const serviceInstance = traceItem[0];
return { dbSpan: undefined, transactionName, serviceInstance };
}
} else if (typeof traceItem === 'string') {
const dbSpan = traceItem;
return { dbSpan, transactionName: undefined, serviceInstance: undefined };
} else {
const serviceInstance = traceItem;
return { dbSpan: undefined, transactionName: undefined, serviceInstance };
}
}
function getTransactionName(
transactionName: string | undefined,
serviceInstance: Instance,
index: number
) {
return transactionName || `GET /api/${serviceInstance.fields['service.name']}/${index}`;
}
function getChildren(
childTraceItems: TracePath,
parentServiceInstance: Instance,
timestamp: number,
index: number
): Span[] {
if (childTraceItems.length === 0) {
return [];
}
const [first, ...rest] = childTraceItems;
const { dbSpan, serviceInstance, transactionName } = getTraceItem(first);
if (dbSpan) {
switch (dbSpan) {
case 'elasticsearch':
return [
parentServiceInstance
.span(elasticsearchSpan(transactionName || 'GET ad-*/_search'))
.timestamp(timestamp)
.duration(1000),
];
case 'redis':
return [
parentServiceInstance
.span(redisSpan(transactionName || 'INCR item:i012345:count'))
.timestamp(timestamp)
.duration(1000),
];
case 'sqlite':
return [
parentServiceInstance
.span(sqliteSpan(transactionName || 'SELECT * FROM items'))
.timestamp(timestamp)
.duration(1000),
];
}
}
const childSpan = serviceInstance
.span({
spanName: getTransactionName(transactionName, serviceInstance, index),
spanType: 'app',
})
.timestamp(timestamp)
.duration(1000)
.children(...getChildren(rest, serviceInstance, timestamp, index));
if (rest[0]) {
const next = getTraceItem(rest[0]);
if (next.serviceInstance) {
return [childSpan.destination(next.serviceInstance.fields['service.name']!)];
}
}
return [childSpan];
}
interface TracePathOpts {
path: TracePath;
transaction?: (transaction: Transaction) => Transaction;
}
type PathDef = TracePath | TracePathOpts;
export interface ServiceMapOpts {
services: Array<string | { [serviceName: string]: AgentName }>;
definePaths: (services: Instance[]) => PathDef[];
environment?: string;
}
export function serviceMap(options: ServiceMapOpts) {
const serviceInstances = options.services.map((s) => {
if (typeof s === 'string') {
return service(s, 'nodejs', options.environment);
}
return service(Object.keys(s)[0], Object.values(s)[0], options.environment);
});
return (timestamp: number) => {
const tracePaths = options.definePaths(serviceInstances);
return tracePaths.map((traceDef, index) => {
const tracePath = 'path' in traceDef ? traceDef.path : traceDef;
const [first] = tracePath;
const firstTraceItem = getTraceItem(first);
if (firstTraceItem.serviceInstance === undefined) {
throw new Error('First trace item must be a service instance');
}
const transactionName = getTransactionName(
firstTraceItem.transactionName,
firstTraceItem.serviceInstance,
index
);
const transaction = firstTraceItem.serviceInstance
.transaction({ transactionName, transactionType: 'request' })
.timestamp(timestamp)
.duration(1000)
.children(...getChildren(tracePath, firstTraceItem.serviceInstance, timestamp, index));
if ('transaction' in traceDef && traceDef.transaction) {
return traceDef.transaction(transaction);
}
return transaction;
});
};
}

View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
type ElasticAgentName =
| 'go'
| 'java'
| 'js-base'
| 'iOS/swift'
| 'rum-js'
| 'nodejs'
| 'python'
| 'dotnet'
| 'ruby'
| 'php'
| 'android/java';
type OpenTelemetryAgentName =
| 'otlp'
| 'opentelemetry/cpp'
| 'opentelemetry/dotnet'
| 'opentelemetry/erlang'
| 'opentelemetry/go'
| 'opentelemetry/java'
| 'opentelemetry/nodejs'
| 'opentelemetry/php'
| 'opentelemetry/python'
| 'opentelemetry/ruby'
| 'opentelemetry/swift'
| 'opentelemetry/webjs';
// Unable to reference AgentName from '@kbn/apm-plugin/typings/es_schemas/ui/fields/agent' due to circular reference
export type AgentName = ElasticAgentName | OpenTelemetryAgentName;

View file

@ -6,116 +6,71 @@
* Side Public License, v 1.
*/
import { apm, ApmFields, Instance } from '@kbn/apm-synthtrace-client';
import { Transaction } from '@kbn/apm-synthtrace-client/src/lib/apm/transaction';
import { AgentName } from '@kbn/apm-plugin/typings/es_schemas/ui/fields/agent';
import { ApmFields, serviceMap } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { RunOptions } from '../cli/utils/parse_run_cli_flags';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
function generateTrace(
timestamp: number,
transactionName: string,
order: Instance[],
db?: 'elasticsearch' | 'redis'
) {
return order
.concat()
.reverse()
.reduce<Transaction | undefined>((prev, instance, index) => {
const invertedIndex = order.length - index - 1;
const duration = 50;
const time = timestamp + invertedIndex * 10;
const transaction: Transaction = instance
.transaction({ transactionName })
.timestamp(time)
.duration(duration);
if (prev) {
const next = order[invertedIndex + 1].fields['service.name']!;
transaction.children(
instance
.span({ spanName: `GET ${next}/api`, spanType: 'external', spanSubtype: 'http' })
.destination(next)
.duration(duration)
.timestamp(time + 1)
.children(prev)
);
} else if (db) {
transaction.children(
instance
.span({ spanName: db, spanType: 'db', spanSubtype: db })
.destination(db)
.duration(duration)
.timestamp(time + 1)
);
}
return transaction;
}, undefined)!;
}
function service(serviceName: string, agentName: AgentName) {
return apm
.service({ name: serviceName, environment: ENVIRONMENT, agentName })
.instance(serviceName);
}
const environment = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
return {
generate: ({ range }) => {
const frontendRum = service('frontend-rum', 'rum-js');
const frontendNode = service('frontend-node', 'nodejs');
const advertService = service('advertService', 'java');
const checkoutService = service('checkoutService', 'go');
const cartService = service('cartService', 'dotnet');
const paymentService = service('paymentService', 'nodejs');
const productCatalogService = service('productCatalogService', 'go');
return range
.interval('1s')
.rate(3)
.generator((timestamp) => {
return [
generateTrace(
timestamp,
'GET /api/adTag',
[frontendRum, frontendNode, advertService],
'elasticsearch'
),
generateTrace(
timestamp,
'POST /api/addToCart',
[frontendRum, frontendNode, cartService],
'redis'
),
generateTrace(timestamp, 'POST /api/checkout', [
frontendRum,
frontendNode,
checkoutService,
paymentService,
]),
generateTrace(
timestamp,
'DELETE /api/clearCart',
[checkoutService, cartService],
'redis'
),
generateTrace(
timestamp,
'GET /api/products',
[frontendRum, frontendNode, productCatalogService],
'elasticsearch'
),
generateTrace(timestamp, 'PUT /api/updateInventory', [
checkoutService,
productCatalogService,
]),
];
});
.generator(
serviceMap({
services: [
{ 'frontend-rum': 'rum-js' },
{ 'frontend-node': 'nodejs' },
{ advertService: 'java' },
{ checkoutService: 'go' },
{ cartService: 'dotnet' },
{ paymentService: 'nodejs' },
{ productCatalogService: 'go' },
],
environment,
definePaths([rum, node, adv, chk, cart, pay, prod]) {
return [
[
[rum, 'fetchAd'],
[node, 'GET /nodejs/adTag'],
[adv, 'APIRestController#getAd'],
['elasticsearch', 'GET ad-*/_search'],
],
[
[rum, 'AddToCart'],
[node, 'POST /nodejs/addToCart'],
[cart, 'POST /dotnet/reserveProduct'],
['redis', 'DECR inventory:i012345:stock'],
],
[
[rum, 'Checkout'],
[node, 'POST /nodejs/placeOrder'],
[chk, 'POST /go/placeOrder'],
[pay, 'POST /nodejs/processPayment'],
],
[
[chk, 'POST /go/clearCart'],
[cart, 'PUT /dotnet/cart/c12345/reset'],
['redis', 'INCR inventory:i012345:stock'],
],
[
[rum, 'ProductDashboard'],
[node, 'GET /nodejs/products'],
[prod, 'GET /go/product-catalog'],
['elasticsearch', 'GET product-*/_search'],
],
[
[chk, 'PUT /go/update-inventory'],
[prod, 'PUT /go/product/i012345'],
],
[pay],
];
},
})
);
},
};
};

View file

@ -8,7 +8,6 @@
"kbn_references": [
"@kbn/datemath",
"@kbn/apm-synthtrace-client",
"@kbn/apm-plugin"
],
"exclude": [
"target/**/*",

View file

@ -5,14 +5,13 @@
* 2.0.
*/
import expect from '@kbn/expect';
import { apm, timerange } from '@kbn/apm-synthtrace-client';
import { timerange, serviceMap } from '@kbn/apm-synthtrace-client';
import {
APIClientRequestParamsOf,
APIReturnType,
} from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
import { RecursivePartial } from '@kbn/apm-plugin/typings/common';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { generateTrace } from '../traces/generate_trace';
export default function ApiTest({ getService }: FtrProviderContext) {
const registry = getService('registry');
@ -44,30 +43,30 @@ export default function ApiTest({ getService }: FtrProviderContext) {
registry.when('Service map', { config: 'trial', archives: [] }, () => {
describe('optional kuery param', () => {
before(async () => {
const go = apm
.service({ name: 'synthbeans-go', environment: 'test', agentName: 'go' })
.instance('synthbeans-go');
const java = apm
.service({ name: 'synthbeans-java', environment: 'test', agentName: 'java' })
.instance('synthbeans-java');
const node = apm
.service({ name: 'synthbeans-node', environment: 'test', agentName: 'nodejs' })
.instance('synthbeans-node');
const events = timerange(start, end)
.interval('15m')
.rate(1)
.generator((timestamp) => {
return [
generateTrace(timestamp, [go, java]),
generateTrace(timestamp, [java, go], 'redis'),
generateTrace(timestamp, [node], 'redis'),
generateTrace(timestamp, [node, java, go], 'elasticsearch').defaults({
'labels.name': 'node-java-go-es',
}),
generateTrace(timestamp, [go, node, java]),
];
});
.generator(
serviceMap({
services: [
{ 'synthbeans-go': 'go' },
{ 'synthbeans-java': 'java' },
{ 'synthbeans-node': 'nodejs' },
],
definePaths([go, java, node]) {
return [
[go, java],
[java, go, 'redis'],
[node, 'redis'],
{
path: [node, java, go, 'elasticsearch'],
transaction: (t) => t.defaults({ 'labels.name': 'node-java-go-es' }),
},
[go, node, java],
];
},
})
);
await synthtraceEsClient.index(events);
});