[APM] Synthtrace scenario fixes (#151280)

- Add filename to environment for consistency
- Use sha256 hash for `error.grouping_key`
This commit is contained in:
Søren Louv-Jansen 2023-04-21 10:25:26 +02:00 committed by GitHub
parent 3bfdefc21a
commit ccb486c953
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 122 additions and 101 deletions

View file

@ -6,6 +6,7 @@
* Side Public License, v 1.
*/
import { createHash } from 'crypto';
import { ApmError } from './apm_error';
import { Entity } from '../entity';
import { Metricset } from './metricset';
@ -63,19 +64,11 @@ export class Instance extends Entity<ApmFields> {
});
}
error({
message,
type,
groupingName,
}: {
message: string;
type?: string;
groupingName?: string;
}) {
error({ message, type }: { message: string; type?: string }) {
return new ApmError({
...this.fields,
'error.exception': [{ message, ...(type ? { type } : {}) }],
'error.grouping_name': groupingName || message,
'error.grouping_name': getErrorGroupingKey(message),
});
}
@ -97,3 +90,7 @@ export class Instance extends Entity<ApmFields> {
});
}
}
export function getErrorGroupingKey(content: string) {
return createHash('sha256').update(content).digest('hex');
}

View file

@ -8,6 +8,6 @@
import path from 'path';
export function getSynthtraceEnvironment(filename: string) {
return `Synthtrace: ${path.parse(filename).name}`;
export function getSynthtraceEnvironment(filename: string, suffix = '') {
return `Synthtrace: ${path.parse(filename).name} ${suffix}`.trim();
}

View file

@ -10,6 +10,11 @@ import { apm, ApmFields } from '@kbn/apm-synthtrace-client';
import { merge, range as lodashRange } from 'lodash';
import { Scenario } from '../cli/scenario';
import { ComponentTemplateName } from '../lib/apm/client/apm_synthtrace_es_client';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
const ENVIRONMENTS = ['production', 'development'].map((env) =>
getSynthtraceEnvironment(__filename, env)
);
const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
const {
@ -37,7 +42,6 @@ const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
},
generate: ({ range }) => {
const TRANSACTION_TYPES = ['request', 'custom'];
const ENVIRONMENTS = ['production', 'development'];
const MIN_DURATION = 10;
const MAX_DURATION = 1000;

View file

@ -9,6 +9,9 @@
import { random } from 'lodash';
import { apm, Instance, ApmFields } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async ({ logger }) => {
const languages = ['go', 'dotnet', 'java', 'python'];
@ -22,7 +25,7 @@ const scenario: Scenario<ApmFields> = async ({ logger }) => {
apm
.service({
name: `${service}-${languages[index % languages.length]}`,
environment: 'production',
environment: ENVIRONMENT,
agentName: languages[index % languages.length],
})
.instance(`instance-${index}`)

View file

@ -8,6 +8,11 @@
import { apm, ApmFields } from '@kbn/apm-synthtrace-client';
import { range as lodashRange } from 'lodash';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
const ENVIRONMENTS = ['production', 'development'].map((env) =>
getSynthtraceEnvironment(__filename, env)
);
const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
const { services: numServices = 10, txGroups: numTxGroups = 10 } = scenarioOpts ?? {};
@ -15,7 +20,6 @@ const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
return {
generate: ({ range }) => {
const TRANSACTION_TYPES = ['request'];
const ENVIRONMENTS = ['production', 'development'];
const MIN_DURATION = 10;
const MAX_DURATION = 1000;

View file

@ -7,20 +7,23 @@
*/
import { apm, ApmFields } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
return {
generate: ({ range }) => {
const withTx = apm
.service('service-with-transactions', 'production', 'java')
.service('service-with-transactions', ENVIRONMENT, 'java')
.instance('instance');
const withErrorsOnly = apm
.service('service-with-errors-only', 'production', 'java')
.service('service-with-errors-only', ENVIRONMENT, 'java')
.instance('instance');
const withAppMetricsOnly = apm
.service('service-with-app-metrics-only', 'production', 'java')
.service('service-with-app-metrics-only', ENVIRONMENT, 'java')
.instance('instance');
return range

View file

@ -60,7 +60,11 @@ describe('transactions with errors', () => {
.errors(instance.error({ message: 'test error' }).timestamp(timestamp))
.serialize();
expect(error['error.grouping_name']).toEqual('test error');
expect(error['error.grouping_key']).toMatchInlineSnapshot(`"0000000000000000000000test error"`);
expect(error['error.grouping_name']).toEqual(
'4274b1899eba687801198c89f64a3fdade080a475c8a54881ba8fa10e7f45691'
);
expect(error['error.grouping_key']).toMatchInlineSnapshot(
`"4274b1899eba687801198c89f64a3fdade080a475c8a54881ba8fa10e7f45691"`
);
});
});

View file

@ -5,6 +5,7 @@
* 2.0.
*/
import { getErrorGroupingKey } from '@kbn/apm-synthtrace-client/src/lib/apm/instance';
import url from 'url';
import { synthtrace } from '../../../../synthtrace';
import { checkA11y } from '../../../support/commands';
@ -12,14 +13,6 @@ import { generateData } from './generate_data';
const start = '2021-10-10T00:00:00.000Z';
const end = '2021-10-10T00:15:00.000Z';
const errorDetailsPageHref = url.format({
pathname:
'/app/apm/services/opbeans-java/errors/0000000000000000000000000Error%200',
query: {
rangeFrom: start,
rangeTo: end,
},
});
describe('Error details', () => {
beforeEach(() => {
@ -41,18 +34,29 @@ describe('Error details', () => {
});
it('has no detectable a11y violations on load', () => {
const errorGroupingKey = getErrorGroupingKey('Error 1');
const errorGroupingKeyShort = errorGroupingKey.slice(0, 5);
const errorDetailsPageHref = url.format({
pathname: `/app/apm/services/opbeans-java/errors/${errorGroupingKey}`,
query: {
rangeFrom: start,
rangeTo: end,
},
});
cy.visitKibana(errorDetailsPageHref);
cy.contains('Error group 00000');
cy.contains(`Error group ${errorGroupingKeyShort}`);
// set skipFailures to true to not fail the test when there are accessibility failures
checkA11y({ skipFailures: true });
});
describe('when error has no occurrences', () => {
it('shows zero occurrences', () => {
const errorGroupingKey = getErrorGroupingKey('Error foo bar');
cy.visitKibana(
url.format({
pathname:
'/app/apm/services/opbeans-java/errors/0000000000000000000000000Error%201',
pathname: `/app/apm/services/opbeans-java/errors/${errorGroupingKey}`,
query: {
rangeFrom: start,
rangeTo: end,
@ -65,9 +69,19 @@ describe('Error details', () => {
});
describe('when error has data', () => {
const errorGroupingKey = getErrorGroupingKey('Error 1');
const errorGroupingKeyShort = errorGroupingKey.slice(0, 5);
const errorDetailsPageHref = url.format({
pathname: `/app/apm/services/opbeans-java/errors/${errorGroupingKey}`,
query: {
rangeFrom: start,
rangeTo: end,
},
});
it('shows errors distribution chart', () => {
cy.visitKibana(errorDetailsPageHref);
cy.contains('Error group 00000');
cy.contains(`Error group ${errorGroupingKeyShort}`);
cy.getByTestSubj('errorDistribution').contains('Error occurrences');
});
@ -89,7 +103,6 @@ describe('Error details', () => {
describe('when clicking on related transaction sample', () => {
it('should redirects to the transaction details page', () => {
cy.visitKibana(errorDetailsPageHref);
cy.contains('Error group 00000');
cy.contains('a', 'GET /apple 🍎').click();
cy.url().should('include', 'opbeans-java/transactions/view');
});

View file

@ -30,6 +30,8 @@ describe('Errors page', () => {
describe('when data is loaded', () => {
before(() => {
synthtrace.clean();
synthtrace.index(
generateData({
from: new Date(start).getTime(),
@ -142,38 +144,28 @@ describe('Check detailed statistics API with multiple errors', () => {
cy.visitKibana(`${javaServiceErrorsPageHref}&pageSize=10`);
cy.wait('@errorsMainStatistics');
cy.get('.euiPagination__list').children().should('have.length', 5);
let requestedGroupIdsPage1: string[];
cy.wait('@errorsDetailedStatistics').then((payload) => {
expect(payload.request.body.groupIds).eql(
JSON.stringify([
'0000000000000000000000000Error 0',
'0000000000000000000000000Error 1',
'0000000000000000000000000Error 2',
'0000000000000000000000000Error 3',
'0000000000000000000000000Error 4',
'0000000000000000000000000Error 5',
'0000000000000000000000000Error 6',
'0000000000000000000000000Error 7',
'0000000000000000000000000Error 8',
'0000000000000000000000000Error 9',
])
);
cy.get('[data-test-subj="errorGroupId"]').each(($el, index) => {
const displayedGroupId = $el.text();
requestedGroupIdsPage1 = JSON.parse(payload.request.body.groupIds);
const requestedGroupId = requestedGroupIdsPage1[index].slice(0, 5);
expect(displayedGroupId).eq(requestedGroupId);
expect(requestedGroupIdsPage1).to.have.length(10);
});
});
cy.getByTestSubj('pagination-button-1').click();
// expect that the requested groupIds on page 2 are different from page 1
cy.wait('@errorsDetailedStatistics').then((payload) => {
expect(payload.request.body.groupIds).eql(
JSON.stringify([
'000000000000000000000000Error 10',
'000000000000000000000000Error 11',
'000000000000000000000000Error 12',
'000000000000000000000000Error 13',
'000000000000000000000000Error 14',
'000000000000000000000000Error 15',
'000000000000000000000000Error 16',
'000000000000000000000000Error 17',
'000000000000000000000000Error 18',
'000000000000000000000000Error 19',
])
);
const requestedGroupIdsPage2 = JSON.parse(payload.request.body.groupIds);
expect(requestedGroupIdsPage1[0]).not.eq(requestedGroupIdsPage2[0]);
expect(requestedGroupIdsPage2).to.have.length(10);
});
});
});

View file

@ -110,7 +110,11 @@ function ErrorGroupList({
width: `${unit * 6}px`,
render: (_, { groupId }) => {
return (
<GroupIdLink serviceName={serviceName} errorGroupId={groupId}>
<GroupIdLink
serviceName={serviceName}
errorGroupId={groupId}
data-test-subj="errorGroupId"
>
{groupId.slice(0, 5) || NOT_AVAILABLE_LABEL}
</GroupIdLink>
);

View file

@ -5,14 +5,11 @@
* 2.0.
*/
import expect from '@kbn/expect';
import {
APIClientRequestParamsOf,
APIReturnType,
} from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
import { RecursivePartial } from '@kbn/apm-plugin/typings/common';
import { timerange } from '@kbn/apm-synthtrace-client';
import { service } from '@kbn/apm-synthtrace-client/src/lib/apm/service';
import { orderBy } from 'lodash';
import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
import { getErrorGroupingKey } from '@kbn/apm-synthtrace-client/src/lib/apm/instance';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { config, generateData } from './generate_data';
@ -31,25 +28,19 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const start = new Date('2021-01-01T00:00:00.000Z').getTime();
const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
async function callErrorGroupSamplesApi(
overrides?: RecursivePartial<
APIClientRequestParamsOf<'GET /internal/apm/services/{serviceName}/errors/{groupId}/samples'>['params']
>
) {
async function callErrorGroupSamplesApi({ groupId }: { groupId: string }) {
const response = await apmApiClient.readUser({
endpoint: 'GET /internal/apm/services/{serviceName}/errors/{groupId}/samples',
params: {
path: {
serviceName,
groupId: 'foo',
...overrides?.path,
groupId,
},
query: {
start: new Date(start).toISOString(),
end: new Date(end).toISOString(),
environment: 'ENVIRONMENT_ALL',
kuery: '',
...overrides?.query,
},
},
});
@ -78,7 +69,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
registry.when('when data is not loaded', { config: 'basic', archives: [] }, () => {
it('handles the empty state', async () => {
const response = await callErrorGroupSamplesApi();
const response = await callErrorGroupSamplesApi({ groupId: 'foo' });
expect(response.status).to.be(200);
expect(response.body.occurrencesCount).to.be(0);
});
@ -97,7 +88,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
let errorsSamplesResponse: ErrorGroupSamples;
before(async () => {
const response = await callErrorGroupSamplesApi({
path: { groupId: '0000000000000000000000000Error 1' },
groupId: '98b75903135eac35ad42419bd3b45cf8b4270c61cbd0ede0f7e8c8a9ac9fdb03',
});
errorsSamplesResponse = response.body;
});
@ -124,7 +115,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
let errorSampleDetailsResponse: ErrorSampleDetails;
before(async () => {
const errorsSamplesResponse = await callErrorGroupSamplesApi({
path: { groupId: '0000000000000000000000000Error 1' },
groupId: '98b75903135eac35ad42419bd3b45cf8b4270c61cbd0ede0f7e8c8a9ac9fdb03',
});
const errorId = errorsSamplesResponse.body.errorSampleIds[0];
@ -133,10 +124,13 @@ export default function ApiTest({ getService }: FtrProviderContext) {
errorSampleDetailsResponse = response.body;
});
it('displays correct error sample data', () => {
it('displays correct error grouping_key', () => {
expect(errorSampleDetailsResponse.error.error.grouping_key).to.equal(
'0000000000000000000000000Error 1'
'98b75903135eac35ad42419bd3b45cf8b4270c61cbd0ede0f7e8c8a9ac9fdb03'
);
});
it('displays correct error message', () => {
expect(errorSampleDetailsResponse.error.error.exception?.[0].message).to.equal('Error 1');
});
});
@ -147,6 +141,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
before(async () => {
const instance = service(serviceName, 'production', 'go').instance('a');
const errorMessage = 'Error 1';
const groupId = getErrorGroupingKey(errorMessage);
await synthtraceEsClient.index([
timerange(start, end)
@ -160,24 +156,20 @@ export default function ApiTest({ getService }: FtrProviderContext) {
.timestamp(timestamp)
.sample(false)
.errors(
instance.error({ message: `Error 1` }).timestamp(timestamp),
instance.error({ message: `Error 1` }).timestamp(timestamp + 1)
instance.error({ message: errorMessage }).timestamp(timestamp),
instance.error({ message: errorMessage }).timestamp(timestamp + 1)
),
instance
.transaction('GET /api/foo')
.duration(100)
.timestamp(timestamp)
.sample(true)
.errors(instance.error({ message: `Error 1` }).timestamp(timestamp)),
.errors(instance.error({ message: errorMessage }).timestamp(timestamp)),
];
}),
]);
errorGroupSamplesResponse = (
await callErrorGroupSamplesApi({
path: { groupId: '0000000000000000000000000Error 1' },
})
).body;
errorGroupSamplesResponse = (await callErrorGroupSamplesApi({ groupId })).body;
});
after(() => synthtraceEsClient.clean());
@ -187,6 +179,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
// this checks whether the order of indexing is different from the order that is returned
// if it is not, scoring/sorting is broken
expect(errorGroupSamplesResponse.errorSampleIds.length).to.be(3);
expect(idsOfErrors).to.not.eql(orderBy(idsOfErrors));
});
});

View file

@ -59,7 +59,7 @@ export async function generateData({
.transaction({ transactionName: transaction.name })
.errors(
serviceGoProdInstance
.error({ message: 'Error 1', type: transaction.name, groupingName: 'Error test' })
.error({ message: 'Error 1', type: transaction.name })
.timestamp(timestamp)
)
.duration(1000)

View file

@ -26,6 +26,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const serviceName = 'synth-go';
const start = new Date('2021-01-01T00:00:00.000Z').getTime();
const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
const groupId = '98b75903135eac35ad42419bd3b45cf8b4270c61cbd0ede0f7e8c8a9ac9fdb03';
async function callApi(
overrides?: RecursivePartial<
@ -82,7 +83,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
before(async () => {
const response = await callApi({
path: { groupId: '0000000000000000000000Error test' },
path: { groupId },
});
erroneousTransactions = response.body;
});
@ -132,7 +133,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
before(async () => {
const fiveMinutes = 5 * 60 * 1000;
const response = await callApi({
path: { groupId: '0000000000000000000000Error test' },
path: { groupId },
query: {
start: new Date(end - fiveMinutes).toISOString(),
end: new Date(end).toISOString(),
@ -182,7 +183,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
describe('when there are no data for the time period', () => {
it('returns an empty array', async () => {
const response = await callApi({
path: { groupId: '0000000000000000000000Error test' },
path: { groupId },
query: {
start: '2021-01-03T00:00:00.000Z',
end: '2021-01-03T00:15:00.000Z',

View file

@ -71,31 +71,34 @@ export default function ApiTest({ getService }: FtrProviderContext) {
after(() => synthtraceEsClient.clean());
describe('returns the correct data', () => {
const NUMBER_OF_BUCKETS = 15;
let errorGroups: ErrorGroups;
before(async () => {
const response = await callApi({ query: { transactionName: firstTransactionName } });
errorGroups = response.body.errorGroups;
});
it('returns correct number of errors and error data', () => {
const numberOfBuckets = 15;
it('returns correct number of errors', () => {
expect(errorGroups.length).to.equal(2);
});
const firstErrorId = `Error 1 transaction ${firstTransactionName}`;
it('error 1 is correct', () => {
const firstErrorId = `b6c1d4d41b0b60b841f40232497344ba36856fcbea0692a4695562ca73e790bd`;
const firstError = errorGroups.find((x) => x.groupId === firstErrorId);
expect(firstError).to.not.be(undefined);
expect(firstError?.groupId).to.be(firstErrorId);
expect(firstError?.name).to.be(firstErrorId);
expect(firstError?.occurrences).to.be(firstTransactionFailureRate * numberOfBuckets);
expect(firstError?.name).to.be(`Error 1 transaction GET /apple 🍎`);
expect(firstError?.occurrences).to.be(firstTransactionFailureRate * NUMBER_OF_BUCKETS);
expect(firstError?.lastSeen).to.be(moment(end).startOf('minute').valueOf());
});
const secondErrorId = `Error 2 transaction ${firstTransactionName}`;
it('error 2 is correct', () => {
const secondErrorId = `c3f388e4f7276d4fab85aa2fad2d2a42e70637f65cd5ec9f085de28b36e69ba5`;
const secondError = errorGroups.find((x) => x.groupId === secondErrorId);
expect(secondError).to.not.be(undefined);
expect(secondError?.groupId).to.be(secondErrorId);
expect(secondError?.name).to.be(secondErrorId);
expect(secondError?.occurrences).to.be(firstTransactionFailureRate * numberOfBuckets);
expect(secondError?.name).to.be(`Error 2 transaction GET /apple 🍎`);
expect(secondError?.occurrences).to.be(firstTransactionFailureRate * NUMBER_OF_BUCKETS);
expect(secondError?.lastSeen).to.be(moment(end).startOf('minute').valueOf());
});
});