github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[APM] Fix condition to use serviceTransactionMetric documents (#180903)

Browse source 
closes https://github.com/elastic/kibana/issues/167578
(https://github.com/elastic/kibana/issues/167578#issuecomment-2061114814)

## Summary

This PR fixes when `/time_range_metadata` returns `hasDocs=true` for
`serviceTransactionMetric` to make sure the UI doesn't lose information
by querying `serviceTransactionMetric` docs in a range where docs from
APM < 8.7 still exists.


**IMPORTANT** ❗ 

Based on the APM Server
[changelog](bbf4eb276b/changelogs/8.7.asciidoc (L71))
and this [issue](https://github.com/elastic/apm-server/issues/9703),
`serviceTransactionMetric` doc was added in 8.7 as well as the
aggregations by `10m` and `60m` intervals for `transaction` docs. In
`8.7`, the `transaction.duration.summary` was also introduced, therefore
the new rollup interval docs will always contain the new field.

Based on that, the logic was changed to

- Verify whether `transaction` `10m` and `60m` are supported within a
given time range
- Verify whether or not `transaction.duration.summary` is supported for
`transaction` `1m` docs
- All other doc types and intervals are presupposed to support
`transaction.duration.summary`

### Range starting from when there is no data (contains docs generated
by APM < 8.7 and APM >= 8.7)

<img width="1210" alt="image"
src="d7767d28-03ef-4f5a-b10c-b8529d9b0f69">


```json
{
    "isUsingServiceDestinationMetrics": false,
    "sources": [
        {
            "documentType": "serviceTransactionMetric",
            "rollupInterval": "1m",
            "hasDocs": false,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "serviceTransactionMetric",
            "rollupInterval": "10m",
            "hasDocs": false,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "serviceTransactionMetric",
            "rollupInterval": "60m",
            "hasDocs": false,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionMetric",
            "rollupInterval": "1m",
            "hasDocs": true,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionMetric",
            "rollupInterval": "10m",
            "hasDocs": true,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionMetric",
            "rollupInterval": "60m",
            "hasDocs": true,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionEvent",
            "rollupInterval": "none",
            "hasDocs": true,
            "hasDurationSummaryField": false
        }
    ]
}
```

### Range starting from when first docs were ingested (contains docs
generated by APM < 8.7 and APM >= 8.7)

<img width="1208" alt="image"
src="982f2359-fb20-4bf8-8d9a-afbb1cd5b0ea">


```json
{
    "isUsingServiceDestinationMetrics": false,
    "sources": [
        {
            "documentType": "serviceTransactionMetric",
            "rollupInterval": "1m",
            "hasDocs": false,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "serviceTransactionMetric",
            "rollupInterval": "10m",
            "hasDocs": false,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "serviceTransactionMetric",
            "rollupInterval": "60m",
            "hasDocs": false,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionMetric",
            "rollupInterval": "1m",
            "hasDocs": true,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionMetric",
            "rollupInterval": "10m",
            "hasDocs": true,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionMetric",
            "rollupInterval": "60m",
            "hasDocs": true,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionEvent",
            "rollupInterval": "none",
            "hasDocs": true,
            "hasDurationSummaryField": false
        }
    ]
}
```

### Range starting from when only APM >= 8.7 docs exist
<img width="1198" alt="image"
src="3a972457-3a0a-440d-85a1-add8a48f37d1">


```json
{
    "isUsingServiceDestinationMetrics": false,
    "sources": [
        {
            "documentType": "serviceTransactionMetric",
            "rollupInterval": "1m",
            "hasDocs": true,
            "hasDurationSummaryField": true
        },
        {
            "documentType": "serviceTransactionMetric",
            "rollupInterval": "10m",
            "hasDocs": true,
            "hasDurationSummaryField": true
        },
        {
            "documentType": "serviceTransactionMetric",
            "rollupInterval": "60m",
            "hasDocs": true,
            "hasDurationSummaryField": true
        },
        {
            "documentType": "transactionMetric",
            "rollupInterval": "1m",
            "hasDocs": true,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionMetric",
            "rollupInterval": "10m",
            "hasDocs": true,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionMetric",
            "rollupInterval": "60m",
            "hasDocs": true,
            "hasDurationSummaryField": false
        },
        {
            "documentType": "transactionEvent",
            "rollupInterval": "none",
            "hasDocs": true,
            "hasDurationSummaryField": false
        }
    ]
}
```
### How to tests

Generate docs without ServiceTransactionMetrics
```bash
node scripts/synthtrace service_summary_field_version_dependent.ts  --versionOverride=8.6.2  --from=now-2h --to=now --clean
```

Generate docs with ServiceTransactionMetrics
```bash
node scripts/synthtrace service_summary_field_version_dependent.ts --from=now-1h --to=now  
```  

Follow the same steps from above.

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Carlos Crespo 2024-05-07 10:54:03 -03:00 • committed by GitHub
parent a5ba2d664b
commit 314d6eef4d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 271 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

33
packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/apm_pipeline.ts
View file

@ -6,11 +6,13 @@
* Side Public License, v 1.
*/
import semver from 'semver';
import { PassThrough, pipeline, Readable } from 'stream';
import { getDedotTransform } from '../../../shared/get_dedot_transform';
import { getSerializeTransform } from '../../../shared/get_serialize_transform';
import { Logger } from '../../../utils/create_logger';
import { fork } from '../../../utils/stream_utils';
import { deleteSummaryFieldTransform } from '../../../utils/transform_helpers';
import { createBreakdownMetricsAggregator } from '../../aggregators/create_breakdown_metrics_aggregator';
import { createServiceMetricsAggregator } from '../../aggregators/create_service_metrics_aggregator';
import { createServiceSummaryMetricsAggregator } from '../../aggregators/create_service_summary_metrics_aggregator';
@ -22,22 +24,32 @@ import { getRoutingTransform } from './get_routing_transform';
export function apmPipeline(logger: Logger, version: string, includeSerialization: boolean = true) {
return (base: Readable) => {
const continousRollupSupported =
!version || semver.gte(semver.coerce(version)?.version ?? version, '8.7.0');
const aggregators = [
createTransactionMetricsAggregator('1m'),
createTransactionMetricsAggregator('10m'),
createTransactionMetricsAggregator('60m'),
createServiceMetricsAggregator('1m'),
createServiceMetricsAggregator('10m'),
createServiceMetricsAggregator('60m'),
createServiceSummaryMetricsAggregator('1m'),
createServiceSummaryMetricsAggregator('10m'),
createServiceSummaryMetricsAggregator('60m'),
createSpanMetricsAggregator('1m'),
createSpanMetricsAggregator('10m'),
createSpanMetricsAggregator('60m'),
...(continousRollupSupported
? [
createTransactionMetricsAggregator('10m'),
createTransactionMetricsAggregator('60m'),
createServiceMetricsAggregator('1m'),
createServiceMetricsAggregator('10m'),
createServiceMetricsAggregator('60m'),
createServiceSummaryMetricsAggregator('1m'),
createServiceSummaryMetricsAggregator('10m'),
createServiceSummaryMetricsAggregator('60m'),
createSpanMetricsAggregator('10m'),
createSpanMetricsAggregator('60m'),
]
: []),
];
const serializationTransform = includeSerialization ? [getSerializeTransform()] : [];
const removeDurationSummaryTransform = !continousRollupSupported
? [deleteSummaryFieldTransform()]
: [];
return pipeline(
// @ts-expect-error Some weird stuff here with the type definition for pipeline. We have tests!
@ -49,6 +61,7 @@ export function apmPipeline(logger: Logger, version: string, includeSerializatio
getApmServerMetadataTransform(version),
getRoutingTransform(),
getDedotTransform(),
...removeDurationSummaryTransform,
(err) => {
if (err) {
logger.error(err);

12
packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/index.ts
View file

@ -65,7 +65,15 @@ export class ApmSynthtraceEsClient extends SynthtraceEsClient<ApmFields> {
this.logger.info(`Updated component template: ${name}`);
}
getDefaultPipeline(includeSerialization: boolean = true) {
return apmPipeline(this.logger, this.version, includeSerialization);
getDefaultPipeline(
{
includeSerialization,
versionOverride,
}: {
includeSerialization?: boolean;
versionOverride?: string;
} = { includeSerialization: true }
) {
return apmPipeline(this.logger, versionOverride ?? this.version, includeSerialization);
}
}

34
packages/kbn-apm-synthtrace/src/scenarios/service_summary_field_version_dependent.ts
View file

@ -8,36 +8,26 @@
import { ApmFields, apm } from '@kbn/apm-synthtrace-client';
import { random } from 'lodash';
import { pipeline, Readable } from 'stream';
import { Readable } from 'stream';
import semver from 'semver';
import { Scenario } from '../cli/scenario';
import {
addObserverVersionTransform,
deleteSummaryFieldTransform,
} from '../lib/utils/transform_helpers';
import { withClient } from '../lib/utils/with_client';
import { RunOptions } from '../cli/utils/parse_run_cli_flags';
import { Logger } from '../lib/utils/create_logger';
const scenario: Scenario<ApmFields> = async ({ logger, versionOverride }) => {
const scenario: Scenario<ApmFields> = async ({
logger,
versionOverride,
}: RunOptions & { logger: Logger }) => {
const version = versionOverride as string;
const isLegacy = versionOverride && semver.lt(version, '8.7.0');
const isLegacy = version ? semver.lt(version as string, '8.7.0') : false;
return {
bootstrap: async ({ apmEsClient }) => {
if (isLegacy) {
apmEsClient.pipeline((base: Readable) => {
const defaultPipeline = apmEsClient.getDefaultPipeline()(
base
) as unknown as NodeJS.ReadableStream;
return pipeline(
defaultPipeline,
addObserverVersionTransform(version),
deleteSummaryFieldTransform(),
(err) => {
if (err) {
logger.error(err);
}
}
);
return apmEsClient.getDefaultPipeline({
versionOverride: version,
})(base);
});
}
},
@ -45,7 +35,7 @@ const scenario: Scenario<ApmFields> = async ({ logger, versionOverride }) => {
const successfulTimestamps = range.ratePerMinute(6);
const instance = apm
.service({
name: `java${isLegacy ? '-legacy' : ''}`,
name: `java`,
environment: 'production',
agentName: 'java',
})

2
packages/kbn-journeys/services/synthtrace.ts
View file

@ -117,7 +117,7 @@ async function initApmSynthtraceClient(options: SynthtraceClientOptions) {
version: packageVersion,
});
synthEsClient.pipeline(synthEsClient.getDefaultPipeline(false));
synthEsClient.pipeline(synthEsClient.getDefaultPipeline({ includeSerialization: false }));
return synthEsClient;
}

2
x-pack/plugins/observability_solution/apm/dev_docs/apm_queries.md
View file

@ -226,7 +226,7 @@ GET apm-*-metric-*,metrics-apm*/_search?terminate_after=1000
Service transaction metrics are aggregated metric documents that hold latency and throughput metrics pivoted by `service.name`, `service.environment` and `transaction.type`. Additionally, `agent.name` and `service.language.name` are included as metadata.
We use the response from the `GET /internal/apm/time_range_metadata` endpoint to determine what data source is available. A data source is considered available if there is either data before the current time range, or, if there is no data at all before the current time range, if there is data within the current time range. This means that existing deployments will use transaction metrics right after upgrading (instead of using service transaction metrics and seeing a mostly blank screen), but also that new deployments immediately get the benefits of service transaction metrics, instead of falling all the way back to transaction events.
We use the response from the `GET /internal/apm/time_range_metadata` endpoint to determine what data source is available. Service transaction metrics docs, introduced in APM >= 8.7, is considered available if there is data before *and* within the current time range. This ensure the UI won't miss information shipped by APM < 8.7. For < 8.7 documents, availability is determined by whether there is data before the current time range, or no data at all before the current time range, but there is data within the current time range. This means that existing deployments will use transaction metrics right after upgrading (instead of using service transaction metrics and seeing a mostly blank screen), but also that new deployments immediately get the benefits of service transaction metrics, instead of falling all the way back to transaction events.
A pre-aggregated document where `_doc_count` is the number of transaction events

4
x-pack/plugins/observability_solution/apm/ftr_e2e/setup_cypress_node_events.ts
View file

@ -28,7 +28,9 @@ export function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.Plugin
version: config.env.APM_PACKAGE_VERSION,
});
synthtraceEsClient.pipeline(synthtraceEsClient.getDefaultPipeline(false));
synthtraceEsClient.pipeline(
synthtraceEsClient.getDefaultPipeline({ includeSerialization: false })
);
initPlugin(on, config);

88
x-pack/plugins/observability_solution/apm/server/lib/helpers/get_document_sources.ts
View file

@ -11,22 +11,13 @@ import { RollupInterval } from '../../../common/rollup';
import { APMEventClient } from './create_es_client/create_apm_event_client';
import { getConfigForDocumentType } from './create_es_client/document_type';
import { TimeRangeMetadata } from '../../../common/time_range_metadata';
import { getDurationLegacyFilter } from './transactions';
import { isDurationSummaryNotSupportedFilter } from './transactions';
const QUERY_INDEX = {
BEFORE: 0,
CURRENT: 1,
DURATION_SUMMARY: 2,
DOCUMENT_TYPE: 0,
DURATION_SUMMARY_NOT_SUPPORTED: 1,
} as const;
interface DocumentTypeData {
documentType: ApmDocumentType;
rollupInterval: RollupInterval;
hasDocBefore: boolean;
hasDocAfter: boolean;
allHaveDurationSummary: boolean;
}
const getRequest = ({
documentType,
rollupInterval,
@ -93,10 +84,8 @@ export async function getDocumentSources({
documentTypesToCheck,
});
const hasAnySourceDocBefore = documentTypesInfo.some((source) => source.hasDocBefore);
return [
...mapToSources(documentTypesInfo, hasAnySourceDocBefore),
...documentTypesInfo,
{
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
@ -120,7 +109,7 @@ const getDocumentTypesInfo = async ({
kuery: string;
enableContinuousRollups: boolean;
documentTypesToCheck: ApmDocumentType[];
}) => {
}): Promise<TimeRangeMetadata['sources']> => {
const getRequests = getDocumentTypeRequestsFn({
enableContinuousRollups,
start,
@ -131,25 +120,36 @@ const getDocumentTypesInfo = async ({
const sourceRequests = documentTypesToCheck.flatMap(getRequests);
const allSearches = sourceRequests
.flatMap(({ before, current, durationSummaryCheck }) => [before, current, durationSummaryCheck])
.flatMap(({ documentTypeQuery, durationSummaryNotSupportedQuery }) => [
documentTypeQuery,
durationSummaryNotSupportedQuery,
])
.filter((request): request is ReturnType<typeof getRequest> => request !== undefined);
const allResponses = (await apmEventClient.msearch('get_document_availability', ...allSearches))
.responses;
const hasAnyLegacyDocuments = sourceRequests.some(
({ documentType, rollupInterval }, index) =>
isLegacyDocType(documentType, rollupInterval) &&
allResponses[index + QUERY_INDEX.DURATION_SUMMARY_NOT_SUPPORTED].hits.total.value > 0
);
return sourceRequests.map(({ documentType, rollupInterval, ...queries }) => {
const numberOfQueries = Object.values(queries).filter(Boolean).length;
// allResponses is sorted by the order of the requests in sourceRequests
const docTypeResponses = allResponses.splice(0, numberOfQueries);
const hasDocs = docTypeResponses[QUERY_INDEX.DOCUMENT_TYPE].hits.total.value > 0;
// can only use >=8.7 document types (ServiceTransactionMetrics or TransactionMetrics with 10m and 60m intervals)
// if there are no legacy documents
const canUseContinousRollupDocs = hasDocs && !hasAnyLegacyDocuments;
return {
documentType,
rollupInterval,
hasDocBefore: docTypeResponses[QUERY_INDEX.BEFORE].hits.total.value > 0,
hasDocAfter: docTypeResponses[QUERY_INDEX.CURRENT].hits.total.value > 0,
allHaveDurationSummary: docTypeResponses[QUERY_INDEX.DURATION_SUMMARY]
? docTypeResponses[QUERY_INDEX.DURATION_SUMMARY].hits.total.value === 0
: true,
hasDocs: isLegacyDocType(documentType, rollupInterval) ? hasDocs : canUseContinousRollupDocs,
// all >=8.7 document types with rollups support duration summary
hasDurationSummaryField: canUseContinousRollupDocs,
};
});
};
@ -168,9 +168,7 @@ const getDocumentTypeRequestsFn =
}) =>
(documentType: ApmDocumentType) => {
const currentRange = rangeQuery(start, end);
const diff = end - start;
const kql = kqlQuery(kuery);
const beforeRange = rangeQuery(start - diff, end - diff);
const rollupIntervals = enableContinuousRollups
? getConfigForDocumentType(documentType).rollupIntervals
@ -179,48 +177,26 @@ const getDocumentTypeRequestsFn =
return rollupIntervals.map((rollupInterval) => ({
documentType,
rollupInterval,
before: getRequest({
documentType,
rollupInterval,
filters: [...kql, ...beforeRange],
}),
current: getRequest({
documentTypeQuery: getRequest({
documentType,
rollupInterval,
filters: [...kql, ...currentRange],
}),
...(documentType !== ApmDocumentType.ServiceTransactionMetric
...(isLegacyDocType(documentType, rollupInterval)
? {
durationSummaryCheck: getRequest({
durationSummaryNotSupportedQuery: getRequest({
documentType,
rollupInterval,
filters: [...kql, ...currentRange, getDurationLegacyFilter()],
filters: [...kql, ...currentRange, isDurationSummaryNotSupportedFilter()],
}),
}
: {}),
: undefined),
}));
};
const mapToSources = (sources: DocumentTypeData[], hasAnySourceDocBefore: boolean) => {
return sources.map((source) => {
const { documentType, hasDocAfter, hasDocBefore, rollupInterval, allHaveDurationSummary } =
source;
const hasDocBeforeOrAfter = hasDocBefore || hasDocAfter;
// If there is any data before, we require that data is available before
// this time range to mark this source as available. If we don't do that,
// users that upgrade to a version that starts generating service tx metrics
// will see a mostly empty screen for a while after upgrading.
// If we only check before, users with a new deployment will use raw transaction
// events.
const hasDocs = hasAnySourceDocBefore ? hasDocBefore : hasDocBeforeOrAfter;
return {
documentType,
rollupInterval,
hasDocs,
hasDurationSummaryField: allHaveDurationSummary,
};
});
const isLegacyDocType = (documentType: ApmDocumentType, rollupInterval: RollupInterval) => {
return (
documentType === ApmDocumentType.TransactionMetric &&
rollupInterval === RollupInterval.OneMinute
);
};

11
x-pack/plugins/observability_solution/apm/server/lib/helpers/transactions/index.ts
View file

@ -164,17 +164,10 @@ export function isRootTransaction(searchAggregatedTransactions: boolean) {
};
}
export function getDurationLegacyFilter(): QueryDslQueryContainer {
export function isDurationSummaryNotSupportedFilter(): QueryDslQueryContainer {
return {
bool: {
must: [
{
bool: {
filter: [{ exists: { field: TRANSACTION_DURATION_HISTOGRAM } }],
must_not: [{ exists: { field: TRANSACTION_DURATION_SUMMARY } }],
},
},
],
must_not: [{ exists: { field: TRANSACTION_DURATION_SUMMARY } }],
},
};
}

77
x-pack/test/apm_api_integration/tests/time_range_metadata/many_apm_server_versions.spec.ts
View file

@ -7,11 +7,7 @@
import expect from '@kbn/expect';
import { apm, timerange } from '@kbn/apm-synthtrace-client';
import moment from 'moment';
import {
addObserverVersionTransform,
ApmSynthtraceEsClient,
deleteSummaryFieldTransform,
} from '@kbn/apm-synthtrace';
import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
import {
TRANSACTION_DURATION_HISTOGRAM,
TRANSACTION_DURATION_SUMMARY,
@ -19,7 +15,7 @@ import {
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import { LatencyAggregationType } from '@kbn/apm-plugin/common/latency_aggregation_types';
import { pipeline, Readable } from 'stream';
import { Readable } from 'stream';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { ApmApiClient } from '../../common/config';
@ -32,9 +28,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const baseTime = new Date('2023-10-01T00:00:00.000Z').getTime();
const startLegacy = moment(baseTime).add(0, 'minutes');
const start = moment(baseTime).add(5, 'minutes');
const end = moment(baseTime).add(10, 'minutes');
const endLegacy = moment(baseTime).add(10, 'minutes');
const end = moment(baseTime).add(15, 'minutes');
// FLAKY: https://github.com/elastic/kibana/issues/177534
registry.when(
'Time range metadata when there are multiple APM Server versions',
{ config: 'basic', archives: [] },
@ -44,7 +40,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
await generateTraceDataForService({
serviceName: 'synth-java-legacy',
start: startLegacy,
end,
end: endLegacy,
isLegacy: true,
synthtrace,
});
@ -78,7 +74,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
// @ts-expect-error
expect(res.hits.total.value).to.be(10);
expect(res.hits.total.value).to.be(20);
});
it('ingests transaction metrics without transaction.duration.summary', async () => {
@ -95,7 +91,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
// @ts-expect-error
expect(res.hits.total.value).to.be(20);
expect(res.hits.total.value).to.be(10);
});
it('has transaction.duration.summary field for every document type', async () => {
@ -103,7 +99,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
endpoint: 'GET /internal/apm/time_range_metadata',
params: {
query: {
start: start.toISOString(),
start: endLegacy.toISOString(),
end: end.toISOString(),
enableContinuousRollups: true,
enableServiceTransactionMetrics: true,
@ -116,8 +112,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const allHasSummaryField = response.body.sources
.filter(
(source) =>
source.documentType === ApmDocumentType.TransactionMetric &&
source.rollupInterval !== RollupInterval.OneMinute
source.documentType !== ApmDocumentType.TransactionEvent &&
source.rollupInterval !== RollupInterval.SixtyMinutes // there is not enough data for 60 minutes
)
.every((source) => {
return source.hasDurationSummaryField;
@ -132,7 +128,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
params: {
query: {
start: startLegacy.toISOString(),
end: end.toISOString(),
end: endLegacy.toISOString(),
enableContinuousRollups: true,
enableServiceTransactionMetrics: true,
useSpanName: false,
@ -148,11 +144,35 @@ export default function ApiTest({ getService }: FtrProviderContext) {
expect(allHasSummaryField).to.eql(false);
});
it('does not support transaction.duration.summary for transactionMetric 1m when not all documents within the range support it ', async () => {
const response = await apmApiClient.readUser({
endpoint: 'GET /internal/apm/time_range_metadata',
params: {
query: {
start: startLegacy.toISOString(),
end: end.toISOString(),
enableContinuousRollups: true,
enableServiceTransactionMetrics: true,
useSpanName: false,
kuery: '',
},
},
});
const hasDurationSummaryField = response.body.sources.find(
(source) =>
source.documentType === ApmDocumentType.TransactionMetric &&
source.rollupInterval === RollupInterval.OneMinute // there is not enough data for 60 minutes in the timerange defined for the tests
)?.hasDurationSummaryField;
expect(hasDurationSummaryField).to.eql(false);
});
it('does not have latency data for synth-java-legacy', async () => {
const res = await getLatencyChartForService({
serviceName: 'synth-java-legacy',
start,
end,
end: endLegacy,
apmApiClient,
useDurationSummary: true,
});
@ -171,18 +191,13 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const res = await getLatencyChartForService({
serviceName: 'synth-java',
start,
end,
end: endLegacy,
apmApiClient,
useDurationSummary: true,
});
expect(res.body.currentPeriod.latencyTimeseries.map(({ y }) => y)).to.eql([
1000000,
1000000,
1000000,
1000000,
1000000,
null,
1000000, 1000000, 1000000, 1000000, 1000000, 1000000,
]);
});
});
@ -256,21 +271,7 @@ function generateTraceDataForService({
);
const apmPipeline = (base: Readable) => {
const defaultPipeline = synthtrace.getDefaultPipeline()(
base
) as unknown as NodeJS.ReadableStream;
return pipeline(
defaultPipeline,
addObserverVersionTransform('8.5.0'),
deleteSummaryFieldTransform(),
(err) => {
if (err) {
// eslint-disable-next-line no-console
console.error(err);
}
}
);
return synthtrace.getDefaultPipeline({ versionOverride: '8.5.0' })(base);
};
return synthtrace.index(events, isLegacy ? apmPipeline : undefined);

195
x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts
View file

@ -11,12 +11,8 @@ import { omit, sortBy } from 'lodash';
import moment, { Moment } from 'moment';
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import {
addObserverVersionTransform,
ApmSynthtraceEsClient,
deleteSummaryFieldTransform,
} from '@kbn/apm-synthtrace';
import { Readable, pipeline } from 'stream';
import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
import { Readable } from 'stream';
import { ToolingLog } from '@kbn/tooling-log';
import { FtrProviderContext } from '../../common/ftr_provider_context';
@ -78,9 +74,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
});
// FLAKY: https://github.com/elastic/kibana/issues/177541
registry.when(
'Time range metadata when generating summary data',
'Time range metadata when generating data with multiple APM server versions',
{ config: 'basic', archives: [] },
() => {
describe('data loaded with and without summary field', () => {
@ -88,7 +83,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const withoutSummaryFieldEnd = moment(withoutSummaryFieldStart).add(2, 'hours');
const withSummaryFieldStart = moment(withoutSummaryFieldEnd);
const withSummaryFieldEnd = moment(withoutSummaryFieldEnd).add(2, 'hours');
const withSummaryFieldEnd = moment(withSummaryFieldStart).add(2, 'hours');
before(async () => {
await getTransactionEvents({
@ -112,42 +107,161 @@ export default function ApiTest({ getService }: FtrProviderContext) {
return apmSynthtraceEsClient.clean();
});
describe('Values for hasDurationSummaryField for transaction metrics', () => {
it('returns true when summary field is available both inside and outside the range', async () => {
describe('aggregators and summary field support', () => {
it('returns support only for legacy transactionMetrics 1m without duration summary field', async () => {
const response = await getTimeRangeMedata({
start: moment(withSummaryFieldStart).add(1, 'hour'),
end: moment(withSummaryFieldEnd),
start: withoutSummaryFieldStart,
end: withoutSummaryFieldEnd,
});
expect(
response.sources.filter(
(source) =>
source.documentType === ApmDocumentType.TransactionMetric &&
source.hasDurationSummaryField
).length
).to.eql(3);
(source) => source.documentType !== ApmDocumentType.TransactionEvent
)
).to.eql([
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.TenMinutes,
hasDocs: false,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.OneMinute,
hasDocs: false,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.SixtyMinutes,
hasDocs: false,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.TenMinutes,
hasDocs: false,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
hasDocs: true,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.SixtyMinutes,
hasDocs: false,
hasDurationSummaryField: false,
},
]);
});
it('returns false when summary field is available inside but not outside the range', async () => {
it('returns support for all document types with duration summary field', async () => {
const response = await getTimeRangeMedata({
start: moment(withSummaryFieldStart).subtract(30, 'minutes'),
end: moment(withSummaryFieldEnd),
start: withSummaryFieldStart,
end: withSummaryFieldEnd,
});
expect(
response.sources.filter(
(source) =>
source.documentType === ApmDocumentType.TransactionMetric &&
!source.hasDurationSummaryField
).length
).to.eql(2);
(source) => source.documentType !== ApmDocumentType.TransactionEvent
)
).to.eql([
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.TenMinutes,
hasDocs: true,
hasDurationSummaryField: true,
},
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.OneMinute,
hasDocs: true,
hasDurationSummaryField: true,
},
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.SixtyMinutes,
hasDocs: true,
hasDurationSummaryField: true,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.TenMinutes,
hasDocs: true,
hasDurationSummaryField: true,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
hasDocs: true,
hasDurationSummaryField: true,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.SixtyMinutes,
hasDocs: true,
hasDurationSummaryField: true,
},
]);
});
it('returns support only for transaction 1m when timerange includes both new and legacy documents', async () => {
const response = await getTimeRangeMedata({
start: withoutSummaryFieldStart,
end: withSummaryFieldEnd,
});
expect(
response.sources.filter(
(source) => source.documentType !== ApmDocumentType.TransactionEvent
)
).to.eql([
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.TenMinutes,
hasDocs: false,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.OneMinute,
hasDocs: false,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.SixtyMinutes,
hasDocs: false,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.TenMinutes,
hasDocs: false,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
hasDocs: true,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.SixtyMinutes,
hasDocs: false,
hasDurationSummaryField: false,
},
]);
});
});
});
}
);
// FLAKY: https://github.com/elastic/kibana/issues/177601
registry.when(
'Time range metadata when generating data',
{ config: 'basic', archives: [] },
@ -400,8 +514,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
describe('when service metrics are only available in the current time range', () => {
before(async () => {
await es.deleteByQuery({
before(async () =>
es.deleteByQuery({
index: 'metrics-apm*',
query: {
bool: {
@ -414,7 +528,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
{
range: {
'@timestamp': {
lte: start.toISOString(),
gte: start.toISOString(),
},
},
},
@ -423,8 +537,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
},
refresh: true,
expand_wildcards: ['open', 'hidden'],
});
});
})
);
it('marks service transaction metrics as unavailable', async () => {
const response = await getTimeRangeMedata({
@ -495,7 +609,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
hasDocs: false,
hasDurationSummaryField: true,
hasDurationSummaryField: false,
},
{
documentType: ApmDocumentType.TransactionMetric,
@ -561,20 +675,7 @@ function getTransactionEvents({
];
const apmPipeline = (base: Readable) => {
const defaultPipeline = synthtrace.getDefaultPipeline()(
base
) as unknown as NodeJS.ReadableStream;
return pipeline(
defaultPipeline,
addObserverVersionTransform('8.5.0'),
deleteSummaryFieldTransform(),
(err) => {
if (err) {
logger.error(err);
}
}
);
return synthtrace.getDefaultPipeline({ versionOverride: '8.5.0' })(base);
};
return synthtrace.index(events, isLegacy ? apmPipeline : undefined);