[8.13] [APM] Fix condition to use serviceTransactionMetric documents (#180903) (#182851)

# Backport This will backport the following commits from `main` to `8.13`: - [[APM] Fix condition to use `serviceTransactionMetric` documents (#180903)](https://github.com/elastic/kibana/pull/180903)
2025-04-24 01:38:56 -04:00 · 2024-05-07 14:22:14 -03:00 · 2024-05-07 14:22:14 -03:00 · f119b71107
commit f119b71107
parent 61e5b3aa58
10 changed files with 273 additions and 184 deletions
--- a/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/apm_pipeline.ts
+++ b/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/apm_pipeline.ts
@ -6,11 +6,13 @@
 * Side Public License, v 1.
 */

+import semver from 'semver';
 import { PassThrough, pipeline, Readable } from 'stream';
 import { getDedotTransform } from '../../../shared/get_dedot_transform';
 import { getSerializeTransform } from '../../../shared/get_serialize_transform';
 import { Logger } from '../../../utils/create_logger';
 import { fork } from '../../../utils/stream_utils';
+import { deleteSummaryFieldTransform } from '../../../utils/transform_helpers';
 import { createBreakdownMetricsAggregator } from '../../aggregators/create_breakdown_metrics_aggregator';
 import { createServiceMetricsAggregator } from '../../aggregators/create_service_metrics_aggregator';
 import { createServiceSummaryMetricsAggregator } from '../../aggregators/create_service_summary_metrics_aggregator';
@ -22,22 +24,32 @@ import { getRoutingTransform } from './get_routing_transform';

 export function apmPipeline(logger: Logger, version: string, includeSerialization: boolean = true) {
  return (base: Readable) => {
+    const continousRollupSupported =
+      !version || semver.gte(semver.coerce(version)?.version ?? version, '8.7.0');
+
    const aggregators = [
      createTransactionMetricsAggregator('1m'),
-      createTransactionMetricsAggregator('10m'),
-      createTransactionMetricsAggregator('60m'),
-      createServiceMetricsAggregator('1m'),
-      createServiceMetricsAggregator('10m'),
-      createServiceMetricsAggregator('60m'),
-      createServiceSummaryMetricsAggregator('1m'),
-      createServiceSummaryMetricsAggregator('10m'),
-      createServiceSummaryMetricsAggregator('60m'),
      createSpanMetricsAggregator('1m'),
-      createSpanMetricsAggregator('10m'),
-      createSpanMetricsAggregator('60m'),
+      ...(continousRollupSupported
+        ? [
+            createTransactionMetricsAggregator('10m'),
+            createTransactionMetricsAggregator('60m'),
+            createServiceMetricsAggregator('1m'),
+            createServiceMetricsAggregator('10m'),
+            createServiceMetricsAggregator('60m'),
+            createServiceSummaryMetricsAggregator('1m'),
+            createServiceSummaryMetricsAggregator('10m'),
+            createServiceSummaryMetricsAggregator('60m'),
+            createSpanMetricsAggregator('10m'),
+            createSpanMetricsAggregator('60m'),
+          ]
+        : []),
    ];

    const serializationTransform = includeSerialization ? [getSerializeTransform()] : [];
+    const removeDurationSummaryTransform = !continousRollupSupported
+      ? [deleteSummaryFieldTransform()]
+      : [];

    return pipeline(
      // @ts-expect-error Some weird stuff here with the type definition for pipeline. We have tests!
@ -49,6 +61,7 @@ export function apmPipeline(logger: Logger, version: string, includeSerializatio
      getApmServerMetadataTransform(version),
      getRoutingTransform(),
      getDedotTransform(),
+      ...removeDurationSummaryTransform,
      (err) => {
        if (err) {
          logger.error(err);
--- a/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/index.ts
+++ b/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/index.ts
@ -65,7 +65,15 @@ export class ApmSynthtraceEsClient extends SynthtraceEsClient<ApmFields> {
    this.logger.info(`Updated component template: ${name}`);
  }

-  getDefaultPipeline(includeSerialization: boolean = true) {
-    return apmPipeline(this.logger, this.version, includeSerialization);
+  getDefaultPipeline(
+    {
+      includeSerialization,
+      versionOverride,
+    }: {
+      includeSerialization?: boolean;
+      versionOverride?: string;
+    } = { includeSerialization: true }
+  ) {
+    return apmPipeline(this.logger, versionOverride ?? this.version, includeSerialization);
  }
 }
--- a/packages/kbn-apm-synthtrace/src/scenarios/service_summary_field_version_dependent.ts
+++ b/packages/kbn-apm-synthtrace/src/scenarios/service_summary_field_version_dependent.ts
@ -8,36 +8,26 @@

 import { ApmFields, apm } from '@kbn/apm-synthtrace-client';
 import { random } from 'lodash';
-import { pipeline, Readable } from 'stream';
+import { Readable } from 'stream';
 import semver from 'semver';
 import { Scenario } from '../cli/scenario';
-import {
-  addObserverVersionTransform,
-  deleteSummaryFieldTransform,
-} from '../lib/utils/transform_helpers';
 import { withClient } from '../lib/utils/with_client';
+import { RunOptions } from '../cli/utils/parse_run_cli_flags';
+import { Logger } from '../lib/utils/create_logger';

-const scenario: Scenario<ApmFields> = async ({ logger, versionOverride }) => {
+const scenario: Scenario<ApmFields> = async ({
+  logger,
+  versionOverride,
+}: RunOptions & { logger: Logger }) => {
  const version = versionOverride as string;
-  const isLegacy = versionOverride && semver.lt(version, '8.7.0');
+  const isLegacy = version ? semver.lt(version as string, '8.7.0') : false;
  return {
    bootstrap: async ({ apmEsClient }) => {
      if (isLegacy) {
        apmEsClient.pipeline((base: Readable) => {
-          const defaultPipeline = apmEsClient.getDefaultPipeline()(
-            base
-          ) as unknown as NodeJS.ReadableStream;
-
-          return pipeline(
-            defaultPipeline,
-            addObserverVersionTransform(version),
-            deleteSummaryFieldTransform(),
-            (err) => {
-              if (err) {
-                logger.error(err);
-              }
-            }
-          );
+          return apmEsClient.getDefaultPipeline({
+            versionOverride: version,
+          })(base);
        });
      }
    },
@ -45,7 +35,7 @@ const scenario: Scenario<ApmFields> = async ({ logger, versionOverride }) => {
      const successfulTimestamps = range.ratePerMinute(6);
      const instance = apm
        .service({
-          name: `java${isLegacy ? '-legacy' : ''}`,
+          name: `java`,
          environment: 'production',
          agentName: 'java',
        })
--- a/x-pack/performance/services/synthtrace/index.ts
+++ b/x-pack/performance/services/synthtrace/index.ts
@ -51,7 +51,9 @@ export class SynthtraceClient {
      version: this.packageVersion,
    });

-    this.synthtraceEsClient.pipeline(this.synthtraceEsClient.getDefaultPipeline(false));
+    this.synthtraceEsClient.pipeline(
+      this.synthtraceEsClient.getDefaultPipeline({ includeSerialization: false })
+    );
  }

  async index(events: SynthtraceGenerator<ApmFields>) {
--- a/x-pack/plugins/apm/dev_docs/apm_queries.md
+++ b/x-pack/plugins/apm/dev_docs/apm_queries.md
@ -226,7 +226,7 @@ GET apm-*-metric-*,metrics-apm*/_search?terminate_after=1000

 Service transaction metrics are aggregated metric documents that hold latency and throughput metrics pivoted by `service.name`, `service.environment` and `transaction.type`. Additionally, `agent.name` and `service.language.name` are included as metadata.

-We use the response from the `GET /internal/apm/time_range_metadata` endpoint to determine what data source is available. A data source is considered available if there is either data before the current time range, or, if there is no data at all before the current time range, if there is data within the current time range. This means that existing deployments will use transaction metrics right after upgrading (instead of using service transaction metrics and seeing a mostly blank screen), but also that new deployments immediately get the benefits of service transaction metrics, instead of falling all the way back to transaction events.
+We use the response from the `GET /internal/apm/time_range_metadata` endpoint to determine what data source is available. Service transaction metrics docs, introduced in APM >= 8.7, is considered available if there is data before *and* within the current time range. This ensure the UI won't miss information shipped by APM < 8.7. For < 8.7 documents, availability is determined by whether there is data before the current time range, or no data at all before the current time range, but there is data within the current time range. This means that existing deployments will use transaction metrics right after upgrading (instead of using service transaction metrics and seeing a mostly blank screen), but also that new deployments immediately get the benefits of service transaction metrics, instead of falling all the way back to transaction events.

 A pre-aggregated document where `_doc_count` is the number of transaction events

--- a/x-pack/plugins/apm/ftr_e2e/setup_cypress_node_events.ts
+++ b/x-pack/plugins/apm/ftr_e2e/setup_cypress_node_events.ts
@ -28,7 +28,9 @@ export function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.Plugin
    version: config.env.APM_PACKAGE_VERSION,
  });

-  synthtraceEsClient.pipeline(synthtraceEsClient.getDefaultPipeline(false));
+  synthtraceEsClient.pipeline(
+    synthtraceEsClient.getDefaultPipeline({ includeSerialization: false })
+  );

  initPlugin(on, config);

--- a/x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts
+++ b/x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts
@ -11,22 +11,13 @@ import { RollupInterval } from '../../../common/rollup';
 import { APMEventClient } from './create_es_client/create_apm_event_client';
 import { getConfigForDocumentType } from './create_es_client/document_type';
 import { TimeRangeMetadata } from '../../../common/time_range_metadata';
-import { getDurationLegacyFilter } from './transactions';
+import { isDurationSummaryNotSupportedFilter } from './transactions';

 const QUERY_INDEX = {
-  BEFORE: 0,
-  CURRENT: 1,
-  DURATION_SUMMARY: 2,
+  DOCUMENT_TYPE: 0,
+  DURATION_SUMMARY_NOT_SUPPORTED: 1,
 } as const;

-interface DocumentTypeData {
-  documentType: ApmDocumentType;
-  rollupInterval: RollupInterval;
-  hasDocBefore: boolean;
-  hasDocAfter: boolean;
-  allHaveDurationSummary: boolean;
-}
-
 const getRequest = ({
  documentType,
  rollupInterval,
@ -93,10 +84,8 @@ export async function getDocumentSources({
    documentTypesToCheck,
  });

-  const hasAnySourceDocBefore = documentTypesInfo.some((source) => source.hasDocBefore);
-
  return [
-    ...mapToSources(documentTypesInfo, hasAnySourceDocBefore),
+    ...documentTypesInfo,
    {
      documentType: ApmDocumentType.TransactionEvent,
      rollupInterval: RollupInterval.None,
@ -120,7 +109,7 @@ const getDocumentTypesInfo = async ({
  kuery: string;
  enableContinuousRollups: boolean;
  documentTypesToCheck: ApmDocumentType[];
-}) => {
+}): Promise<TimeRangeMetadata['sources']> => {
  const getRequests = getDocumentTypeRequestsFn({
    enableContinuousRollups,
    start,
@ -131,25 +120,36 @@ const getDocumentTypesInfo = async ({
  const sourceRequests = documentTypesToCheck.flatMap(getRequests);

  const allSearches = sourceRequests
-    .flatMap(({ before, current, durationSummaryCheck }) => [before, current, durationSummaryCheck])
+    .flatMap(({ documentTypeQuery, durationSummaryNotSupportedQuery }) => [
+      documentTypeQuery,
+      durationSummaryNotSupportedQuery,
+    ])
    .filter((request): request is ReturnType<typeof getRequest> => request !== undefined);

  const allResponses = (await apmEventClient.msearch('get_document_availability', ...allSearches))
    .responses;

+  const hasAnyLegacyDocuments = sourceRequests.some(
+    ({ documentType, rollupInterval }, index) =>
+      isLegacyDocType(documentType, rollupInterval) &&
+      allResponses[index + QUERY_INDEX.DURATION_SUMMARY_NOT_SUPPORTED].hits.total.value > 0
+  );
+
  return sourceRequests.map(({ documentType, rollupInterval, ...queries }) => {
    const numberOfQueries = Object.values(queries).filter(Boolean).length;
    // allResponses is sorted by the order of the requests in sourceRequests
    const docTypeResponses = allResponses.splice(0, numberOfQueries);
+    const hasDocs = docTypeResponses[QUERY_INDEX.DOCUMENT_TYPE].hits.total.value > 0;
+    // can only use >=8.7 document types (ServiceTransactionMetrics or TransactionMetrics with 10m and 60m intervals)
+    // if there are no legacy documents
+    const canUseContinousRollupDocs = hasDocs && !hasAnyLegacyDocuments;

    return {
      documentType,
      rollupInterval,
-      hasDocBefore: docTypeResponses[QUERY_INDEX.BEFORE].hits.total.value > 0,
-      hasDocAfter: docTypeResponses[QUERY_INDEX.CURRENT].hits.total.value > 0,
-      allHaveDurationSummary: docTypeResponses[QUERY_INDEX.DURATION_SUMMARY]
-        ? docTypeResponses[QUERY_INDEX.DURATION_SUMMARY].hits.total.value === 0
-        : true,
+      hasDocs: isLegacyDocType(documentType, rollupInterval) ? hasDocs : canUseContinousRollupDocs,
+      // all >=8.7 document types with rollups support duration summary
+      hasDurationSummaryField: canUseContinousRollupDocs,
    };
  });
 };
@ -168,9 +168,7 @@ const getDocumentTypeRequestsFn =
  }) =>
  (documentType: ApmDocumentType) => {
    const currentRange = rangeQuery(start, end);
-    const diff = end - start;
    const kql = kqlQuery(kuery);
-    const beforeRange = rangeQuery(start - diff, end - diff);

    const rollupIntervals = enableContinuousRollups
      ? getConfigForDocumentType(documentType).rollupIntervals
@ -179,48 +177,26 @@ const getDocumentTypeRequestsFn =
    return rollupIntervals.map((rollupInterval) => ({
      documentType,
      rollupInterval,
-      before: getRequest({
-        documentType,
-        rollupInterval,
-        filters: [...kql, ...beforeRange],
-      }),
-      current: getRequest({
+      documentTypeQuery: getRequest({
        documentType,
        rollupInterval,
        filters: [...kql, ...currentRange],
      }),
-      ...(documentType !== ApmDocumentType.ServiceTransactionMetric
+      ...(isLegacyDocType(documentType, rollupInterval)
        ? {
-            durationSummaryCheck: getRequest({
+            durationSummaryNotSupportedQuery: getRequest({
              documentType,
              rollupInterval,
-              filters: [...kql, ...currentRange, getDurationLegacyFilter()],
+              filters: [...kql, ...currentRange, isDurationSummaryNotSupportedFilter()],
            }),
          }
-        : {}),
+        : undefined),
    }));
  };

-const mapToSources = (sources: DocumentTypeData[], hasAnySourceDocBefore: boolean) => {
-  return sources.map((source) => {
-    const { documentType, hasDocAfter, hasDocBefore, rollupInterval, allHaveDurationSummary } =
-      source;
-
-    const hasDocBeforeOrAfter = hasDocBefore || hasDocAfter;
-
-    // If there is any data before, we require that data is available before
-    // this time range to mark this source as available. If we don't do that,
-    // users that upgrade to a version that starts generating service tx metrics
-    // will see a mostly empty screen for a while after upgrading.
-    // If we only check before, users with a new deployment will use raw transaction
-    // events.
-    const hasDocs = hasAnySourceDocBefore ? hasDocBefore : hasDocBeforeOrAfter;
-
-    return {
-      documentType,
-      rollupInterval,
-      hasDocs,
-      hasDurationSummaryField: allHaveDurationSummary,
-    };
-  });
+const isLegacyDocType = (documentType: ApmDocumentType, rollupInterval: RollupInterval) => {
+  return (
+    documentType === ApmDocumentType.TransactionMetric &&
+    rollupInterval === RollupInterval.OneMinute
+  );
 };
--- a/x-pack/plugins/apm/server/lib/helpers/transactions/index.ts
+++ b/x-pack/plugins/apm/server/lib/helpers/transactions/index.ts
@ -164,17 +164,10 @@ export function isRootTransaction(searchAggregatedTransactions: boolean) {
      };
 }

-export function getDurationLegacyFilter(): QueryDslQueryContainer {
+export function isDurationSummaryNotSupportedFilter(): QueryDslQueryContainer {
  return {
    bool: {
-      must: [
-        {
-          bool: {
-            filter: [{ exists: { field: TRANSACTION_DURATION_HISTOGRAM } }],
-            must_not: [{ exists: { field: TRANSACTION_DURATION_SUMMARY } }],
-          },
-        },
-      ],
+      must_not: [{ exists: { field: TRANSACTION_DURATION_SUMMARY } }],
    },
  };
 }
--- a/x-pack/test/apm_api_integration/tests/time_range_metadata/many_apm_server_versions.spec.ts
+++ b/x-pack/test/apm_api_integration/tests/time_range_metadata/many_apm_server_versions.spec.ts
@ -7,11 +7,7 @@
 import expect from '@kbn/expect';
 import { apm, timerange } from '@kbn/apm-synthtrace-client';
 import moment from 'moment';
-import {
-  addObserverVersionTransform,
-  ApmSynthtraceEsClient,
-  deleteSummaryFieldTransform,
-} from '@kbn/apm-synthtrace';
+import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
 import {
  TRANSACTION_DURATION_HISTOGRAM,
  TRANSACTION_DURATION_SUMMARY,
@ -19,7 +15,7 @@ import {
 import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
 import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
 import { LatencyAggregationType } from '@kbn/apm-plugin/common/latency_aggregation_types';
-import { pipeline, Readable } from 'stream';
+import { Readable } from 'stream';
 import { FtrProviderContext } from '../../common/ftr_provider_context';
 import { ApmApiClient } from '../../common/config';

@ -32,7 +28,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
  const baseTime = new Date('2023-10-01T00:00:00.000Z').getTime();
  const startLegacy = moment(baseTime).add(0, 'minutes');
  const start = moment(baseTime).add(5, 'minutes');
-  const end = moment(baseTime).add(10, 'minutes');
+  const endLegacy = moment(baseTime).add(10, 'minutes');
+  const end = moment(baseTime).add(15, 'minutes');

  registry.when(
    'Time range metadata when there are multiple APM Server versions',
@ -43,7 +40,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
          await generateTraceDataForService({
            serviceName: 'synth-java-legacy',
            start: startLegacy,
-            end,
+            end: endLegacy,
            isLegacy: true,
            synthtrace,
          });
@ -77,7 +74,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
          });

          // @ts-expect-error
-          expect(res.hits.total.value).to.be(10);
+          expect(res.hits.total.value).to.be(20);
        });

        it('ingests transaction metrics without transaction.duration.summary', async () => {
@ -94,7 +91,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
          });

          // @ts-expect-error
-          expect(res.hits.total.value).to.be(20);
+          expect(res.hits.total.value).to.be(10);
        });

        it('has transaction.duration.summary field for every document type', async () => {
@ -102,7 +99,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
            endpoint: 'GET /internal/apm/time_range_metadata',
            params: {
              query: {
-                start: start.toISOString(),
+                start: endLegacy.toISOString(),
                end: end.toISOString(),
                enableContinuousRollups: true,
                enableServiceTransactionMetrics: true,
@ -115,8 +112,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
          const allHasSummaryField = response.body.sources
            .filter(
              (source) =>
-                source.documentType === ApmDocumentType.TransactionMetric &&
-                source.rollupInterval !== RollupInterval.OneMinute
+                source.documentType !== ApmDocumentType.TransactionEvent &&
+                source.rollupInterval !== RollupInterval.SixtyMinutes // there is not enough data for 60 minutes
            )
            .every((source) => {
              return source.hasDurationSummaryField;
@ -131,7 +128,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
            params: {
              query: {
                start: startLegacy.toISOString(),
-                end: end.toISOString(),
+                end: endLegacy.toISOString(),
                enableContinuousRollups: true,
                enableServiceTransactionMetrics: true,
                useSpanName: false,
@ -147,11 +144,35 @@ export default function ApiTest({ getService }: FtrProviderContext) {
          expect(allHasSummaryField).to.eql(false);
        });

+        it('does not support transaction.duration.summary for transactionMetric 1m when not all documents within the range support it ', async () => {
+          const response = await apmApiClient.readUser({
+            endpoint: 'GET /internal/apm/time_range_metadata',
+            params: {
+              query: {
+                start: startLegacy.toISOString(),
+                end: end.toISOString(),
+                enableContinuousRollups: true,
+                enableServiceTransactionMetrics: true,
+                useSpanName: false,
+                kuery: '',
+              },
+            },
+          });
+
+          const hasDurationSummaryField = response.body.sources.find(
+            (source) =>
+              source.documentType === ApmDocumentType.TransactionMetric &&
+              source.rollupInterval === RollupInterval.OneMinute // there is not enough data for 60 minutes in the timerange defined for the tests
+          )?.hasDurationSummaryField;
+
+          expect(hasDurationSummaryField).to.eql(false);
+        });
+
        it('does not have latency data for synth-java-legacy', async () => {
          const res = await getLatencyChartForService({
            serviceName: 'synth-java-legacy',
            start,
-            end,
+            end: endLegacy,
            apmApiClient,
            useDurationSummary: true,
          });
@ -170,18 +191,13 @@ export default function ApiTest({ getService }: FtrProviderContext) {
          const res = await getLatencyChartForService({
            serviceName: 'synth-java',
            start,
-            end,
+            end: endLegacy,
            apmApiClient,
            useDurationSummary: true,
          });

          expect(res.body.currentPeriod.latencyTimeseries.map(({ y }) => y)).to.eql([
-            1000000,
-            1000000,
-            1000000,
-            1000000,
-            1000000,
-            null,
+            1000000, 1000000, 1000000, 1000000, 1000000, 1000000,
          ]);
        });
      });
@ -255,21 +271,7 @@ function generateTraceDataForService({
    );

  const apmPipeline = (base: Readable) => {
-    const defaultPipeline = synthtrace.getDefaultPipeline()(
-      base
-    ) as unknown as NodeJS.ReadableStream;
-
-    return pipeline(
-      defaultPipeline,
-      addObserverVersionTransform('8.5.0'),
-      deleteSummaryFieldTransform(),
-      (err) => {
-        if (err) {
-          // eslint-disable-next-line no-console
-          console.error(err);
-        }
-      }
-    );
+    return synthtrace.getDefaultPipeline({ versionOverride: '8.5.0' })(base);
  };

  return synthtrace.index(events, isLegacy ? apmPipeline : undefined);
--- a/x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts
+++ b/x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts
@ -11,12 +11,8 @@ import { omit, sortBy } from 'lodash';
 import moment, { Moment } from 'moment';
 import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
 import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
-import {
-  addObserverVersionTransform,
-  ApmSynthtraceEsClient,
-  deleteSummaryFieldTransform,
-} from '@kbn/apm-synthtrace';
-import { Readable, pipeline } from 'stream';
+import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
+import { Readable } from 'stream';
 import { ToolingLog } from '@kbn/tooling-log';
 import { FtrProviderContext } from '../../common/ftr_provider_context';

@ -79,7 +75,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
  });

  registry.when(
-    'Time range metadata when generating summary data',
+    'Time range metadata when generating data with multiple APM server versions',
    { config: 'basic', archives: [] },
    () => {
      describe('data loaded with and without summary field', () => {
@ -87,7 +83,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
        const withoutSummaryFieldEnd = moment(withoutSummaryFieldStart).add(2, 'hours');

        const withSummaryFieldStart = moment(withoutSummaryFieldEnd);
-        const withSummaryFieldEnd = moment(withoutSummaryFieldEnd).add(2, 'hours');
+        const withSummaryFieldEnd = moment(withSummaryFieldStart).add(2, 'hours');

        before(async () => {
          await getTransactionEvents({
@ -111,35 +107,155 @@ export default function ApiTest({ getService }: FtrProviderContext) {
          return synthtraceEsClient.clean();
        });

-        describe('Values for hasDurationSummaryField for transaction metrics', () => {
-          it('returns true when summary field is available both inside and outside the range', async () => {
+        describe('aggregators and summary field support', () => {
+          it('returns support only for legacy transactionMetrics 1m without duration summary field', async () => {
            const response = await getTimeRangeMedata({
-              start: moment(withSummaryFieldStart).add(1, 'hour'),
-              end: moment(withSummaryFieldEnd),
+              start: withoutSummaryFieldStart,
+              end: withoutSummaryFieldEnd,
            });

            expect(
              response.sources.filter(
-                (source) =>
-                  source.documentType === ApmDocumentType.TransactionMetric &&
-                  source.hasDurationSummaryField
-              ).length
-            ).to.eql(3);
+                (source) => source.documentType !== ApmDocumentType.TransactionEvent
+              )
+            ).to.eql([
+              {
+                documentType: ApmDocumentType.ServiceTransactionMetric,
+                rollupInterval: RollupInterval.TenMinutes,
+                hasDocs: false,
+                hasDurationSummaryField: false,
+              },
+              {
+                documentType: ApmDocumentType.ServiceTransactionMetric,
+                rollupInterval: RollupInterval.OneMinute,
+                hasDocs: false,
+                hasDurationSummaryField: false,
+              },
+              {
+                documentType: ApmDocumentType.ServiceTransactionMetric,
+                rollupInterval: RollupInterval.SixtyMinutes,
+                hasDocs: false,
+                hasDurationSummaryField: false,
+              },
+              {
+                documentType: ApmDocumentType.TransactionMetric,
+                rollupInterval: RollupInterval.TenMinutes,
+                hasDocs: false,
+                hasDurationSummaryField: false,
+              },
+              {
+                documentType: ApmDocumentType.TransactionMetric,
+                rollupInterval: RollupInterval.OneMinute,
+                hasDocs: true,
+                hasDurationSummaryField: false,
+              },
+              {
+                documentType: ApmDocumentType.TransactionMetric,
+                rollupInterval: RollupInterval.SixtyMinutes,
+                hasDocs: false,
+                hasDurationSummaryField: false,
+              },
+            ]);
          });

-          it('returns false when summary field is available inside but not outside the range', async () => {
+          it('returns support for all document types with duration summary field', async () => {
            const response = await getTimeRangeMedata({
-              start: moment(withSummaryFieldStart).subtract(30, 'minutes'),
-              end: moment(withSummaryFieldEnd),
+              start: withSummaryFieldStart,
+              end: withSummaryFieldEnd,
            });

            expect(
              response.sources.filter(
-                (source) =>
-                  source.documentType === ApmDocumentType.TransactionMetric &&
-                  !source.hasDurationSummaryField
-              ).length
-            ).to.eql(2);
+                (source) => source.documentType !== ApmDocumentType.TransactionEvent
+              )
+            ).to.eql([
+              {
+                documentType: ApmDocumentType.ServiceTransactionMetric,
+                rollupInterval: RollupInterval.TenMinutes,
+                hasDocs: true,
+                hasDurationSummaryField: true,
+              },
+              {
+                documentType: ApmDocumentType.ServiceTransactionMetric,
+                rollupInterval: RollupInterval.OneMinute,
+                hasDocs: true,
+                hasDurationSummaryField: true,
+              },
+              {
+                documentType: ApmDocumentType.ServiceTransactionMetric,
+                rollupInterval: RollupInterval.SixtyMinutes,
+                hasDocs: true,
+                hasDurationSummaryField: true,
+              },
+              {
+                documentType: ApmDocumentType.TransactionMetric,
+                rollupInterval: RollupInterval.TenMinutes,
+                hasDocs: true,
+                hasDurationSummaryField: true,
+              },
+              {
+                documentType: ApmDocumentType.TransactionMetric,
+                rollupInterval: RollupInterval.OneMinute,
+                hasDocs: true,
+                hasDurationSummaryField: true,
+              },
+              {
+                documentType: ApmDocumentType.TransactionMetric,
+                rollupInterval: RollupInterval.SixtyMinutes,
+                hasDocs: true,
+                hasDurationSummaryField: true,
+              },
+            ]);
+          });
+
+          it('returns support only for transaction 1m when timerange includes both new and legacy documents', async () => {
+            const response = await getTimeRangeMedata({
+              start: withoutSummaryFieldStart,
+              end: withSummaryFieldEnd,
+            });
+
+            expect(
+              response.sources.filter(
+                (source) => source.documentType !== ApmDocumentType.TransactionEvent
+              )
+            ).to.eql([
+              {
+                documentType: ApmDocumentType.ServiceTransactionMetric,
+                rollupInterval: RollupInterval.TenMinutes,
+                hasDocs: false,
+                hasDurationSummaryField: false,
+              },
+              {
+                documentType: ApmDocumentType.ServiceTransactionMetric,
+                rollupInterval: RollupInterval.OneMinute,
+                hasDocs: false,
+                hasDurationSummaryField: false,
+              },
+              {
+                documentType: ApmDocumentType.ServiceTransactionMetric,
+                rollupInterval: RollupInterval.SixtyMinutes,
+                hasDocs: false,
+                hasDurationSummaryField: false,
+              },
+              {
+                documentType: ApmDocumentType.TransactionMetric,
+                rollupInterval: RollupInterval.TenMinutes,
+                hasDocs: false,
+                hasDurationSummaryField: false,
+              },
+              {
+                documentType: ApmDocumentType.TransactionMetric,
+                rollupInterval: RollupInterval.OneMinute,
+                hasDocs: true,
+                hasDurationSummaryField: false,
+              },
+              {
+                documentType: ApmDocumentType.TransactionMetric,
+                rollupInterval: RollupInterval.SixtyMinutes,
+                hasDocs: false,
+                hasDurationSummaryField: false,
+              },
+            ]);
          });
        });
      });
@ -398,8 +514,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
      });

      describe('when service metrics are only available in the current time range', () => {
-        before(async () => {
-          await es.deleteByQuery({
+        before(async () =>
+          es.deleteByQuery({
            index: 'metrics-apm*',
            query: {
              bool: {
@ -412,7 +528,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
                  {
                    range: {
                      '@timestamp': {
-                        lte: start.toISOString(),
+                        gte: start.toISOString(),
                      },
                    },
                  },
@ -421,8 +537,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
            },
            refresh: true,
            expand_wildcards: ['open', 'hidden'],
-          });
-        });
+          })
+        );

        it('marks service transaction metrics as unavailable', async () => {
          const response = await getTimeRangeMedata({
@ -493,7 +609,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
              documentType: ApmDocumentType.TransactionMetric,
              rollupInterval: RollupInterval.OneMinute,
              hasDocs: false,
-              hasDurationSummaryField: true,
+              hasDurationSummaryField: false,
            },
            {
              documentType: ApmDocumentType.TransactionMetric,
@ -559,20 +675,7 @@ function getTransactionEvents({
  ];

  const apmPipeline = (base: Readable) => {
-    const defaultPipeline = synthtrace.getDefaultPipeline()(
-      base
-    ) as unknown as NodeJS.ReadableStream;
-
-    return pipeline(
-      defaultPipeline,
-      addObserverVersionTransform('8.5.0'),
-      deleteSummaryFieldTransform(),
-      (err) => {
-        if (err) {
-          logger.error(err);
-        }
-      }
-    );
+    return synthtrace.getDefaultPipeline({ versionOverride: '8.5.0' })(base);
  };

  return synthtrace.index(events, isLegacy ? apmPipeline : undefined);