From a9dda8c81906a7abc98c371a362a5f3dfb0354ed Mon Sep 17 00:00:00 2001 From: jennypavlova Date: Tue, 11 Mar 2025 13:50:15 +0100 Subject: [PATCH] [8.x] [APM] Fix: Span Links with OTel data (#212806) (#213881) # Backport This will backport the following commits from `main` to `8.x`: - [[APM] Fix: Span Links with OTel data (#212806)](https://github.com/elastic/kibana/pull/212806) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../shared/kbn-apm-types/src/es_fields/apm.ts | 5 ++ .../__snapshots__/es_fields.test.ts.snap | 12 +++++ .../routes/span_links/get_linked_children.ts | 48 +++++++++++++++++-- .../routes/span_links/get_linked_parents.ts | 34 +++++++++++-- .../server/routes/traces/get_trace_items.ts | 19 +++++++- 5 files changed, 107 insertions(+), 11 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-apm-types/src/es_fields/apm.ts b/x-pack/platform/packages/shared/kbn-apm-types/src/es_fields/apm.ts index 5fd1ff31c17c..24d0f88542ff 100644 --- a/x-pack/platform/packages/shared/kbn-apm-types/src/es_fields/apm.ts +++ b/x-pack/platform/packages/shared/kbn-apm-types/src/es_fields/apm.ts @@ -194,6 +194,11 @@ export const TELEMETRY_SDK_NAME = 'telemetry.sdk.name'; export const TELEMETRY_SDK_LANGUAGE = 'telemetry.sdk.language'; export const TELEMETRY_SDK_VERSION = 'telemetry.sdk.version'; +// OpenTelemetry span links + +export const LINKS_SPAN_ID = 'links.span_id'; +export const LINKS_TRACE_ID = 'links.trace_id'; + // Metadata export const TIER = '_tier'; export const INDEX = '_index'; diff --git a/x-pack/solutions/observability/plugins/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap b/x-pack/solutions/observability/plugins/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap index c45bee0a51be..15e3d35299b8 100644 --- a/x-pack/solutions/observability/plugins/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap +++ b/x-pack/solutions/observability/plugins/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap @@ -178,6 +178,10 @@ exports[`Error LABEL_TELEMETRY_AUTO_VERSION 1`] = `undefined`; exports[`Error LABEL_TYPE 1`] = `undefined`; +exports[`Error LINKS_SPAN_ID 1`] = `undefined`; + +exports[`Error LINKS_TRACE_ID 1`] = `undefined`; + exports[`Error LOG_LEVEL 1`] = `undefined`; exports[`Error METRIC_CGROUP_MEMORY_LIMIT_BYTES 1`] = `undefined`; @@ -545,6 +549,10 @@ exports[`Span LABEL_TELEMETRY_AUTO_VERSION 1`] = `undefined`; exports[`Span LABEL_TYPE 1`] = `undefined`; +exports[`Span LINKS_SPAN_ID 1`] = `undefined`; + +exports[`Span LINKS_TRACE_ID 1`] = `undefined`; + exports[`Span LOG_LEVEL 1`] = `undefined`; exports[`Span METRIC_CGROUP_MEMORY_LIMIT_BYTES 1`] = `undefined`; @@ -922,6 +930,10 @@ exports[`Transaction LABEL_TELEMETRY_AUTO_VERSION 1`] = `undefined`; exports[`Transaction LABEL_TYPE 1`] = `undefined`; +exports[`Transaction LINKS_SPAN_ID 1`] = `undefined`; + +exports[`Transaction LINKS_TRACE_ID 1`] = `undefined`; + exports[`Transaction LOG_LEVEL 1`] = `undefined`; exports[`Transaction METRIC_CGROUP_MEMORY_LIMIT_BYTES 1`] = `undefined`; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/span_links/get_linked_children.ts b/x-pack/solutions/observability/plugins/apm/server/routes/span_links/get_linked_children.ts index b78585bffc3e..6b94ba9e0fa1 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/span_links/get_linked_children.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/span_links/get_linked_children.ts @@ -4,10 +4,11 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { rangeQuery } from '@kbn/observability-plugin/server'; +import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { isEmpty } from 'lodash'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import type { SpanLink } from '@kbn/apm-types'; import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { PROCESSOR_EVENT, @@ -17,6 +18,8 @@ import { SPAN_LINKS_SPAN_ID, TRACE_ID, TRANSACTION_ID, + LINKS_TRACE_ID, + LINKS_SPAN_ID, } from '../../../common/es_fields/apm'; import { getBufferedTimerange } from './utils'; import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; @@ -40,7 +43,12 @@ async function fetchLinkedChildrenOfSpan({ }); const requiredFields = asMutableArray([TRACE_ID, PROCESSOR_EVENT] as const); - const optionalFields = asMutableArray([SPAN_ID, TRANSACTION_ID] as const); + const optionalFields = asMutableArray([ + SPAN_ID, + TRANSACTION_ID, + LINKS_SPAN_ID, + LINKS_TRACE_ID, + ] as const); const response = await apmEventClient.search('fetch_linked_children_of_span', { apm: { @@ -55,8 +63,28 @@ async function fetchLinkedChildrenOfSpan({ bool: { filter: [ ...rangeQuery(startWithBuffer, endWithBuffer), - { term: { [SPAN_LINKS_TRACE_ID]: traceId } }, - ...(spanId ? [{ term: { [SPAN_LINKS_SPAN_ID]: spanId } }] : []), + { + bool: { + minimum_should_match: 1, + should: [ + ...termQuery(SPAN_LINKS_TRACE_ID, traceId), + ...termQuery(LINKS_TRACE_ID, traceId), + ], + }, + }, + ...(spanId + ? [ + { + bool: { + minimum_should_match: 1, + should: [ + ...termQuery(SPAN_LINKS_SPAN_ID, spanId), + ...termQuery(LINKS_SPAN_ID, spanId), + ], + }, + }, + ] + : []), ], }, }, @@ -67,11 +95,21 @@ async function fetchLinkedChildrenOfSpan({ const source = 'span' in hit._source ? hit._source : undefined; const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + const spanLinks: SpanLink[] = + event.links?.span_id && event.links?.trace_id + ? [ + { + span: { id: event.links.span_id as string }, + trace: { id: event.links.trace_id as string }, + }, + ] + : []; + return { ...event, span: { ...event.span, - links: source?.span?.links ?? [], + links: source?.span?.links ?? spanLinks ?? [], }, }; }); diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/span_links/get_linked_parents.ts b/x-pack/solutions/observability/plugins/apm/server/routes/span_links/get_linked_parents.ts index 741fd80b6eba..3344ede885ae 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/span_links/get_linked_parents.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/span_links/get_linked_parents.ts @@ -4,14 +4,18 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { rangeQuery } from '@kbn/observability-plugin/server'; +import { existsQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { SPAN_ID, SPAN_LINKS, TRACE_ID, TRANSACTION_ID, PROCESSOR_EVENT, + LINKS_SPAN_ID, + LINKS_TRACE_ID, } from '../../../common/es_fields/apm'; import type { SpanRaw } from '../../../typings/es_schemas/raw/span_raw'; import type { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; @@ -32,12 +36,19 @@ export async function getLinkedParentsOfSpan({ end: number; processorEvent: ProcessorEvent; }) { + const optionalFields = asMutableArray([LINKS_SPAN_ID, LINKS_TRACE_ID] as const); + const events = + processorEvent === ProcessorEvent.span + ? [ProcessorEvent.span] + : [processorEvent, ProcessorEvent.span]; + const response = await apmEventClient.search('get_linked_parents_of_span', { apm: { - events: [processorEvent], + events, }, _source: [SPAN_LINKS], body: { + fields: [...optionalFields], track_total_hits: false, size: 1, query: { @@ -45,7 +56,12 @@ export async function getLinkedParentsOfSpan({ filter: [ ...rangeQuery(start, end), { term: { [TRACE_ID]: traceId } }, - { exists: { field: SPAN_LINKS } }, + { + bool: { + should: [...existsQuery(SPAN_LINKS), ...existsQuery(LINKS_SPAN_ID)], + minimum_should_match: 1, + }, + }, { term: { [PROCESSOR_EVENT]: processorEvent } }, ...(processorEvent === ProcessorEvent.transaction ? [{ term: { [TRANSACTION_ID]: spanId } }] @@ -57,6 +73,16 @@ export async function getLinkedParentsOfSpan({ }); const source = response.hits.hits?.[0]?._source as Pick; + const fields = unflattenKnownApmEventFields(response.hits.hits?.[0]?.fields); + const spanLinksFromFields = + fields?.links?.span_id && fields?.links?.trace_id + ? [ + { + trace: { id: fields.links.trace_id as string }, + span: { id: fields.links.span_id as string }, + }, + ] + : []; - return source?.span?.links || []; + return source?.span?.links ?? spanLinksFromFields; } diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/traces/get_trace_items.ts b/x-pack/solutions/observability/plugins/apm/server/routes/traces/get_trace_items.ts index 4f6222f9fb8a..9da71ddb0ea3 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/traces/get_trace_items.ts @@ -12,6 +12,7 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; import { last, omit } from 'lodash'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import type { SpanLink } from '@kbn/apm-types'; import { asMutableArray } from '../../../common/utils/as_mutable_array'; import type { APMConfig } from '../..'; import { @@ -27,6 +28,8 @@ import { ERROR_LOG_MESSAGE, EVENT_OUTCOME, FAAS_COLDSTART, + LINKS_SPAN_ID, + LINKS_TRACE_ID, PARENT_ID, PROCESSOR_EVENT, SERVICE_ENVIRONMENT, @@ -301,6 +304,8 @@ async function getTraceDocsPerPage({ SPAN_COMPOSITE_SUM, SPAN_SYNC, CHILD_ID, + LINKS_SPAN_ID, + LINKS_TRACE_ID, ] as const); const body = { @@ -347,6 +352,16 @@ async function getTraceDocsPerPage({ return { hits: res.hits.hits.map((hit) => { const sort = hit.sort; + const fields = unflattenKnownApmEventFields(hit?.fields); + const spanLinksFromFields: SpanLink[] = + fields?.links?.span_id && fields?.links?.trace_id + ? [ + { + trace: { id: fields.links.trace_id as string }, + span: { id: fields.links.span_id as string }, + }, + ] + : []; const spanLinksSource = 'span' in hit._source ? hit._source.span?.links : undefined; if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) { @@ -365,7 +380,7 @@ async function getTraceDocsPerPage({ composite: spanEvent.span.composite ? (spanEvent.span.composite as Required['composite']) : undefined, - links: spanLinksSource, + links: spanLinksSource ?? spanLinksFromFields, }, ...(spanEvent.child ? { child: spanEvent.child as WaterfallSpan['child'] } : {}), }; @@ -384,7 +399,7 @@ async function getTraceDocsPerPage({ }, span: { ...txEvent.span, - links: spanLinksSource, + links: spanLinksSource ?? spanLinksFromFields, }, };