[8.16] [APM] Fix: Span Links with OTel data (#212806) (#213889)

# Backport

This will backport the following commits from `main` to `8.16`:
- [[APM] Fix: Span Links with OTel data
(#212806)](https://github.com/elastic/kibana/pull/212806)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT
[{"author":{"name":"jennypavlova","email":"dzheni.pavlova@elastic.co"},"sourceCommit":{"committedDate":"2025-03-10T18:47:07Z","message":"[APM]
Fix: Span Links with OTel data (#212806)\n\nCloses #212796 \n\nThis PR
fixes OTel span links\n\n## Testing\n\n- Using the edge oblt go to
Service inventory and find the `accounting`\nservice and click on it\n-
Click on the transactions tab and scroll to the waterfall\n- Click on
the Span link inside the Span\n- Navigate to the linked service \n- Then
try the navigation back by finding the span link again in the\nwaterfall
and go back to the accounting service / its
trace\n\n\n\n\nhttps://github.com/user-attachments/assets/83bb6ec3-86d5-45ad-8b81-6df73751fa31\n\n---------\n\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"662c30a260f676c9e1180e7d413553d0166726d7","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","backport:prev-minor","backport:prev-major","Team:obs-ux-infra_services","v9.1.0"],"title":"[APM]
Fix: Span Links with OTel
data","number":212806,"url":"https://github.com/elastic/kibana/pull/212806","mergeCommit":{"message":"[APM]
Fix: Span Links with OTel data (#212806)\n\nCloses #212796 \n\nThis PR
fixes OTel span links\n\n## Testing\n\n- Using the edge oblt go to
Service inventory and find the `accounting`\nservice and click on it\n-
Click on the transactions tab and scroll to the waterfall\n- Click on
the Span link inside the Span\n- Navigate to the linked service \n- Then
try the navigation back by finding the span link again in the\nwaterfall
and go back to the accounting service / its
trace\n\n\n\n\nhttps://github.com/user-attachments/assets/83bb6ec3-86d5-45ad-8b81-6df73751fa31\n\n---------\n\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"662c30a260f676c9e1180e7d413553d0166726d7"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/212806","number":212806,"mergeCommit":{"message":"[APM]
Fix: Span Links with OTel data (#212806)\n\nCloses #212796 \n\nThis PR
fixes OTel span links\n\n## Testing\n\n- Using the edge oblt go to
Service inventory and find the `accounting`\nservice and click on it\n-
Click on the transactions tab and scroll to the waterfall\n- Click on
the Span link inside the Span\n- Navigate to the linked service \n- Then
try the navigation back by finding the span link again in the\nwaterfall
and go back to the accounting service / its
trace\n\n\n\n\nhttps://github.com/user-attachments/assets/83bb6ec3-86d5-45ad-8b81-6df73751fa31\n\n---------\n\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"662c30a260f676c9e1180e7d413553d0166726d7"}}]}]
BACKPORT-->

packages/kbn-apm-types/src/es_fields/apm.ts

@@ -192,6 +192,11 @@ export const METRIC_OTEL_JVM_GC_DURATION = 'process.runtime.jvm.gc.duration';
 export const VALUE_OTEL_JVM_PROCESS_MEMORY_HEAP = 'heap';
 export const VALUE_OTEL_JVM_PROCESS_MEMORY_NON_HEAP = 'non_heap';
 
+// OpenTelemetry span links
+
+export const LINKS_SPAN_ID = 'links.span_id';
+export const LINKS_TRACE_ID = 'links.trace_id';
+
 // Metadata
 export const TIER = '_tier';
 export const INDEX = '_index';

x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap

@@ -178,6 +178,10 @@ exports[`Error LABEL_TELEMETRY_AUTO_VERSION 1`] = `undefined`;
 
 exports[`Error LABEL_TYPE 1`] = `undefined`;
 
+exports[`Error LINKS_SPAN_ID 1`] = `undefined`;
+
+exports[`Error LINKS_TRACE_ID 1`] = `undefined`;
+
 exports[`Error LOG_LEVEL 1`] = `undefined`;
 
 exports[`Error METRIC_CGROUP_MEMORY_LIMIT_BYTES 1`] = `undefined`;
@@ -539,6 +543,10 @@ exports[`Span LABEL_TELEMETRY_AUTO_VERSION 1`] = `undefined`;
 
 exports[`Span LABEL_TYPE 1`] = `undefined`;
 
+exports[`Span LINKS_SPAN_ID 1`] = `undefined`;
+
+exports[`Span LINKS_TRACE_ID 1`] = `undefined`;
+
 exports[`Span LOG_LEVEL 1`] = `undefined`;
 
 exports[`Span METRIC_CGROUP_MEMORY_LIMIT_BYTES 1`] = `undefined`;
@@ -910,6 +918,10 @@ exports[`Transaction LABEL_TELEMETRY_AUTO_VERSION 1`] = `undefined`;
 
 exports[`Transaction LABEL_TYPE 1`] = `undefined`;
 
+exports[`Transaction LINKS_SPAN_ID 1`] = `undefined`;
+
+exports[`Transaction LINKS_TRACE_ID 1`] = `undefined`;
+
 exports[`Transaction LOG_LEVEL 1`] = `undefined`;
 
 exports[`Transaction METRIC_CGROUP_MEMORY_LIMIT_BYTES 1`] = `undefined`;

x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts

@@ -4,10 +4,11 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { rangeQuery } from '@kbn/observability-plugin/server';
+import { rangeQuery, termQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { isEmpty } from 'lodash';
 import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import type { SpanLink } from '@kbn/apm-types';
 import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import {
   PROCESSOR_EVENT,
@@ -17,9 +18,11 @@ import {
   SPAN_LINKS_SPAN_ID,
   TRACE_ID,
   TRANSACTION_ID,
+  LINKS_TRACE_ID,
+  LINKS_SPAN_ID,
 } from '../../../common/es_fields/apm';
 import { getBufferedTimerange } from './utils';
-import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
+import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 
 async function fetchLinkedChildrenOfSpan({
   traceId,
@@ -40,7 +43,12 @@ async function fetchLinkedChildrenOfSpan({
   });
 
   const requiredFields = asMutableArray([TRACE_ID, PROCESSOR_EVENT] as const);
-  const optionalFields = asMutableArray([SPAN_ID, TRANSACTION_ID] as const);
+  const optionalFields = asMutableArray([
+    SPAN_ID,
+    TRANSACTION_ID,
+    LINKS_SPAN_ID,
+    LINKS_TRACE_ID,
+  ] as const);
 
   const response = await apmEventClient.search('fetch_linked_children_of_span', {
     apm: {
@@ -55,8 +63,28 @@ async function fetchLinkedChildrenOfSpan({
         bool: {
           filter: [
             ...rangeQuery(startWithBuffer, endWithBuffer),
-            { term: { [SPAN_LINKS_TRACE_ID]: traceId } },
-            ...(spanId ? [{ term: { [SPAN_LINKS_SPAN_ID]: spanId } }] : []),
+            {
+              bool: {
+                minimum_should_match: 1,
+                should: [
+                  ...termQuery(SPAN_LINKS_TRACE_ID, traceId),
+                  ...termQuery(LINKS_TRACE_ID, traceId),
+                ],
+              },
+            },
+            ...(spanId
+              ? [
+                  {
+                    bool: {
+                      minimum_should_match: 1,
+                      should: [
+                        ...termQuery(SPAN_LINKS_SPAN_ID, spanId),
+                        ...termQuery(LINKS_SPAN_ID, spanId),
+                      ],
+                    },
+                  },
+                ]
+              : []),
           ],
         },
       },
@@ -67,11 +95,21 @@ async function fetchLinkedChildrenOfSpan({
     const source = 'span' in hit._source ? hit._source : undefined;
     const event = unflattenKnownApmEventFields(hit.fields, requiredFields);
 
+    const spanLinks: SpanLink[] =
+      event.links?.span_id && event.links?.trace_id
+        ? [
+            {
+              span: { id: event.links.span_id as string },
+              trace: { id: event.links.trace_id as string },
+            },
+          ]
+        : [];
+
     return {
       ...event,
       span: {
         ...event.span,
-        links: source?.span?.links ?? [],
+        links: source?.span?.links ?? spanLinks ?? [],
       },
     };
   });

x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts

@@ -4,18 +4,22 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { rangeQuery } from '@kbn/observability-plugin/server';
+import { existsQuery, rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import {
   SPAN_ID,
   SPAN_LINKS,
   TRACE_ID,
   TRANSACTION_ID,
   PROCESSOR_EVENT,
+  LINKS_SPAN_ID,
+  LINKS_TRACE_ID,
 } from '../../../common/es_fields/apm';
-import { SpanRaw } from '../../../typings/es_schemas/raw/span_raw';
-import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
-import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
+import type { SpanRaw } from '../../../typings/es_schemas/raw/span_raw';
+import type { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
+import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 
 export async function getLinkedParentsOfSpan({
   apmEventClient,
@@ -32,12 +36,19 @@ export async function getLinkedParentsOfSpan({
   end: number;
   processorEvent: ProcessorEvent;
 }) {
+  const optionalFields = asMutableArray([LINKS_SPAN_ID, LINKS_TRACE_ID] as const);
+  const events =
+    processorEvent === ProcessorEvent.span
+      ? [ProcessorEvent.span]
+      : [processorEvent, ProcessorEvent.span];
+
   const response = await apmEventClient.search('get_linked_parents_of_span', {
     apm: {
-      events: [processorEvent],
+      events,
     },
     _source: [SPAN_LINKS],
     body: {
+      fields: [...optionalFields],
       track_total_hits: false,
       size: 1,
       query: {
@@ -45,7 +56,12 @@ export async function getLinkedParentsOfSpan({
           filter: [
             ...rangeQuery(start, end),
             { term: { [TRACE_ID]: traceId } },
-            { exists: { field: SPAN_LINKS } },
+            {
+              bool: {
+                should: [...existsQuery(SPAN_LINKS), ...existsQuery(LINKS_SPAN_ID)],
+                minimum_should_match: 1,
+              },
+            },
             { term: { [PROCESSOR_EVENT]: processorEvent } },
             ...(processorEvent === ProcessorEvent.transaction
               ? [{ term: { [TRANSACTION_ID]: spanId } }]
@@ -57,6 +73,16 @@ export async function getLinkedParentsOfSpan({
   });
 
   const source = response.hits.hits?.[0]?._source as Pick<TransactionRaw | SpanRaw, 'span'>;
+  const fields = unflattenKnownApmEventFields(response.hits.hits?.[0]?.fields);
+  const spanLinksFromFields =
+    fields?.links?.span_id && fields?.links?.trace_id
+      ? [
+          {
+            trace: { id: fields.links.trace_id as string },
+            span: { id: fields.links.span_id as string },
+          },
+        ]
+      : [];
 
-  return source?.span?.links || [];
+  return source?.span?.links ?? spanLinksFromFields;
 }

x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts

@@ -12,6 +12,7 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { last, omit } from 'lodash';
 import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import type { SpanLink } from '@kbn/apm-types';
 import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import { APMConfig } from '../..';
 import {
@@ -27,6 +28,8 @@ import {
   ERROR_LOG_MESSAGE,
   EVENT_OUTCOME,
   FAAS_COLDSTART,
+  LINKS_SPAN_ID,
+  LINKS_TRACE_ID,
   PARENT_ID,
   PROCESSOR_EVENT,
   SERVICE_ENVIRONMENT,
@@ -301,6 +304,8 @@ async function getTraceDocsPerPage({
     SPAN_COMPOSITE_SUM,
     SPAN_SYNC,
     CHILD_ID,
+    LINKS_SPAN_ID,
+    LINKS_TRACE_ID,
   ] as const);
 
   const body = {
@@ -347,6 +352,16 @@ async function getTraceDocsPerPage({
   return {
     hits: res.hits.hits.map((hit) => {
       const sort = hit.sort;
+      const fields = unflattenKnownApmEventFields(hit?.fields);
+      const spanLinksFromFields: SpanLink[] =
+        fields?.links?.span_id && fields?.links?.trace_id
+          ? [
+              {
+                trace: { id: fields.links.trace_id as string },
+                span: { id: fields.links.span_id as string },
+              },
+            ]
+          : [];
       const spanLinksSource = 'span' in hit._source ? hit._source.span?.links : undefined;
 
       if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) {
@@ -365,7 +380,7 @@ async function getTraceDocsPerPage({
             composite: spanEvent.span.composite
               ? (spanEvent.span.composite as Required<WaterfallSpan['span']>['composite'])
               : undefined,
-            links: spanLinksSource,
+            links: spanLinksSource ?? spanLinksFromFields,
           },
           ...(spanEvent.child ? { child: spanEvent.child as WaterfallSpan['child'] } : {}),
         };
@@ -384,7 +399,7 @@ async function getTraceDocsPerPage({
         },
         span: {
           ...txEvent.span,
-          links: spanLinksSource,
+          links: spanLinksSource ?? spanLinksFromFields,
         },
       };