[ObsUX][APM] Add tooltip for exit spans missing destination (#213714)

Closes https://github.com/elastic/kibana/issues/212638

### Summary

In order to help users, support agents and engineers to spot eventual
problems on the Service Map, we want to display a warning on the exit
spans that lack the span.destination.service.resource on the trace
waterfall.


![image](https://github.com/user-attachments/assets/eee3f962-a91c-49bb-9f06-c989ed8500c5)

x-pack/solutions/observability/plugins/apm/common/waterfall/typings.ts

@@ -62,6 +62,7 @@ export interface WaterfallSpan {
     sync?: boolean;
     duration: { us: number };
     links?: SpanLink[];
+    destination?: { service?: { resource?: string } };
   };
   transaction?: {
     id?: string;

x-pack/solutions/observability/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/span_missing_destination_tooltip.tsx

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { EuiIcon, EuiToolTip } from '@elastic/eui';
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+
+export function SpanMissingDestinationTooltip() {
+  return (
+    <EuiToolTip
+      title={i18n.translate('xpack.apm.waterfallItem.euiToolTip.spanMissingDestinationLabel', {
+        defaultMessage: 'Missing destination',
+      })}
+      content={i18n.translate(
+        'xpack.apm.waterfallItem.euiToolTip.spanMissingDestinationDescription',
+        {
+          defaultMessage:
+            'This exit span is missing the span.destination.service.resource field which might prevent linking it to downstream transactions on features that depend on this information. i.e.: Service Map. Make sure the instrumentation of this exit span follows OTel Semantic Conventions',
+        }
+      )}
+    >
+      <EuiIcon type="warning" size="s" color="danger" />
+    </EuiToolTip>
+  );
+}

x-pack/solutions/observability/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_helpers/waterfall_helpers.ts

@@ -9,6 +9,7 @@ import { euiPaletteColorBlind } from '@elastic/eui';
 import type { Dictionary } from 'lodash';
 import { first, flatten, groupBy, isEmpty, sortBy, uniq } from 'lodash';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { isOpenTelemetryAgentName } from '../../../../../../../../common/agent_name';
 import type { CriticalPathSegment } from '../../../../../../../../common/critical_path/types';
 import type { APIReturnType } from '../../../../../../../services/rest/create_call_apm_api';
 import type { Transaction } from '../../../../../../../../typings/es_schemas/ui/transaction';
@@ -75,6 +76,7 @@ interface IWaterfallItemBase<TDocument, TDoctype> {
   legendValues: Record<WaterfallLegendType, string>;
   spanLinksCount: SpanLinksCount;
   isOrphan?: boolean;
+  missingDestination?: boolean;
 }
 
 export type IWaterfallError = Omit<
@@ -548,6 +550,18 @@ function buildTree({
           hasInitializedChildren: false,
         };
 
+        // It is missing a destination when a child (currentNode) is a transaction
+        // and its parent (node) is a span without destination for Otel agents.
+
+        if (
+          currentNode.item.docType === 'transaction' &&
+          node.item.docType === 'span' &&
+          !node.item.doc.span?.destination?.service?.resource &&
+          isOpenTelemetryAgentName(node.item.doc.agent.name)
+        ) {
+          node.item.missingDestination = true;
+        }
+
         node.children.push(currentNode);
         queue.push(currentNode);
       });

x-pack/solutions/observability/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_item.tsx

@@ -23,6 +23,7 @@ import { FailureBadge } from './failure_badge';
 import { useApmRouter } from '../../../../../../hooks/use_apm_router';
 import { useAnyOfApmParams } from '../../../../../../hooks/use_apm_params';
 import { OrphanItemTooltipIcon } from './orphan_item_tooltip_icon';
+import { SpanMissingDestinationTooltip } from './span_missing_destination_tooltip';
 
 type ItemType = 'transaction' | 'span' | 'error';
 
@@ -285,6 +286,7 @@ export function WaterfallItem({
           <PrefixIcon item={item} />
         </SpanActionToolTip>
         {item.isOrphan ? <OrphanItemTooltipIcon docType={item.docType} /> : null}
+        {item.missingDestination ? <SpanMissingDestinationTooltip /> : null}
         <HttpStatusCode item={item} />
         <NameLabel item={item} />
 

x-pack/solutions/observability/plugins/apm/server/routes/traces/__snapshots__/queries.test.ts.snap

@@ -26,6 +26,7 @@ Object {
     "parent.id",
     "transaction.id",
     "span.id",
+    "span.destination.service.resource",
     "error.culprit",
     "error.log.message",
     "error.exception.message",

x-pack/solutions/observability/plugins/apm/server/routes/traces/get_trace_items.ts

@@ -38,6 +38,7 @@ import {
   SPAN_COMPOSITE_COMPRESSION_STRATEGY,
   SPAN_COMPOSITE_COUNT,
   SPAN_COMPOSITE_SUM,
+  SPAN_DESTINATION_SERVICE_RESOURCE,
   SPAN_DURATION,
   SPAN_ID,
   SPAN_LINKS,
@@ -105,6 +106,7 @@ export async function getTraceItems({
     PARENT_ID,
     TRANSACTION_ID,
     SPAN_ID,
+    SPAN_DESTINATION_SERVICE_RESOURCE,
     ERROR_CULPRIT,
     ERROR_LOG_MESSAGE,
     ERROR_EXC_MESSAGE,
@@ -301,6 +303,7 @@ async function getTraceDocsPerPage({
     SPAN_COMPOSITE_COMPRESSION_STRATEGY,
     SPAN_COMPOSITE_SUM,
     SPAN_SYNC,
+    SPAN_DESTINATION_SERVICE_RESOURCE,
     CHILD_ID,
     LINKS_SPAN_ID,
     LINKS_TRACE_ID,