[APM] Missing items in the trace waterfall shouldn't break it entirely (#210210)

closes https://github.com/elastic/kibana/issues/120464

When orphan items are found, I am re-parenting them to the root
transaction and adding an indication.

Test architecture:
APP_A -> APP_B -> APP_C

`APP_B` is not instrumented with Elastic APM, so it is not available in
the trace, thus APP_C has a parent which is not available in the current
trace. `APP_C` is reparented to point to `APP_A`.

Before:
<img width="1509" alt="Screenshot 2025-02-07 at 12 55 06"
src="https://github.com/user-attachments/assets/a973fa5d-1acf-4fff-b525-01957490494e"
/>


After [1]:
<img width="1499" alt="Screenshot 2025-02-07 at 12 03 34"
src="https://github.com/user-attachments/assets/8c49df04-de09-4d17-b6f8-f4b848e89f91"
/>

After [2]:
<img width="712" alt="Screenshot 2025-02-07 at 11 35 28"
src="https://github.com/user-attachments/assets/2b62a7cf-5979-4636-bc05-c25c96e9d71b"
/>

## How to test it:
- Run synthtrace `distributed_trace.ts` scenario.
- Find a trace.id
- Remove one of the elements from the trace.

x-pack/solutions/observability/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/orphan_item_tooltip_icon.tsx

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { EuiIcon, EuiToolTip } from '@elastic/eui';
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+
+interface Props {
+  docType: 'transaction' | 'span';
+}
+
+export function OrphanItemTooltipIcon({ docType }: Props) {
+  return (
+    <EuiToolTip
+      title={i18n.translate('xpack.apm.waterfallItem.euiToolTip.orphanLabel', {
+        defaultMessage: 'Orphan',
+      })}
+      content={i18n.translate('xpack.apm.waterfallItem.euiToolTip.orphanDescriotion', {
+        defaultMessage:
+          'This {type} was initially orphaned due to missing trace context but has been reparented to the root transaction to restore the execution flow',
+        values: { type: docType },
+      })}
+    >
+      <EuiIcon type="unlink" size="s" color="danger" />
+    </EuiToolTip>
+  );
+}

x-pack/solutions/observability/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_helpers/waterfall_helpers.test.ts

@@ -15,15 +15,17 @@ import type {
   IWaterfallSpanOrTransaction,
   IWaterfallNode,
   IWaterfallNodeFlatten,
+  IWaterfallSpan,
 } from './waterfall_helpers';
 import {
   getClockSkew,
   getOrderedWaterfallItems,
   getWaterfall,
-  getOrphanTraceItemsCount,
+  getOrphanItemsIds,
   buildTraceTree,
   convertTreeToList,
   updateTraceTreeNode,
+  reparentOrphanItems,
 } from './waterfall_helpers';
 import type { APMError } from '../../../../../../../../typings/es_schemas/ui/apm_error';
 import type {
@@ -716,45 +718,132 @@ describe('waterfall_helpers', () => {
     });
   });
 
-  describe('getOrphanTraceItemsCount', () => {
+  describe('getOrphanItemsIds', () => {
     const myTransactionItem = {
-      processor: { event: 'transaction' },
-      trace: { id: 'myTrace' },
-      transaction: {
-        id: 'myTransactionId1',
-      },
-    } as WaterfallTransaction;
+      doc: {
+        processor: { event: 'transaction' },
+        trace: { id: 'myTrace' },
+        transaction: {
+          id: 'myTransactionId1',
+        },
+      } as WaterfallTransaction,
+      docType: 'transaction',
+      id: 'myTransactionId1',
+    } as IWaterfallTransaction;
 
     it('should return missing items count: 0 if there are no orphan items', () => {
-      const traceItems: Array<WaterfallTransaction | WaterfallSpan> = [
+      const traceItems: IWaterfallSpanOrTransaction[] = [
         myTransactionItem,
         {
-          processor: { event: 'span' },
-          span: {
-            id: 'mySpanId',
-          },
-          parent: {
-            id: 'myTransactionId1',
-          },
-        } as WaterfallSpan,
+          doc: {
+            processor: { event: 'span' },
+            span: {
+              id: 'mySpanId',
+            },
+            parent: {
+              id: 'myTransactionId1',
+            },
+          } as WaterfallSpan,
+          docType: 'span',
+          id: 'mySpanId',
+          parentId: 'myTransactionId1',
+        } as IWaterfallSpan,
       ];
-      expect(getOrphanTraceItemsCount(traceItems)).toBe(0);
+      expect(getOrphanItemsIds(traceItems).length).toBe(0);
     });
 
     it('should return missing items count if there are orphan items', () => {
-      const traceItems: Array<WaterfallTransaction | WaterfallSpan> = [
+      const traceItems: IWaterfallSpanOrTransaction[] = [
         myTransactionItem,
         {
-          processor: { event: 'span' },
-          span: {
-            id: 'myOrphanSpanId',
-          },
-          parent: {
-            id: 'myNotExistingTransactionId1',
-          },
-        } as WaterfallSpan,
+          doc: {
+            processor: { event: 'span' },
+            span: {
+              id: 'myOrphanSpanId',
+            },
+            parent: {
+              id: 'myNotExistingTransactionId1',
+            },
+          } as WaterfallSpan,
+          docType: 'span',
+          id: 'myOrphanSpanId',
+          parentId: 'myNotExistingTransactionId1',
+        } as IWaterfallSpan,
       ];
-      expect(getOrphanTraceItemsCount(traceItems)).toBe(1);
+      expect(getOrphanItemsIds(traceItems).length).toBe(1);
+    });
+  });
+
+  describe('reparentOrphanItems', () => {
+    const myTransactionItem = {
+      doc: {
+        processor: { event: 'transaction' },
+        trace: { id: 'myTrace' },
+        transaction: {
+          id: 'myTransactionId1',
+        },
+      } as WaterfallTransaction,
+      docType: 'transaction',
+      id: 'myTransactionId1',
+    } as IWaterfallTransaction;
+
+    it('should not reparent since no orphan items exist', () => {
+      const traceItems: IWaterfallSpanOrTransaction[] = [
+        myTransactionItem,
+        {
+          doc: {
+            processor: { event: 'span' },
+            span: {
+              id: 'mySpanId',
+            },
+            parent: {
+              id: 'myTransactionId1',
+            },
+          } as WaterfallSpan,
+          docType: 'span',
+          id: 'mySpanId',
+          parentId: 'myTransactionId1',
+        } as IWaterfallSpan,
+      ];
+      expect(reparentOrphanItems([], traceItems, 'myTransactionId1')).toEqual(traceItems);
+    });
+
+    it('should reparent orphan items to root transaction', () => {
+      const traceItems: IWaterfallSpanOrTransaction[] = [
+        myTransactionItem,
+        {
+          doc: {
+            processor: { event: 'span' },
+            span: {
+              id: 'myOrphanSpanId',
+            },
+            parent: {
+              id: 'myNotExistingTransactionId1',
+            },
+          } as WaterfallSpan,
+          docType: 'span',
+          id: 'myOrphanSpanId',
+          parentId: 'myNotExistingTransactionId1',
+        } as IWaterfallSpan,
+      ];
+      expect(reparentOrphanItems(['myOrphanSpanId'], traceItems, 'myTransactionId1')).toEqual([
+        myTransactionItem,
+        {
+          doc: {
+            processor: { event: 'span' },
+            span: {
+              id: 'myOrphanSpanId',
+            },
+            parent: {
+              id: 'myNotExistingTransactionId1',
+            },
+          } as WaterfallSpan,
+          docType: 'span',
+          id: 'myOrphanSpanId',
+          parentId: 'myTransactionId1',
+          isOrphan: true,
+        } as IWaterfallSpan,
+      ]);
     });
   });
 

x-pack/solutions/observability/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_helpers/waterfall_helpers.ts

@@ -74,6 +74,7 @@ interface IWaterfallItemBase<TDocument, TDoctype> {
   duration: number;
   legendValues: Record<WaterfallLegendType, string>;
   spanLinksCount: SpanLinksCount;
+  isOrphan?: boolean;
 }
 
 export type IWaterfallError = Omit<
@@ -402,25 +403,27 @@ function getErrorCountByParentId(errorDocs: TraceAPIResponse['traceItems']['erro
   }, {});
 }
 
-export const getOrphanTraceItemsCount = (
-  traceDocs: Array<WaterfallTransaction | WaterfallSpan>
-) => {
-  const waterfallItemsIds = new Set(
-    traceDocs.map((doc) =>
-      doc.processor.event === 'span'
-        ? (doc?.span as WaterfallSpan['span']).id
-        : doc?.transaction?.id
-    )
-  );
+export function getOrphanItemsIds(waterfall: IWaterfallSpanOrTransaction[]) {
+  const waterfallItemsIds = new Set(waterfall.map((item) => item.id));
+  return waterfall
+    .filter((item) => item.parentId && !waterfallItemsIds.has(item.parentId))
+    .map((item) => item.id);
+}
 
-  let missingTraceItemsCounter = 0;
-  traceDocs.some((item) => {
-    if (item.parent?.id && !waterfallItemsIds.has(item.parent.id)) {
-      missingTraceItemsCounter++;
+export function reparentOrphanItems(
+  orphanItemsIds: string[],
+  waterfallItems: IWaterfallSpanOrTransaction[],
+  newParentId?: string
+) {
+  const orphanIdsMap = new Set(orphanItemsIds);
+  return waterfallItems.map((item) => {
+    if (orphanIdsMap.has(item.id)) {
+      item.parentId = newParentId;
+      item.isOrphan = true;
     }
+    return item;
   });
-  return missingTraceItemsCounter;
-};
+}
 
 export function getWaterfall(apiResponse: TraceAPIResponse): IWaterfall {
   const { traceItems, entryTransaction } = apiResponse;
@@ -447,13 +450,20 @@ export function getWaterfall(apiResponse: TraceAPIResponse): IWaterfall {
     traceItems.spanLinksCountById
   );
 
-  const childrenByParentId = getChildrenGroupedByParentId(reparentSpans(waterfallItems));
-
   const entryWaterfallTransaction = getEntryWaterfallTransaction(
     entryTransaction.transaction.id,
     waterfallItems
   );
 
+  const orphanItemsIds = getOrphanItemsIds(waterfallItems);
+  const childrenByParentId = getChildrenGroupedByParentId(
+    reparentOrphanItems(
+      orphanItemsIds,
+      reparentSpans(waterfallItems),
+      entryWaterfallTransaction?.id
+    )
+  );
+
   const items = getOrderedWaterfallItems(childrenByParentId, entryWaterfallTransaction);
   const errorItems = getWaterfallErrors(traceItems.errorDocs, items, entryWaterfallTransaction);
 
@@ -462,8 +472,6 @@ export function getWaterfall(apiResponse: TraceAPIResponse): IWaterfall {
   const duration = getWaterfallDuration(items);
   const legends = getLegends(items);
 
-  const orphanTraceItemsCount = getOrphanTraceItemsCount(traceItems.traceDocs);
-
   return {
     entryWaterfallTransaction,
     rootWaterfallTransaction,
@@ -478,7 +486,7 @@ export function getWaterfall(apiResponse: TraceAPIResponse): IWaterfall {
     totalErrorsCount: traceItems.errorDocs.length,
     traceDocsTotal: traceItems.traceDocsTotal,
     maxTraceItems: traceItems.maxTraceItems,
-    orphanTraceItemsCount,
+    orphanTraceItemsCount: orphanItemsIds.length,
   };
 }
 

x-pack/solutions/observability/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_item.tsx

@@ -22,6 +22,7 @@ import type { IWaterfallSpanOrTransaction } from './waterfall_helpers/waterfall_
 import { FailureBadge } from './failure_badge';
 import { useApmRouter } from '../../../../../../hooks/use_apm_router';
 import { useAnyOfApmParams } from '../../../../../../hooks/use_apm_params';
+import { OrphanItemTooltipIcon } from './orphan_item_tooltip_icon';
 
 type ItemType = 'transaction' | 'span' | 'error';
 
@@ -283,6 +284,7 @@ export function WaterfallItem({
         <SpanActionToolTip item={item}>
           <PrefixIcon item={item} />
         </SpanActionToolTip>
+        {item.isOrphan ? <OrphanItemTooltipIcon docType={item.docType} /> : null}
         <HttpStatusCode item={item} />
         <NameLabel item={item} />