[Profiling] Add support for inline frames (#158190)

This PR inserts inline frames into the StackFrames and StackTraces
generated from the StackTraceResponse by
`search_stacktraces.ts/searchStackTraces()`.

It includes a test for doing the inlining.

The inline frames are tagged as `Inline: true`, but I am currently not
sure if this information is passed to the client. In a follow-up PR I'd
like to mark or highlight inlined frames in the UI.

x-pack/plugins/profiling/common/__fixtures__/stacktraces.ts

@@ -143,6 +143,7 @@ const defaultStackFrame = {
   FunctionName: '',
   FunctionOffset: 0,
   LineNumber: 0,
+  Inline: false,
 };
 
 export const stackFrames = new Map([
@@ -153,9 +154,13 @@ export const stackFrames = new Map([
       FunctionName: 'java.lang.Runnable java.util.concurrent.ThreadPoolExecutor.getTask()',
       FunctionOffset: 26,
       LineNumber: 1061,
+      Inline: false,
     },
   ],
-  [frameID.B, { FileName: '', FunctionName: 'sock_sendmsg', FunctionOffset: 0, LineNumber: 0 }],
+  [
+    frameID.B,
+    { FileName: '', FunctionName: 'sock_sendmsg', FunctionOffset: 0, LineNumber: 0, Inline: false },
+  ],
   [frameID.C, defaultStackFrame],
   [frameID.D, defaultStackFrame],
   [frameID.E, defaultStackFrame],
@@ -165,7 +170,10 @@ export const stackFrames = new Map([
   [frameID.I, defaultStackFrame],
   [frameID.J, defaultStackFrame],
   [frameID.K, defaultStackFrame],
-  [frameID.L, { FileName: '', FunctionName: 'udp_sendmsg', FunctionOffset: 0, LineNumber: 0 }],
+  [
+    frameID.L,
+    { FileName: '', FunctionName: 'udp_sendmsg', FunctionOffset: 0, LineNumber: 0, Inline: false },
+  ],
   [frameID.M, defaultStackFrame],
   [frameID.N, defaultStackFrame],
   [frameID.O, defaultStackFrame],

x-pack/plugins/profiling/common/callee.ts

@@ -26,6 +26,7 @@ export interface CalleeTree {
 
   FileID: string[];
   FrameType: number[];
+  Inline: boolean[];
   ExeFilename: string[];
   AddressOrLine: number[];
   FunctionName: string[];
@@ -49,6 +50,7 @@ export function createCalleeTree(
     Edges: new Array(totalFrames),
     FileID: new Array(totalFrames),
     FrameType: new Array(totalFrames),
+    Inline: new Array(totalFrames),
     ExeFilename: new Array(totalFrames),
     AddressOrLine: new Array(totalFrames),
     FunctionName: new Array(totalFrames),
@@ -64,6 +66,7 @@ export function createCalleeTree(
 
   tree.FileID[0] = '';
   tree.FrameType[0] = 0;
+  tree.Inline[0] = false;
   tree.ExeFilename[0] = '';
   tree.AddressOrLine[0] = 0;
   tree.FunctionName[0] = '';
@@ -129,6 +132,7 @@ export function createCalleeTree(
         tree.FunctionOffset[node] = frame.FunctionOffset;
         tree.SourceLine[node] = frame.LineNumber;
         tree.SourceFilename[node] = frame.FileName;
+        tree.Inline[node] = frame.Inline;
         tree.CountInclusive[node] = samples;
         tree.CountExclusive[node] = 0;
 

x-pack/plugins/profiling/common/flamegraph.ts

@@ -26,6 +26,7 @@ export interface BaseFlameGraph {
 
   FileID: string[];
   FrameType: number[];
+  Inline: boolean[];
   ExeFilename: string[];
   AddressOrLine: number[];
   FunctionName: string[];
@@ -47,6 +48,7 @@ export function createBaseFlameGraph(tree: CalleeTree, totalSeconds: number): Ba
 
     FileID: tree.FileID.slice(0, tree.Size),
     FrameType: tree.FrameType.slice(0, tree.Size),
+    Inline: tree.Inline.slice(0, tree.Size),
     ExeFilename: tree.ExeFilename.slice(0, tree.Size),
     AddressOrLine: tree.AddressOrLine.slice(0, tree.Size),
     FunctionName: tree.FunctionName.slice(0, tree.Size),
@@ -88,6 +90,7 @@ export function createFlameGraph(base: BaseFlameGraph): ElasticFlameGraph {
 
     FileID: base.FileID,
     FrameType: base.FrameType,
+    Inline: base.Inline,
     ExeFilename: base.ExeFilename,
     AddressOrLine: base.AddressOrLine,
     FunctionName: base.FunctionName,
@@ -137,6 +140,7 @@ export function createFlameGraph(base: BaseFlameGraph): ElasticFlameGraph {
     const metadata = createStackFrameMetadata({
       FileID: graph.FileID[i],
       FrameType: graph.FrameType[i],
+      Inline: graph.Inline[i],
       ExeFileName: graph.ExeFilename[i],
       AddressOrLine: graph.AddressOrLine[i],
       FunctionName: graph.FunctionName[i],

x-pack/plugins/profiling/common/functions.ts

@@ -87,6 +87,7 @@ export function createTopNFunctions(
           FileID: fileID,
           AddressOrLine: addressOrLine,
           FrameType: stackTrace.Types[i],
+          Inline: frame.Inline,
           FunctionName: frame.FunctionName,
           FunctionOffset: frame.FunctionOffset,
           SourceLine: frame.LineNumber,

x-pack/plugins/profiling/common/profiling.ts

@@ -103,6 +103,7 @@ export interface StackFrame {
   FunctionName: string;
   FunctionOffset: number;
   LineNumber: number;
+  Inline: boolean;
 }
 
 export const emptyStackFrame: StackFrame = {
@@ -110,6 +111,7 @@ export const emptyStackFrame: StackFrame = {
   FunctionName: '',
   FunctionOffset: 0,
   LineNumber: 0,
+  Inline: false,
 };
 
 export interface Executable {
@@ -127,6 +129,8 @@ export interface StackFrameMetadata {
   FileID: FileID;
   // StackTrace.Type
   FrameType: FrameType;
+  // StackFrame.Inline
+  Inline: boolean;
 
   // StackTrace.AddressOrLine
   AddressOrLine: number;
@@ -165,6 +169,7 @@ export function createStackFrameMetadata(
   metadata.FrameID = options.FrameID ?? '';
   metadata.FileID = options.FileID ?? '';
   metadata.FrameType = options.FrameType ?? 0;
+  metadata.Inline = options.Inline ?? false;
   metadata.AddressOrLine = options.AddressOrLine ?? 0;
   metadata.FunctionName = options.FunctionName ?? '';
   metadata.FunctionOffset = options.FunctionOffset ?? 0;
@@ -307,6 +312,7 @@ export function groupStackFrameMetadataByStackTrace(
         FileID: fileID,
         AddressOrLine: addressOrLine,
         FrameType: trace.Types[i],
+        Inline: frame.Inline,
         FunctionName: frame.FunctionName,
         FunctionOffset: frame.FunctionOffset,
         SourceLine: frame.LineNumber,

x-pack/plugins/profiling/common/stack_traces.ts

@@ -11,7 +11,7 @@ interface ProfilingEvents {
   [key: string]: number;
 }
 
-interface ProfilingStackTrace {
+export interface ProfilingStackTrace {
   ['file_ids']: string[];
   ['frame_ids']: string[];
   ['address_or_lines']: number[];

x-pack/plugins/profiling/server/routes/search_stacktraces.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { decodeStackTraceResponse } from './search_stacktraces';
+import { decodeStackTraceResponse, makeFrameID } from './search_stacktraces';
 import { StackTraceResponse } from '../../common/stack_traces';
 
 describe('Stack trace response operations', () => {
@@ -47,10 +47,10 @@ describe('Stack trace response operations', () => {
       },
       stack_traces: {
         a: {
-          file_ids: ['abc'],
+          file_ids: ['abc', 'def'],
-          frame_ids: ['abc123'],
+          frame_ids: ['abc123', 'def456'],
-          address_or_lines: [123],
+          address_or_lines: [123, 456],
-          type_ids: [0],
+          type_ids: [0, 1],
         },
       },
       stack_frames: {
@@ -60,9 +60,16 @@ describe('Stack trace response operations', () => {
           function_offset: [0],
           line_number: [0],
         },
+        def: {
+          file_name: ['def.c'],
+          function_name: ['main', 'inlined'],
+          function_offset: [1, 2],
+          line_number: [3, 4],
+        },
       },
       executables: {
         abc: 'pthread.c',
+        def: 'def.c',
       },
       total_frames: 1,
     };
@@ -73,10 +80,10 @@ describe('Stack trace response operations', () => {
         [
           'a',
           {
-            FileIDs: ['abc'],
+            FileIDs: ['abc', 'def', 'def'],
-            FrameIDs: ['abc123'],
+            FrameIDs: ['abc123', makeFrameID('def456', 0), makeFrameID('def456', 1)],
-            AddressOrLines: [123],
+            AddressOrLines: [123, 456, 456],
-            Types: [0],
+            Types: [0, 1, 1],
           },
         ],
       ]),
@@ -90,18 +97,39 @@ describe('Stack trace response operations', () => {
             LineNumber: 0,
           },
         ],
+        [
+          makeFrameID('def456', 0),
+          {
+            FileName: 'def.c',
+            FunctionName: 'main',
+            FunctionOffset: 1,
+            LineNumber: 3,
+          },
+        ],
+        [
+          makeFrameID('def456', 1),
+          {
+            FileName: 'def.c',
+            FunctionName: 'inlined',
+            FunctionOffset: 2,
+            LineNumber: 4,
+          },
+        ],
+      ]),
+      executables: new Map([
+        ['abc', { FileName: 'pthread.c' }],
+        ['def', { FileName: 'def.c' }],
       ]),
-      executables: new Map([['abc', { FileName: 'pthread.c' }]]),
       totalFrames: 1,
     };
 
     const decoded = decodeStackTraceResponse(original);
 
     expect(decoded.executables.size).toEqual(expected.executables.size);
-    expect(decoded.executables.size).toEqual(1);
+    expect(decoded.executables.size).toEqual(2);
 
     expect(decoded.stackFrames.size).toEqual(expected.stackFrames.size);
-    expect(decoded.stackFrames.size).toEqual(1);
+    expect(decoded.stackFrames.size).toEqual(3);
 
     expect(decoded.stackTraces.size).toEqual(expected.stackTraces.size);
     expect(decoded.stackTraces.size).toEqual(1);

x-pack/plugins/profiling/server/routes/search_stacktraces.ts

@@ -13,40 +13,81 @@ import {
   StackTrace,
   StackTraceID,
 } from '../../common/profiling';
-import { StackTraceResponse } from '../../common/stack_traces';
+import { StackTraceResponse, ProfilingStackTrace } from '../../common/stack_traces';
 import { ProfilingESClient } from '../utils/create_profiling_es_client';
 import { ProjectTimeQuery } from './query';
 
+export const makeFrameID = (frameID: string, n: number): string => {
+  return n === 0 ? frameID : frameID + ';' + n.toString();
+};
+
+// createInlineTrace builds a new StackTrace with inline frames.
+const createInlineTrace = (
+  trace: ProfilingStackTrace,
+  frames: Map<StackFrameID, StackFrame>
+): StackTrace => {
+  // The arrays need to be extended with the inline frame information.
+  const frameIDs: string[] = [];
+  const fileIDs: string[] = [];
+  const addressOrLines: number[] = [];
+  const typeIDs: number[] = [];
+
+  for (let i = 0; i < trace.frame_ids.length; i++) {
+    const frameID = trace.frame_ids[i];
+    frameIDs.push(frameID);
+    fileIDs.push(trace.file_ids[i]);
+    addressOrLines.push(trace.address_or_lines[i]);
+    typeIDs.push(trace.type_ids[i]);
+
+    for (let j = 1; ; j++) {
+      const inlineID = makeFrameID(frameID, j);
+      const frame = frames.get(inlineID);
+      if (!frame) {
+        break;
+      }
+      frameIDs.push(inlineID);
+      fileIDs.push(trace.file_ids[i]);
+      addressOrLines.push(trace.address_or_lines[i]);
+      typeIDs.push(trace.type_ids[i]);
+    }
+  }
+
+  return {
+    FrameIDs: frameIDs,
+    FileIDs: fileIDs,
+    AddressOrLines: addressOrLines,
+    Types: typeIDs,
+  } as StackTrace;
+};
+
 export function decodeStackTraceResponse(response: StackTraceResponse) {
   const stackTraceEvents: Map<StackTraceID, number> = new Map();
   for (const [key, value] of Object.entries(response.stack_trace_events ?? {})) {
     stackTraceEvents.set(key, value);
   }
 
-  const stackTraces: Map<StackTraceID, StackTrace> = new Map();
-  for (const [key, value] of Object.entries(response.stack_traces ?? {})) {
-    stackTraces.set(key, {
-      FrameIDs: value.frame_ids,
-      FileIDs: value.file_ids,
-      AddressOrLines: value.address_or_lines,
-      Types: value.type_ids,
-    } as StackTrace);
-  }
-
   const stackFrames: Map<StackFrameID, StackFrame> = new Map();
-  for (const [key, value] of Object.entries(response.stack_frames ?? {})) {
+  for (const [frameID, frame] of Object.entries(response.stack_frames ?? {})) {
     // Each field in a stackframe is represented by an array. This is
     // necessary to support inline frames.
     //
-    // We only take the first available inline stackframe until the UI
+    // We store the inlined frames with a modified (and unique) ID.
-    // can support all of them.
+    // We can do so since we don't display the frame IDs.
-    stackFrames.set(key, {
+    for (let i = 0; i < frame.function_name.length; i++) {
-      FileName: value.file_name[0],
+      stackFrames.set(makeFrameID(frameID, i), {
-      FunctionName: value.function_name[0],
+        FileName: frame.file_name[i],
-      FunctionOffset: value.function_offset[0],
+        FunctionName: frame.function_name[i],
-      LineNumber: value.line_number[0],
+        FunctionOffset: frame.function_offset[i],
+        LineNumber: frame.line_number[i],
+        Inline: i > 0,
       } as StackFrame);
     }
+  }
+
+  const stackTraces: Map<StackTraceID, StackTrace> = new Map();
+  for (const [traceID, trace] of Object.entries(response.stack_traces ?? {})) {
+    stackTraces.set(traceID, createInlineTrace(trace, stackFrames));
+  }
 
   const executables: Map<FileID, Executable> = new Map();
   for (const [key, value] of Object.entries(response.executables ?? {})) {