[Profiling] Ignore error frames (#176537)

This change allows the Universal Profiling agent to send error frames,
which will give us more accurate values for CO2 emission and $ costs.

The reason is that unwinding errors resulting in 0-length stacktraces
happen quite often. These are not sent to the backend currently, so the
related CPU activity doesn't go into the calculations. This can make up
showing 10% less CPU / CO2 / costs in the UI.
Adding artificial error frames in case of unwinding errors guarantees
that stacktraces always have a length of > 0.

Once we settled on how error frames can be displayed in a user-friendly
way, this code can be removed.

---------

Co-authored-by: Joel Höner <joel@elastic.co>
Co-authored-by: Caue Marcondes <caue.marcondes@elastic.co>

docs/management/advanced-options.asciidoc

@@ -433,6 +433,9 @@ Set the default environment for the APM app. When left empty, data from all envi
 [[observability-apm-enable-profiling]]`observability:apmEnableProfilingIntegration`::
 Enable the Universal Profiling integration in APM.
 
+[[observability-profiling-show-error-frames]]`observability:profilingShowErrorFrames`::
+Show error frames in the Universal Profiling views to indicate stack unwinding failures.
+
 [[observability-apm-enable-table-search-bar]]`observability:apmEnableTableSearchBar`::
 beta:[] Enables faster searching in APM tables by adding a handy search bar with live filtering. Available for the following tables: Services, Transactions, and Errors.
 

packages/kbn-profiling-utils/common/flamegraph.test.ts

@@ -10,7 +10,7 @@ import { createFlameGraph } from './flamegraph';
 import { baseFlamegraph } from './__fixtures__/base_flamegraph';
 
 describe('Flamegraph', () => {
-  const flamegraph = createFlameGraph(baseFlamegraph);
+  const flamegraph = createFlameGraph(baseFlamegraph, false);
 
   it('base flamegraph has non-zero total seconds', () => {
     expect(baseFlamegraph.TotalSeconds).toEqual(4.980000019073486);

packages/kbn-profiling-utils/common/flamegraph.ts

@@ -8,7 +8,7 @@
 
 import { createFrameGroupID } from './frame_group';
 import { fnv1a64 } from './hash';
-import { createStackFrameMetadata, getCalleeLabel } from './profiling';
+import { createStackFrameMetadata, getCalleeLabel, isErrorFrame } from './profiling';
 import { convertTonsToKgs } from './utils';
 
 /**
@@ -81,9 +81,25 @@ export interface ElasticFlameGraph
  * This allows us to create a flamegraph in two steps (e.g. first on the server
  * and finally in the browser).
  * @param base BaseFlameGraph
+ * @param showErrorFrames
  * @returns ElasticFlameGraph
  */
-export function createFlameGraph(base: BaseFlameGraph): ElasticFlameGraph {
+export function createFlameGraph(
+  base: BaseFlameGraph,
+  showErrorFrames: boolean
+): ElasticFlameGraph {
+  if (!showErrorFrames) {
+    // This loop jumps over the error frames in the graph.
+    // Error frames only appear as child nodes of the root frame.
+    // Error frames only have a single child node.
+    for (let i = 0; i < base.Edges[0].length; i++) {
+      const childNodeID = base.Edges[0][i];
+      if (isErrorFrame(base.FrameType[childNodeID])) {
+        base.Edges[0][i] = base.Edges[childNodeID][0];
+      }
+    }
+  }
+
   const graph: ElasticFlameGraph = {
     Size: base.Size,
     SamplingRate: base.SamplingRate,

packages/kbn-profiling-utils/common/functions.test.ts

@@ -12,8 +12,10 @@ import { decodeStackTraceResponse } from '..';
 import { stacktraces } from './__fixtures__/stacktraces';
 
 describe('TopN function operations', () => {
-  const { events, stackTraces, stackFrames, executables, samplingRate } =
-    decodeStackTraceResponse(stacktraces);
+  const { events, stackTraces, stackFrames, executables, samplingRate } = decodeStackTraceResponse(
+    stacktraces,
+    false
+  );
   const maxTopN = 5;
   const topNFunctions = createTopNFunctions({
     events,
@@ -23,6 +25,7 @@ describe('TopN function operations', () => {
     startIndex: 0,
     endIndex: maxTopN,
     samplingRate,
+    showErrorFrames: false,
   });
   const exclusiveCounts = topNFunctions.TopN.map((value) => value.CountExclusive);
 

packages/kbn-profiling-utils/common/functions.ts

@@ -24,6 +24,7 @@ import {
   emptyStackFrame,
   emptyStackTrace,
 } from '..';
+import { isErrorFrame } from './profiling';
 
 interface TopNFunctionAndFrameGroup {
   Frame: StackFrameMetadata;
@@ -68,6 +69,7 @@ export function createTopNFunctions({
   stackFrames,
   stackTraces,
   startIndex,
+  showErrorFrames,
 }: {
   endIndex: number;
   events: Map<StackTraceID, number>;
@@ -76,6 +78,7 @@ export function createTopNFunctions({
   stackFrames: Map<StackFrameID, StackFrame>;
   stackTraces: Map<StackTraceID, StackTrace>;
   startIndex: number;
+  showErrorFrames: boolean;
 }): TopNFunctions {
   // The `count` associated with a frame provides the total number of
   // traces in which that node has appeared at least once. However, a
@@ -101,7 +104,11 @@ export function createTopNFunctions({
 
     const lenStackTrace = stackTrace.FrameIDs.length;
 
-    for (let i = 0; i < lenStackTrace; i++) {
+    // Error frames only appear as first frame in a stacktrace.
+    const start =
+      !showErrorFrames && lenStackTrace > 0 && isErrorFrame(stackTrace.Types[0]) ? 1 : 0;
+
+    for (let i = start; i < lenStackTrace; i++) {
       const frameID = stackTrace.FrameIDs[i];
       const fileID = stackTrace.FileIDs[i];
       const addressOrLine = stackTrace.AddressOrLines[i];

packages/kbn-profiling-utils/common/profiling.test.ts

@@ -14,6 +14,7 @@ import {
   getCalleeSource,
   getFrameSymbolStatus,
   getLanguageType,
+  normalizeFrameType,
 } from './profiling';
 
 describe('Stack frame metadata operations', () => {
@@ -101,6 +102,20 @@ describe('getFrameSymbolStatus', () => {
   });
 });
 
+describe('normalizeFrameType', () => {
+  it('rewrites any frame with error bit to the generic error variant', () => {
+    expect(normalizeFrameType(0x83 as FrameType)).toEqual(FrameType.Error);
+  });
+  it('rewrites unknown frame types to "unsymbolized" variant', () => {
+    expect(normalizeFrameType(0x123 as FrameType)).toEqual(FrameType.Unsymbolized);
+  });
+  it('passes regular known frame types through untouched', () => {
+    expect(normalizeFrameType(FrameType.JVM)).toEqual(FrameType.JVM);
+    expect(normalizeFrameType(FrameType.Native)).toEqual(FrameType.Native);
+    expect(normalizeFrameType(FrameType.JavaScript)).toEqual(FrameType.JavaScript);
+  });
+});
+
 describe('getLanguageType', () => {
   [FrameType.Native, FrameType.Kernel].map((type) =>
     it(`returns native for ${type}`, () => {

packages/kbn-profiling-utils/common/profiling.ts

@@ -33,6 +33,8 @@ export enum FrameType {
   Perl,
   JavaScript,
   PHPJIT,
+  ErrorFlag = 0x80,
+  Error = 0xff,
 }
 
 const frameTypeDescriptions = {
@@ -46,15 +48,41 @@ const frameTypeDescriptions = {
   [FrameType.Perl]: 'Perl',
   [FrameType.JavaScript]: 'JavaScript',
   [FrameType.PHPJIT]: 'PHP JIT',
+  [FrameType.ErrorFlag]: 'ErrorFlag',
+  [FrameType.Error]: 'Error',
 };
 
+export function isErrorFrame(ft: FrameType): boolean {
+  // eslint-disable-next-line no-bitwise
+  return (ft & FrameType.ErrorFlag) !== 0;
+}
+
+/**
+ * normalize the given frame type
+ * @param ft FrameType
+ * @returns FrameType
+ */
+export function normalizeFrameType(ft: FrameType): FrameType {
+  // Normalize any frame type with error bit into our uniform error variant.
+  if (isErrorFrame(ft)) {
+    return FrameType.Error;
+  }
+
+  // Guard against new / unknown frame types, rewriting them to "unsymbolized".
+  if (!(ft in frameTypeDescriptions)) {
+    return FrameType.Unsymbolized;
+  }
+
+  return ft;
+}
+
 /**
  * get frame type name
  * @param ft FrameType
  * @returns string
  */
 export function describeFrameType(ft: FrameType): string {
-  return frameTypeDescriptions[ft];
+  return frameTypeDescriptions[normalizeFrameType(ft)];
 }
 
 export interface StackTraceEvent {
@@ -214,7 +242,9 @@ function getExeFileName(metadata: StackFrameMetadata) {
  */
 export function getCalleeLabel(metadata: StackFrameMetadata) {
   const inlineLabel = metadata.Inline ? '-> ' : '';
-  if (metadata.FunctionName !== '') {
+  if (metadata.FrameType === FrameType.Error) {
+    return `Error: unwinding error code #${metadata.AddressOrLine.toString()}`;
+  } else if (metadata.FunctionName !== '') {
     const sourceFilename = metadata.SourceFilename;
     const sourceURL = sourceFilename ? sourceFilename.split('/').pop() : '';
     return `${inlineLabel}${getExeFileName(metadata)}: ${getFunctionName(
@@ -298,6 +328,10 @@ export function getLanguageType(param: LanguageTypeParams) {
  * @returns string
  */
 export function getCalleeSource(frame: StackFrameMetadata): string {
+  if (frame.FrameType === FrameType.Error) {
+    return `unwinding error code #${frame.AddressOrLine.toString()}`;
+  }
+
   const frameSymbolStatus = getFrameSymbolStatus({
     sourceFilename: frame.SourceFilename,
     sourceLine: frame.SourceLine,

packages/kbn-profiling-utils/common/stack_traces.test.ts

@@ -29,7 +29,7 @@ describe('Stack trace response operations', () => {
       samplingRate: 1.0,
     };
 
-    const decoded = decodeStackTraceResponse(original);
+    const decoded = decodeStackTraceResponse(original, false);
 
     expect(decoded.executables.size).toEqual(expected.executables.size);
     expect(decoded.executables.size).toEqual(0);
@@ -141,7 +141,7 @@ describe('Stack trace response operations', () => {
       samplingRate: 1.0,
     };
 
-    const decoded = decodeStackTraceResponse(original);
+    const decoded = decodeStackTraceResponse(original, false);
 
     expect(decoded.executables.size).toEqual(expected.executables.size);
     expect(decoded.executables.size).toEqual(2);
@@ -223,7 +223,7 @@ describe('Stack trace response operations', () => {
       samplingRate: 1.0,
     };
 
-    const decoded = decodeStackTraceResponse(original);
+    const decoded = decodeStackTraceResponse(original, false);
 
     expect(decoded.executables.size).toEqual(expected.executables.size);
     expect(decoded.executables.size).toEqual(1);

packages/kbn-profiling-utils/common/stack_traces.ts

@@ -10,6 +10,7 @@ import { ProfilingESField } from './elasticsearch';
 import {
   Executable,
   FileID,
+  isErrorFrame,
   StackFrame,
   StackFrameID,
   StackTrace,
@@ -111,7 +112,8 @@ export const makeFrameID = (frameID: string, n: number): string => {
 // createInlineTrace builds a new StackTrace with inline frames.
 const createInlineTrace = (
   trace: ProfilingStackTrace,
-  frames: Map<StackFrameID, StackFrame>
+  frames: Map<StackFrameID, StackFrame>,
+  showErrorFrames: boolean
 ): StackTrace => {
   // The arrays need to be extended with the inline frame information.
   const frameIDs: string[] = [];
@@ -119,7 +121,11 @@ const createInlineTrace = (
   const addressOrLines: number[] = [];
   const typeIDs: number[] = [];
 
-  for (let i = 0; i < trace.frame_ids.length; i++) {
+  // Error frames only appear as first frame in a stacktrace.
+  const start =
+    !showErrorFrames && trace.frame_ids.length > 0 && isErrorFrame(trace.type_ids[0]) ? 1 : 0;
+
+  for (let i = start; i < trace.frame_ids.length; i++) {
     const frameID = trace.frame_ids[i];
     frameIDs.push(frameID);
     fileIDs.push(trace.file_ids[i]);
@@ -153,9 +159,13 @@ const createInlineTrace = (
 /**
  * Decodes stack trace response
  * @param response StackTraceResponse
+ * @param showErrorFrames
  * @returns DecodedStackTraceResponse
  */
-export function decodeStackTraceResponse(response: StackTraceResponse): DecodedStackTraceResponse {
+export function decodeStackTraceResponse(
+  response: StackTraceResponse,
+  showErrorFrames: boolean
+): DecodedStackTraceResponse {
   const stackTraceEvents: Map<StackTraceID, number> = new Map();
   for (const [key, value] of Object.entries(response.stack_trace_events ?? {})) {
     stackTraceEvents.set(key, value);
@@ -181,7 +191,7 @@ export function decodeStackTraceResponse(response: StackTraceResponse): DecodedS
 
   const stackTraces: Map<StackTraceID, StackTrace> = new Map();
   for (const [traceID, trace] of Object.entries(response.stack_traces ?? {})) {
-    stackTraces.set(traceID, createInlineTrace(trace, stackFrames));
+    stackTraces.set(traceID, createInlineTrace(trace, stackFrames, showErrorFrames));
   }
 
   const executables: Map<FileID, Executable> = new Map();

packages/kbn-profiling-utils/index.ts

@@ -12,6 +12,7 @@ export { ProfilingESField } from './common/elasticsearch';
 export {
   groupStackFrameMetadataByStackTrace,
   describeFrameType,
+  normalizeFrameType,
   FrameType,
   getCalleeFunction,
   getCalleeSource,

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -568,6 +568,10 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'boolean',
     _meta: { description: 'Non-default value of setting.' },
   },
+  'observability:profilingShowErrorFrames': {
+    type: 'boolean',
+    _meta: { description: 'Non-default value of setting.' },
+  },
   'observability:profilingPerVCPUWattX86': {
     type: 'integer',
     _meta: { description: 'Non-default value of setting.' },

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -156,6 +156,7 @@ export interface UsageStats {
   'observability:apmTraceExplorerTab': boolean;
   'observability:apmEnableCriticalPath': boolean;
   'observability:apmEnableProfilingIntegration': boolean;
+  'observability:profilingShowErrorFrames': boolean;
   'securitySolution:enableGroupedNav': boolean;
   'securitySolution:showRelatedIntegrations': boolean;
   'visualization:visualize:legacyGaugeChartsLibrary': boolean;

src/plugins/telemetry/schema/oss_plugins.json

@@ -10168,6 +10168,12 @@
             "description": "Non-default value of setting."
           }
         },
+        "observability:profilingShowErrorFrames": {
+          "type": "boolean",
+          "_meta": {
+            "description": "Non-default value of setting."
+          }
+        },
         "observability:profilingPerVCPUWattX86": {
           "type": "integer",
           "_meta": {

x-pack/plugins/observability/common/index.ts

@@ -44,6 +44,7 @@ export {
   enableCriticalPath,
   syntheticsThrottlingEnabled,
   apmEnableProfilingIntegration,
+  profilingShowErrorFrames,
   profilingCo2PerKWH,
   profilingDatacenterPUE,
   profilingPervCPUWattX86,

x-pack/plugins/observability/common/ui_settings_keys.ts

@@ -32,6 +32,7 @@ export const apmEnableContinuousRollups = 'observability:apmEnableContinuousRoll
 export const syntheticsThrottlingEnabled = 'observability:syntheticsThrottlingEnabled';
 export const enableLegacyUptimeApp = 'observability:enableLegacyUptimeApp';
 export const apmEnableProfilingIntegration = 'observability:apmEnableProfilingIntegration';
+export const profilingShowErrorFrames = 'observability:profilingShowErrorFrames';
 export const profilingPervCPUWattX86 = 'observability:profilingPerVCPUWattX86';
 export const profilingPervCPUWattArm64 = 'observability:profilingPervCPUWattArm64';
 export const profilingCo2PerKWH = 'observability:profilingCo2PerKWH';

x-pack/plugins/observability/server/ui_settings.ts

@@ -31,6 +31,7 @@ import {
   syntheticsThrottlingEnabled,
   enableLegacyUptimeApp,
   apmEnableProfilingIntegration,
+  profilingShowErrorFrames,
   profilingCo2PerKWH,
   profilingDatacenterPUE,
   profilingPervCPUWattX86,
@@ -432,6 +433,15 @@ export const uiSettings: Record<string, UiSettings> = {
     schema: schema.boolean(),
     requiresPageReload: false,
   },
+  [profilingShowErrorFrames]: {
+    category: [observabilityFeatureId],
+    name: i18n.translate('xpack.observability.profilingShowErrorFramesSettingName', {
+      defaultMessage: 'Show error frames in the Universal Profiling views',
+    }),
+    value: false,
+    schema: schema.boolean(),
+    requiresPageReload: true,
+  },
   [profilingPervCPUWattX86]: {
     category: [observabilityFeatureId],
     name: i18n.translate('xpack.observability.profilingPervCPUWattX86UiSettingName', {

x-pack/plugins/profiling/common/columnar_view_model.test.ts

@@ -11,7 +11,7 @@ import { createColumnarViewModel } from './columnar_view_model';
 import { baseFlamegraph } from './__fixtures__/base_flamegraph';
 
 describe('Columnar view model operations', () => {
-  const graph = createFlameGraph(baseFlamegraph);
+  const graph = createFlameGraph(baseFlamegraph, false);
 
   describe('color values are generated by default', () => {
     const viewModel = createColumnarViewModel(graph);

x-pack/plugins/profiling/common/frame_type_colors.ts

@@ -5,22 +5,23 @@
  * 2.0.
  */
 
-import { FrameType } from '@kbn/profiling-utils';
+import { FrameType, normalizeFrameType } from '@kbn/profiling-utils';
 
 /*
  * Helper to calculate the color of a given block to be drawn. The desirable outcomes of this are:
  * Each of the following frame types should get a different set of color hues:
  *
- *   0 = Unsymbolized frame
- *   1 = Python
- *   2 = PHP
- *   3 = Native
- *   4 = Kernel
- *   5 = JVM/Hotspot
- *   6 = Ruby
- *   7 = Perl
- *   8 = JavaScript
- *   9 = PHP JIT
+ *   0x00 = Unsymbolized frame
+ *   0x01 = Python
+ *   0x02 = PHP
+ *   0x03 = Native
+ *   0x04 = Kernel
+ *   0x05 = JVM/Hotspot
+ *   0x06 = Ruby
+ *   0x07 = Perl
+ *   0x08 = JavaScript
+ *   0x09 = PHP JIT
+ *   0xFF = Error frame
  *
  * This is most easily achieved by mapping frame types to different color variations, using
  * the x-position we can use different colors for adjacent blocks while keeping a similar hue
@@ -38,10 +39,12 @@ export const FRAME_TYPE_COLOR_MAP = {
   [FrameType.Perl]: [0xf98bb9, 0xfaa2c7, 0xfbb9d5, 0xfdd1e3],
   [FrameType.JavaScript]: [0xcbc3e3, 0xd5cfe8, 0xdfdbee, 0xeae7f3],
   [FrameType.PHPJIT]: [0xccfc82, 0xd1fc8e, 0xd6fc9b, 0xdbfca7],
+  [FrameType.ErrorFlag]: [0x0, 0x0, 0x0, 0x0], // This is a special case, it's not a real frame type
+  [FrameType.Error]: [0xfd8484, 0xfd9d9d, 0xfeb5b5, 0xfecece],
 };
 
 export function frameTypeToRGB(frameType: FrameType, x: number): number {
-  return FRAME_TYPE_COLOR_MAP[frameType][x % 4];
+  return FRAME_TYPE_COLOR_MAP[normalizeFrameType(frameType)][x % 4];
 }
 
 export function rgbToRGBA(rgb: number): number[] {

x-pack/plugins/profiling/e2e/cypress/e2e/profiling_views/settings.cy.ts

@@ -39,6 +39,7 @@ describe('Settings page', () => {
     cy.contains('Per vCPU Watts - arm64');
     cy.contains('AWS EDP discount rate (%)');
     cy.contains('Cost per vCPU per hour ($)');
+    cy.contains('Show error frames in the Universal Profiling views');
   });
 
   it('updates values', () => {

x-pack/plugins/profiling/public/embeddables/flamegraph/embeddable_flamegraph.tsx

@@ -9,6 +9,7 @@ import { EMBEDDABLE_FLAMEGRAPH } from '@kbn/observability-shared-plugin/public';
 import { createFlameGraph } from '@kbn/profiling-utils';
 import React from 'react';
 import { render, unmountComponentAtNode } from 'react-dom';
+import { profilingShowErrorFrames } from '@kbn/observability-plugin/common';
 import { FlameGraph } from '../../components/flamegraph';
 import { AsyncEmbeddableComponent } from '../async_embeddable_component';
 import {
@@ -16,6 +17,7 @@ import {
   ProfilingEmbeddablesDependencies,
 } from '../profiling_embeddable_provider';
 import { EmbeddableFlamegraphEmbeddableInput } from './embeddable_flamegraph_factory';
+import { useProfilingDependencies } from '../../components/contexts/profiling_dependencies/use_profiling_dependencies';
 
 export class EmbeddableFlamegraph extends Embeddable<
   EmbeddableFlamegraphEmbeddableInput,
@@ -34,18 +36,9 @@ export class EmbeddableFlamegraph extends Embeddable<
 
   render(domNode: HTMLElement) {
     this._domNode = domNode;
-    const { data, isLoading } = this.input;
-    const flamegraph = !isLoading && data ? createFlameGraph(data) : undefined;
-
     render(
       <ProfilingEmbeddableProvider deps={this.deps}>
-        <AsyncEmbeddableComponent isLoading={isLoading}>
-          <>
-            {flamegraph && (
-              <FlameGraph primaryFlamegraph={flamegraph} id="embddable_profiling" isEmbedded />
-            )}
-          </>
-        </AsyncEmbeddableComponent>
+        <Flamegraph {...this.input} />
       </ProfilingEmbeddableProvider>,
       domNode
     );
@@ -63,3 +56,18 @@ export class EmbeddableFlamegraph extends Embeddable<
     }
   }
 }
+
+function Flamegraph({ isLoading, data }: EmbeddableFlamegraphEmbeddableInput) {
+  const { core } = useProfilingDependencies().start;
+  const showErrorFrames = core.uiSettings.get<boolean>(profilingShowErrorFrames);
+  const flamegraph = !isLoading && data ? createFlameGraph(data, showErrorFrames) : undefined;
+  return (
+    <AsyncEmbeddableComponent isLoading={isLoading}>
+      <>
+        {flamegraph && (
+          <FlameGraph primaryFlamegraph={flamegraph} id="embddable_profiling" isEmbedded />
+        )}
+      </>
+    </AsyncEmbeddableComponent>
+  );
+}

x-pack/plugins/profiling/public/services.ts

@@ -4,6 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 import { HttpFetchQuery } from '@kbn/core/public';
 import {
   createFlameGraph,
@@ -51,6 +52,7 @@ export interface Services {
     timeFrom: number;
     timeTo: number;
     kuery: string;
+    showErrorFrames: boolean;
   }) => Promise<ElasticFlameGraph>;
   fetchHasSetup: (params: { http: AutoAbortedHttpService }) => Promise<ProfilingSetupStatus>;
   postSetupResources: (params: { http: AutoAbortedHttpService }) => Promise<void>;
@@ -101,7 +103,7 @@ export function getServices(): Services {
       return (await http.get(paths.TopNFunctions, { query })) as Promise<TopNFunctions>;
     },
 
-    fetchElasticFlamechart: async ({ http, timeFrom, timeTo, kuery }) => {
+    fetchElasticFlamechart: async ({ http, timeFrom, timeTo, kuery, showErrorFrames }) => {
       const query: HttpFetchQuery = {
         timeFrom,
         timeTo,
@@ -109,7 +111,7 @@ export function getServices(): Services {
       };
 
       const baseFlamegraph = (await http.get(paths.Flamechart, { query })) as BaseFlameGraph;
-      return createFlameGraph(baseFlamegraph);
+      return createFlameGraph(baseFlamegraph, showErrorFrames);
     },
     fetchHasSetup: async ({ http }) => {
       const hasSetup = (await http.get(paths.HasSetupESResources, {})) as ProfilingSetupStatus;

x-pack/plugins/profiling/public/views/flamegraphs/differential_flamegraphs/index.tsx

@@ -6,6 +6,7 @@
  */
 import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSpacer } from '@elastic/eui';
 import React from 'react';
+import { profilingShowErrorFrames } from '@kbn/observability-plugin/common';
 import { AsyncComponent } from '../../../components/async_component';
 import { useProfilingDependencies } from '../../../components/contexts/profiling_dependencies/use_profiling_dependencies';
 import { FlameGraph } from '../../../components/flamegraph';
@@ -49,8 +50,11 @@ export function DifferentialFlameGraphsView() {
 
   const {
     services: { fetchElasticFlamechart },
+    start: { core },
   } = useProfilingDependencies();
 
+  const showErrorFrames = core.uiSettings.get<boolean>(profilingShowErrorFrames);
+
   const state = useTimeRangeAsync(
     ({ http }) => {
       return Promise.all([
@@ -59,6 +63,7 @@ export function DifferentialFlameGraphsView() {
           timeFrom: new Date(timeRange.start).getTime(),
           timeTo: new Date(timeRange.end).getTime(),
           kuery,
+          showErrorFrames,
         }),
         comparisonTimeRange.start && comparisonTimeRange.end
           ? fetchElasticFlamechart({
@@ -66,6 +71,7 @@ export function DifferentialFlameGraphsView() {
               timeFrom: new Date(comparisonTimeRange.start).getTime(),
               timeTo: new Date(comparisonTimeRange.end).getTime(),
               kuery: comparisonKuery,
+              showErrorFrames,
             })
           : Promise.resolve(undefined),
       ]).then(([primaryFlamegraph, comparisonFlamegraph]) => {
@@ -83,6 +89,7 @@ export function DifferentialFlameGraphsView() {
       comparisonTimeRange.start,
       comparisonTimeRange.end,
       comparisonKuery,
+      showErrorFrames,
     ]
   );
 

x-pack/plugins/profiling/public/views/flamegraphs/flamegraph/index.tsx

@@ -6,6 +6,7 @@
  */
 import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import React from 'react';
+import { profilingShowErrorFrames } from '@kbn/observability-plugin/common';
 import { AsyncComponent } from '../../../components/async_component';
 import { useProfilingDependencies } from '../../../components/contexts/profiling_dependencies/use_profiling_dependencies';
 import { FlameGraph } from '../../../components/flamegraph';
@@ -25,8 +26,11 @@ export function FlameGraphView() {
 
   const {
     services: { fetchElasticFlamechart },
+    start: { core },
   } = useProfilingDependencies();
 
+  const showErrorFrames = core.uiSettings.get<boolean>(profilingShowErrorFrames);
+
   const state = useTimeRangeAsync(
     ({ http }) => {
       return fetchElasticFlamechart({
@@ -34,9 +38,10 @@ export function FlameGraphView() {
         timeFrom: new Date(timeRange.start).getTime(),
         timeTo: new Date(timeRange.end).getTime(),
         kuery,
+        showErrorFrames,
       });
     },
-    [fetchElasticFlamechart, timeRange.start, timeRange.end, kuery]
+    [fetchElasticFlamechart, timeRange.start, timeRange.end, kuery, showErrorFrames]
   );
 
   const { data } = state;

x-pack/plugins/profiling/public/views/settings/index.tsx

@@ -25,6 +25,7 @@ import {
   profilingPervCPUWattArm64,
   profilingAWSCostDiscountRate,
   profilingCostPervCPUPerHour,
+  profilingShowErrorFrames,
 } from '@kbn/observability-plugin/common';
 import { useEditableSettings, useUiTracker } from '@kbn/observability-shared-plugin/public';
 import { isEmpty } from 'lodash';
@@ -47,6 +48,7 @@ const co2Settings = [
   profilingPervCPUWattArm64,
 ];
 const costSettings = [profilingAWSCostDiscountRate, profilingCostPervCPUPerHour];
+const miscSettings = [profilingShowErrorFrames];
 
 export function Settings() {
   const trackProfilingEvent = useUiTracker({ app: 'profiling' });
@@ -57,7 +59,7 @@ export function Settings() {
   } = useProfilingDependencies();
 
   const { fields, handleFieldChange, unsavedChanges, saveAll, isSaving, cleanUnsavedChanges } =
-    useEditableSettings('profiling', [...co2Settings, ...costSettings]);
+    useEditableSettings('profiling', [...co2Settings, ...costSettings, ...miscSettings]);
 
   async function handleSave() {
     try {
@@ -163,6 +165,20 @@ export function Settings() {
             },
             settings: costSettings,
           },
+          {
+            label: i18n.translate('xpack.profiling.settings.miscSection', {
+              defaultMessage: 'Miscellaneous settings',
+            }),
+            description: {
+              title: (
+                <FormattedMessage
+                  id="xpack.profiling.settings.misc.title"
+                  defaultMessage="Universal Profiling miscellaneous settings."
+                />
+              ),
+            },
+            settings: miscSettings,
+          },
         ].map((item) => (
           <>
             <EuiPanel key={item.label} grow={false} hasShadow={false} hasBorder paddingSize="none">

x-pack/plugins/profiling/server/routes/search_stacktraces.ts

@@ -13,11 +13,13 @@ export async function searchStackTraces({
   filter,
   sampleSize,
   durationSeconds,
+  showErrorFrames,
 }: {
   client: ProfilingESClient;
   filter: ProjectTimeQuery;
   sampleSize: number;
   durationSeconds: number;
+  showErrorFrames: boolean;
 }) {
   const response = await client.profilingStacktraces({
     query: filter,
@@ -25,5 +27,5 @@ export async function searchStackTraces({
     durationSeconds,
   });
 
-  return decodeStackTraceResponse(response);
+  return decodeStackTraceResponse(response, showErrorFrames);
 }

x-pack/plugins/profiling/server/routes/topn.test.ts

@@ -86,6 +86,7 @@ describe('TopN data from Elasticsearch', () => {
         searchField: ProfilingESField.StacktraceID,
         highCardinality: false,
         kuery: '',
+        showErrorFrames: false,
       });
 
       expect(client.search).toHaveBeenCalledTimes(2);

x-pack/plugins/profiling/server/routes/topn.ts

@@ -13,6 +13,7 @@ import {
   ProfilingESField,
   TopNType,
 } from '@kbn/profiling-utils';
+import { profilingShowErrorFrames } from '@kbn/observability-plugin/common';
 import { IDLE_SOCKET_TIMEOUT, RouteRegisterParameters } from '.';
 import { getRoutePaths, INDEX_EVENTS } from '../../common';
 import { computeBucketWidthFromTimeRangeAndBucketCount } from '../../common/histogram';
@@ -33,6 +34,7 @@ export async function topNElasticSearchQuery({
   searchField,
   highCardinality,
   kuery,
+  showErrorFrames,
 }: {
   client: ProfilingESClient;
   logger: Logger;
@@ -41,6 +43,7 @@ export async function topNElasticSearchQuery({
   searchField: string;
   highCardinality: boolean;
   kuery: string;
+  showErrorFrames: boolean;
 }): Promise<TopNResponse> {
   const filter = createCommonFilter({ timeFrom, timeTo, kuery });
   const targetSampleSize = 20000; // minimum number of samples to get statistically sound results
@@ -136,6 +139,7 @@ export async function topNElasticSearchQuery({
         filter: stackTraceFilter,
         sampleSize: targetSampleSize,
         durationSeconds: totalSeconds,
+        showErrorFrames,
       });
     }
   );
@@ -178,7 +182,9 @@ export function queryTopNCommon({
     },
     async (context, request, response) => {
       const { timeFrom, timeTo, kuery } = request.query;
-      const client = await getClient(context);
+      const [client, core] = await Promise.all([getClient(context), context.core]);
+
+      const showErrorFrames = await core.uiSettings.client.get<boolean>(profilingShowErrorFrames);
 
       try {
         return response.ok({
@@ -190,6 +196,7 @@ export function queryTopNCommon({
             searchField,
             highCardinality,
             kuery,
+            showErrorFrames,
           }),
         });
       } catch (error) {

x-pack/plugins/profiling_data_access/server/services/functions/index.ts

@@ -11,6 +11,7 @@ import {
   profilingDatacenterPUE,
   profilingPervCPUWattArm64,
   profilingPervCPUWattX86,
+  profilingShowErrorFrames,
 } from '@kbn/observability-plugin/common';
 import { CoreRequestHandlerContext, ElasticsearchClient } from '@kbn/core/server';
 import { createTopNFunctions } from '@kbn/profiling-utils';
@@ -52,6 +53,7 @@ export function createFetchFunctions({ createProfilingEsClient }: RegisterServic
       pervCPUWattArm64,
       awsCostDiscountRate,
       costPervCPUPerHour,
+      showErrorFrames,
     ] = await Promise.all([
       core.uiSettings.client.get<number>(profilingCo2PerKWH),
       core.uiSettings.client.get<number>(profilingDatacenterPUE),
@@ -59,6 +61,7 @@ export function createFetchFunctions({ createProfilingEsClient }: RegisterServic
       core.uiSettings.client.get<number>(profilingPervCPUWattArm64),
       core.uiSettings.client.get<number>(profilingAWSCostDiscountRate),
       core.uiSettings.client.get<number>(profilingCostPervCPUPerHour),
+      core.uiSettings.client.get<boolean>(profilingShowErrorFrames),
     ]);
 
     const profilingEsClient = createProfilingEsClient({ esClient });
@@ -77,6 +80,7 @@ export function createFetchFunctions({ createProfilingEsClient }: RegisterServic
         pervCPUWattArm64,
         awsCostDiscountRate: percentToFactor(awsCostDiscountRate),
         costPervCPUPerHour,
+        showErrorFrames,
       }
     );
 
@@ -89,6 +93,7 @@ export function createFetchFunctions({ createProfilingEsClient }: RegisterServic
         stackFrames,
         stackTraces,
         startIndex,
+        showErrorFrames,
       });
     });
 

x-pack/plugins/profiling_data_access/server/services/search_stack_traces/index.ts

@@ -22,6 +22,7 @@ export async function searchStackTraces({
   pervCPUWattArm64,
   awsCostDiscountRate,
   costPervCPUPerHour,
+  showErrorFrames,
 }: {
   client: ProfilingESClient;
   sampleSize: number;
@@ -35,6 +36,7 @@ export async function searchStackTraces({
   pervCPUWattArm64: number;
   awsCostDiscountRate: number;
   costPervCPUPerHour: number;
+  showErrorFrames: boolean;
 }) {
   const response = await client.profilingStacktraces({
     query: {
@@ -64,5 +66,5 @@ export async function searchStackTraces({
     costPervCPUPerHour,
   });
 
-  return decodeStackTraceResponse(response);
+  return decodeStackTraceResponse(response, showErrorFrames);
 }