[Profiling] Retrieve profiling data via Elasticsearch plugin (#147152)

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Dario Gieselaar <dario.gieselaar@elastic.co>

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -450,6 +450,10 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'integer',
     _meta: { description: 'Non-default value of setting.' },
   },
+  'observability:profilingElasticsearchPlugin': {
+    type: 'boolean',
+    _meta: { description: 'Non-default value of setting.' },
+  },
   'banners:placement': {
     type: 'keyword',
     _meta: { description: 'Non-default value of setting.' },

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -46,6 +46,7 @@ export interface UsageStats {
   'observability:apmAWSLambdaRequestCostPerMillion': number;
   'observability:enableInfrastructureHostsView': boolean;
   'observability:apmAgentExplorerView': boolean;
+  'observability:profilingElasticsearchPlugin': boolean;
   'visualize:enableLabs': boolean;
   'visualization:heatmap:maxBuckets': number;
   'visualization:colorMapping': string;

src/plugins/telemetry/schema/oss_plugins.json

@@ -8934,6 +8934,12 @@
             "description": "Non-default value of setting."
           }
         },
+        "observability:profilingElasticsearchPlugin": {
+          "type": "boolean",
+          "_meta": {
+            "description": "Non-default value of setting."
+          }
+        },
         "banners:placement": {
           "type": "keyword",
           "_meta": {

x-pack/plugins/observability/common/index.ts

@@ -29,6 +29,7 @@ export {
   apmAWSLambdaPriceFactor,
   apmAWSLambdaRequestCostPerMillion,
   enableCriticalPath,
+  profilingElasticsearchPlugin,
 } from './ui_settings_keys';
 
 export {

x-pack/plugins/observability/common/ui_settings_keys.ts

@@ -23,3 +23,4 @@ export const enableAgentExplorerView = 'observability:apmAgentExplorerView';
 export const apmAWSLambdaPriceFactor = 'observability:apmAWSLambdaPriceFactor';
 export const apmAWSLambdaRequestCostPerMillion = 'observability:apmAWSLambdaRequestCostPerMillion';
 export const enableCriticalPath = 'observability:apmEnableCriticalPath';
+export const profilingElasticsearchPlugin = 'observability:profilingElasticsearchPlugin';

x-pack/plugins/observability/server/ui_settings.ts

@@ -26,6 +26,7 @@ import {
   apmAWSLambdaRequestCostPerMillion,
   enableCriticalPath,
   enableInfrastructureHostsView,
+  profilingElasticsearchPlugin,
 } from '../common/ui_settings_keys';
 
 const technicalPreviewLabel = i18n.translate(
@@ -321,4 +322,21 @@ export const uiSettings: Record<string, UiSettings> = {
     type: 'boolean',
     showInLabs: true,
   },
+  [profilingElasticsearchPlugin]: {
+    category: [observabilityFeatureId],
+    name: i18n.translate('xpack.observability.profilingElasticsearchPlugin', {
+      defaultMessage: 'Use Elasticsearch profiler plugin',
+    }),
+    description: i18n.translate('xpack.observability.profilingElasticsearchPluginDescription', {
+      defaultMessage:
+        '{technicalPreviewLabel} Whether to load stacktraces using Elasticsearch profiler plugin.',
+      values: {
+        technicalPreviewLabel: `<em>[${technicalPreviewLabel}]</em>`,
+      },
+    }),
+    schema: schema.boolean(),
+    value: true,
+    requiresPageReload: true,
+    type: 'boolean',
+  },
 };

x-pack/plugins/profiling/common/stack_traces.ts

@@ -7,6 +7,45 @@
 
 import { ProfilingESField } from './elasticsearch';
 
+interface ProfilingEvents {
+  [key: string]: number;
+}
+
+interface ProfilingStackTrace {
+  ['file_ids']: string[];
+  ['frame_ids']: string[];
+  ['address_or_lines']: number[];
+  ['type_ids']: number[];
+}
+
+interface ProfilingStackTraces {
+  [key: string]: ProfilingStackTrace;
+}
+
+interface ProfilingStackFrame {
+  ['file_name']: string | undefined;
+  ['function_name']: string;
+  ['function_offset']: number | undefined;
+  ['line_number']: number | undefined;
+  ['source_type']: number | undefined;
+}
+
+interface ProfilingStackFrames {
+  [key: string]: ProfilingStackFrame;
+}
+
+interface ProfilingExecutables {
+  [key: string]: string;
+}
+
+export interface StackTraceResponse {
+  ['stack_trace_events']?: ProfilingEvents;
+  ['stack_traces']?: ProfilingStackTraces;
+  ['stack_frames']?: ProfilingStackFrames;
+  ['executables']?: ProfilingExecutables;
+  ['total_frames']: number;
+}
+
 export enum StackTracesDisplayOption {
   StackTraces = 'stackTraces',
   Percentage = 'percentage',

x-pack/plugins/profiling/server/routes/flamechart.ts

@@ -6,6 +6,7 @@
  */
 
 import { schema } from '@kbn/config-schema';
+
 import { RouteRegisterParameters } from '.';
 import { getRoutePaths } from '../../common';
 import { createCalleeTree } from '../../common/callee';
@@ -13,7 +14,7 @@ import { handleRouteHandlerError } from '../utils/handle_route_error_handler';
 import { createBaseFlameGraph } from '../../common/flamegraph';
 import { withProfilingSpan } from '../utils/with_profiling_span';
 import { getClient } from './compat';
-import { getExecutablesAndStackTraces } from './get_executables_and_stacktraces';
+import { getStackTraces } from './get_stacktraces';
 import { createCommonFilter } from './query';
 
 export function registerFlameChartSearchRoute({
@@ -39,6 +40,7 @@ export function registerFlameChartSearchRoute({
 
       try {
         const esClient = await getClient(context);
+        const profilingElasticsearchClient = createProfilingEsClient({ request, esClient });
         const filter = createCommonFilter({
           timeFrom,
           timeTo,
@@ -46,16 +48,19 @@ export function registerFlameChartSearchRoute({
         });
         const totalSeconds = timeTo - timeFrom;
 
+        const t0 = Date.now();
         const { stackTraceEvents, stackTraces, executables, stackFrames, totalFrames } =
-          await getExecutablesAndStackTraces({
+          await getStackTraces({
+            context,
             logger,
-            client: createProfilingEsClient({ request, esClient }),
+            client: profilingElasticsearchClient,
             filter,
             sampleSize: targetSampleSize,
           });
+        logger.info(`querying stacktraces took ${Date.now() - t0} ms`);
 
         const flamegraph = await withProfilingSpan('create_flamegraph', async () => {
-          const t0 = Date.now();
+          const t1 = Date.now();
           const tree = createCalleeTree(
             stackTraceEvents,
             stackTraces,
@@ -63,11 +68,11 @@ export function registerFlameChartSearchRoute({
             executables,
             totalFrames
           );
-          logger.info(`creating callee tree took ${Date.now() - t0} ms`);
+          logger.info(`creating callee tree took ${Date.now() - t1} ms`);
 
-          const t1 = Date.now();
+          const t2 = Date.now();
           const fg = createBaseFlameGraph(tree, totalSeconds);
-          logger.info(`creating flamegraph took ${Date.now() - t1} ms`);
+          logger.info(`creating flamegraph took ${Date.now() - t2} ms`);
 
           return fg;
         });

x-pack/plugins/profiling/server/routes/functions.ts

@@ -12,7 +12,7 @@ import { createTopNFunctions } from '../../common/functions';
 import { handleRouteHandlerError } from '../utils/handle_route_error_handler';
 import { withProfilingSpan } from '../utils/with_profiling_span';
 import { getClient } from './compat';
-import { getExecutablesAndStackTraces } from './get_executables_and_stacktraces';
+import { getStackTraces } from './get_stacktraces';
 import { createCommonFilter } from './query';
 
 const querySchema = schema.object({
@@ -44,21 +44,24 @@ export function registerTopNFunctionsSearchRoute({
 
         const targetSampleSize = 20000; // minimum number of samples to get statistically sound results
         const esClient = await getClient(context);
+        const profilingElasticsearchClient = createProfilingEsClient({ request, esClient });
         const filter = createCommonFilter({
           timeFrom,
           timeTo,
           kuery,
         });
 
-        const { stackFrames, stackTraceEvents, stackTraces, executables } =
-          await getExecutablesAndStackTraces({
-            client: createProfilingEsClient({ request, esClient }),
-            filter,
-            logger,
-            sampleSize: targetSampleSize,
-          });
-
         const t0 = Date.now();
+        const { stackTraceEvents, stackTraces, executables, stackFrames } = await getStackTraces({
+          context,
+          logger,
+          client: profilingElasticsearchClient,
+          filter,
+          sampleSize: targetSampleSize,
+        });
+        logger.info(`querying stacktraces took ${Date.now() - t0} ms`);
+
+        const t1 = Date.now();
         const topNFunctions = await withProfilingSpan('create_topn_functions', async () => {
           return createTopNFunctions(
             stackTraceEvents,
@@ -69,7 +72,7 @@ export function registerTopNFunctionsSearchRoute({
             endIndex
           );
         });
-        logger.info(`creating topN functions took ${Date.now() - t0} ms`);
+        logger.info(`creating topN functions took ${Date.now() - t1} ms`);
 
         logger.info('returning payload response to client');
 

x-pack/plugins/profiling/server/routes/get_stacktraces.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { RequestHandlerContext } from '@kbn/core/server';
+import { Logger } from '@kbn/logging';
+import { profilingElasticsearchPlugin } from '@kbn/observability-plugin/common';
+import { ProfilingESClient } from '../utils/create_profiling_es_client';
+import { getExecutablesAndStackTraces } from './get_executables_and_stacktraces';
+import { ProjectTimeQuery } from './query';
+import { searchStackTraces } from './search_stacktraces';
+
+export async function getStackTraces({
+  context,
+  logger,
+  client,
+  filter,
+  sampleSize,
+}: {
+  context: RequestHandlerContext;
+  logger: Logger;
+  client: ProfilingESClient;
+  filter: ProjectTimeQuery;
+  sampleSize: number;
+}) {
+  const core = await context.core;
+  const useElasticsearchPlugin = await core.uiSettings.client.get<boolean>(
+    profilingElasticsearchPlugin
+  );
+
+  if (useElasticsearchPlugin) {
+    return await searchStackTraces({
+      client,
+      filter,
+      sampleSize,
+    });
+  }
+
+  return await getExecutablesAndStackTraces({
+    logger,
+    client,
+    filter,
+    sampleSize,
+  });
+}

x-pack/plugins/profiling/server/routes/search_stacktraces.test.ts

@@ -0,0 +1,192 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { decodeStackTraceResponse } from './search_stacktraces';
+import { StackTraceResponse } from '../../common/stack_traces';
+
+describe('Stack trace response operations', () => {
+  test('empty stack trace response', () => {
+    const original: StackTraceResponse = {
+      total_frames: 0,
+    };
+
+    const expected = {
+      stackTraceEvents: new Map(),
+      stackTraces: new Map(),
+      stackFrames: new Map(),
+      executables: new Map(),
+      totalFrames: 0,
+    };
+
+    const decoded = decodeStackTraceResponse(original);
+
+    expect(decoded.executables.size).toEqual(expected.executables.size);
+    expect(decoded.executables.size).toEqual(0);
+
+    expect(decoded.stackFrames.size).toEqual(expected.stackFrames.size);
+    expect(decoded.stackFrames.size).toEqual(0);
+
+    expect(decoded.stackTraces.size).toEqual(expected.stackTraces.size);
+    expect(decoded.stackTraces.size).toEqual(0);
+
+    expect(decoded.stackTraceEvents.size).toEqual(expected.stackTraceEvents.size);
+    expect(decoded.stackTraceEvents.size).toEqual(0);
+
+    expect(decoded.totalFrames).toEqual(expected.totalFrames);
+    expect(decoded.totalFrames).toEqual(0);
+  });
+
+  test('stack trace response without undefineds', () => {
+    const original: StackTraceResponse = {
+      stack_trace_events: {
+        a: 1,
+      },
+      stack_traces: {
+        a: {
+          file_ids: ['abc'],
+          frame_ids: ['abc123'],
+          address_or_lines: [123],
+          type_ids: [0],
+        },
+      },
+      stack_frames: {
+        abc: {
+          file_name: 'pthread.c',
+          function_name: 'pthread_create',
+          function_offset: 0,
+          line_number: 0,
+          source_type: 5,
+        },
+      },
+      executables: {
+        abc: 'pthread.c',
+      },
+      total_frames: 1,
+    };
+
+    const expected = {
+      stackTraceEvents: new Map([['a', 1]]),
+      stackTraces: new Map([
+        [
+          'a',
+          {
+            FileIDs: ['abc'],
+            FrameIDs: ['abc123'],
+            AddressOrLines: [123],
+            Types: [0],
+          },
+        ],
+      ]),
+      stackFrames: new Map([
+        [
+          'abc',
+          {
+            FileName: 'pthread.c',
+            FunctionName: 'pthread_create',
+            FunctionOffset: 0,
+            LineNumber: 0,
+            SourceType: 5,
+          },
+        ],
+      ]),
+      executables: new Map([['abc', { FileName: 'pthread.c' }]]),
+      totalFrames: 1,
+    };
+
+    const decoded = decodeStackTraceResponse(original);
+
+    expect(decoded.executables.size).toEqual(expected.executables.size);
+    expect(decoded.executables.size).toEqual(1);
+
+    expect(decoded.stackFrames.size).toEqual(expected.stackFrames.size);
+    expect(decoded.stackFrames.size).toEqual(1);
+
+    expect(decoded.stackTraces.size).toEqual(expected.stackTraces.size);
+    expect(decoded.stackTraces.size).toEqual(1);
+
+    expect(decoded.stackTraceEvents.size).toEqual(expected.stackTraceEvents.size);
+    expect(decoded.stackTraceEvents.size).toEqual(1);
+
+    expect(decoded.totalFrames).toEqual(expected.totalFrames);
+    expect(decoded.totalFrames).toEqual(1);
+  });
+
+  test('stack trace response with undefineds', () => {
+    const original: StackTraceResponse = {
+      stack_trace_events: {
+        a: 1,
+      },
+      stack_traces: {
+        a: {
+          file_ids: ['abc'],
+          frame_ids: ['abc123'],
+          address_or_lines: [123],
+          type_ids: [0],
+        },
+      },
+      stack_frames: {
+        abc: {
+          file_name: undefined,
+          function_name: 'pthread_create',
+          function_offset: undefined,
+          line_number: undefined,
+          source_type: undefined,
+        },
+      },
+      executables: {
+        abc: 'pthread.c',
+      },
+      total_frames: 1,
+    };
+
+    const expected = {
+      stackTraceEvents: new Map([['a', 1]]),
+      stackTraces: new Map([
+        [
+          'a',
+          {
+            FileIDs: ['abc'],
+            FrameIDs: ['abc123'],
+            AddressOrLines: [123],
+            Types: [0],
+          },
+        ],
+      ]),
+      stackFrames: new Map([
+        [
+          'abc',
+          {
+            FileName: null,
+            FunctionName: 'pthread_create',
+            FunctionOffset: null,
+            LineNumber: null,
+            SourceType: null,
+          },
+        ],
+      ]),
+      executables: new Map([['abc', { FileName: 'pthread.c' }]]),
+      totalFrames: 1,
+    };
+
+    const decoded = decodeStackTraceResponse(original);
+
+    expect(decoded.executables.size).toEqual(expected.executables.size);
+    expect(decoded.executables.size).toEqual(1);
+
+    expect(decoded.stackFrames.size).toEqual(expected.stackFrames.size);
+    expect(decoded.stackFrames.size).toEqual(1);
+
+    expect(decoded.stackTraces.size).toEqual(expected.stackTraces.size);
+    expect(decoded.stackTraces.size).toEqual(1);
+
+    expect(decoded.stackTraceEvents.size).toEqual(expected.stackTraceEvents.size);
+    expect(decoded.stackTraceEvents.size).toEqual(1);
+
+    expect(decoded.totalFrames).toEqual(expected.totalFrames);
+    expect(decoded.totalFrames).toEqual(1);
+  });
+});

x-pack/plugins/profiling/server/routes/search_stacktraces.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  Executable,
+  FileID,
+  StackFrame,
+  StackFrameID,
+  StackTrace,
+  StackTraceID,
+} from '../../common/profiling';
+import { StackTraceResponse } from '../../common/stack_traces';
+import { ProfilingESClient } from '../utils/create_profiling_es_client';
+import { ProjectTimeQuery } from './query';
+
+export function decodeStackTraceResponse(response: StackTraceResponse) {
+  const stackTraceEvents: Map<StackTraceID, number> = new Map();
+  for (const [key, value] of Object.entries(response.stack_trace_events ?? {})) {
+    stackTraceEvents.set(key, value);
+  }
+
+  const stackTraces: Map<StackTraceID, StackTrace> = new Map();
+  for (const [key, value] of Object.entries(response.stack_traces ?? {})) {
+    stackTraces.set(key, {
+      FrameIDs: value.frame_ids,
+      FileIDs: value.file_ids,
+      AddressOrLines: value.address_or_lines,
+      Types: value.type_ids,
+    } as StackTrace);
+  }
+
+  const stackFrames: Map<StackFrameID, StackFrame> = new Map();
+  for (const [key, value] of Object.entries(response.stack_frames ?? {})) {
+    stackFrames.set(key, {
+      FileName: value.file_name,
+      FunctionName: value.function_name,
+      FunctionOffset: value.function_offset,
+      LineNumber: value.line_number,
+      SourceType: value.source_type,
+    } as StackFrame);
+  }
+
+  const executables: Map<FileID, Executable> = new Map();
+  for (const [key, value] of Object.entries(response.executables ?? {})) {
+    executables.set(key, {
+      FileName: value,
+    } as Executable);
+  }
+
+  return {
+    stackTraceEvents,
+    stackTraces,
+    stackFrames,
+    executables,
+    totalFrames: response.total_frames,
+  };
+}
+
+export async function searchStackTraces({
+  client,
+  filter,
+  sampleSize,
+}: {
+  client: ProfilingESClient;
+  filter: ProjectTimeQuery;
+  sampleSize: number;
+}) {
+  const response = await client.profilingStacktraces({ query: filter, sampleSize });
+
+  return decodeStackTraceResponse(response);
+}

x-pack/plugins/profiling/server/routes/topn.test.ts

@@ -42,6 +42,17 @@ describe('TopN data from Elasticsearch', () => {
       (operationName, request) =>
         context.elasticsearch.client.asCurrentUser.search(request) as Promise<any>
     ),
+    profilingStacktraces: jest.fn(
+      (request) =>
+        context.elasticsearch.client.asCurrentUser.transport.request({
+          method: 'POST',
+          path: encodeURI('_profiling/stacktraces'),
+          body: {
+            query: request.query,
+            sample_size: request.sampleSize,
+          },
+        }) as Promise<any>
+    ),
   };
   const logger = loggerMock.create();
 

x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts

@@ -10,8 +10,10 @@ import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import type { KibanaRequest } from '@kbn/core/server';
 import { unwrapEsResponse } from '@kbn/observability-plugin/server';
 import { MgetRequest, MgetResponse } from '@elastic/elasticsearch/lib/api/types';
+import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { ProfilingESEvent } from '../../common/elasticsearch';
 import { withProfilingSpan } from './with_profiling_span';
+import { StackTraceResponse } from '../../common/stack_traces';
 
 export function cancelEsRequestOnAbort<T extends Promise<any>>(
   promise: T,
@@ -34,6 +36,10 @@ export interface ProfilingESClient {
     operationName: string,
     mgetRequest: MgetRequest
   ): Promise<MgetResponse<TDocument>>;
+  profilingStacktraces({}: {
+    query: QueryDslQueryContainer;
+    sampleSize: number;
+  }): Promise<StackTraceResponse>;
 }
 
 export function createProfilingEsClient({
@@ -84,5 +90,31 @@ export function createProfilingEsClient({
 
       return unwrapEsResponse(promise);
     },
+    profilingStacktraces({ query, sampleSize }) {
+      const controller = new AbortController();
+
+      const promise = withProfilingSpan('_profiling/stacktraces', () => {
+        return cancelEsRequestOnAbort(
+          esClient.transport.request(
+            {
+              method: 'POST',
+              path: encodeURI('/_profiling/stacktraces'),
+              body: {
+                query,
+                sample_size: sampleSize,
+              },
+            },
+            {
+              signal: controller.signal,
+              meta: true,
+            }
+          ),
+          request,
+          controller
+        );
+      });
+
+      return unwrapEsResponse(promise) as Promise<StackTraceResponse>;
+    },
   };
 }