[Profiling] Profiling data access plugin (#165198)

This is part 1 of a series of PRs to expose the flamegraph to be used by
other plugins.

**The problem**
Currently for plugin-A to show data from plugin-B, it needs to add
dependency on plugin-B. If plugin-B wants to show data from plugin-A, it
also needs to add plugin-A as a dependency, and here is where the
problem happens. In such scenario, we have a cyclic dependency problem.

**The solution**
To solve this problem a new plugin is created, `profiling-data-access`.
This plugin exposes services that consumer plugins can call in other to
have the data they need to show on their end. The `profiling` plugin is
also using this new plugin. For now, only the flamegraph service is
available, The idea is to slowly migrate the data fetching from
profiling to this new plugin in other to facilitate the integration
across plugins.

**Why some many files?**
As I said, only the flamegraph logic was moved to the new plugin, but it
has many files that it needs to properly build the response of the
service call. I moved all these files to the `common` folder inside the
new plugin and adjusted the imports in the profiling plugin.

<img width="1032" alt="Screenshot 2023-08-31 at 09 29 27"
src="287bd28e-b834-45e0-8042-576d1fcff6cd">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -545,6 +545,7 @@ packages/kbn-plugin-helpers @elastic/kibana-operations
 examples/portable_dashboards_example @elastic/kibana-presentation
 examples/preboot_example @elastic/kibana-security @elastic/kibana-core
 src/plugins/presentation_util @elastic/kibana-presentation
+x-pack/plugins/profiling_data_access @elastic/profiling-ui
 x-pack/plugins/profiling @elastic/profiling-ui
 x-pack/packages/kbn-random-sampling @elastic/kibana-visualizations
 packages/kbn-react-field @elastic/kibana-data-discovery

docs/developer/plugin-list.asciidoc

@@ -697,6 +697,10 @@ Elastic.
 |Universal Profiling provides fleet-wide, whole-system, continuous profiling with zero instrumentation. Get a comprehensive understanding of what lines of code are consuming compute resources throughout your entire fleet by visualizing your data in Kibana using the flamegraph, stacktraces, and top functions views.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/profiling_data_access[profilingDataAccess]
+|WARNING: Missing README.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/remote_clusters/README.md[remoteClusters]
 |This plugin helps users manage their remote clusters, which enable cross-cluster search and cross-cluster replication.
 

package.json

@@ -556,6 +556,7 @@
     "@kbn/portable-dashboards-example": "link:examples/portable_dashboards_example",
     "@kbn/preboot-example-plugin": "link:examples/preboot_example",
     "@kbn/presentation-util-plugin": "link:src/plugins/presentation_util",
+    "@kbn/profiling-data-access-plugin": "link:x-pack/plugins/profiling_data_access",
     "@kbn/profiling-plugin": "link:x-pack/plugins/profiling",
     "@kbn/random-sampling": "link:x-pack/packages/kbn-random-sampling",
     "@kbn/react-field": "link:packages/kbn-react-field",

tsconfig.base.json

@@ -1084,6 +1084,8 @@
       "@kbn/preboot-example-plugin/*": ["examples/preboot_example/*"],
       "@kbn/presentation-util-plugin": ["src/plugins/presentation_util"],
       "@kbn/presentation-util-plugin/*": ["src/plugins/presentation_util/*"],
+      "@kbn/profiling-data-access-plugin": ["x-pack/plugins/profiling_data_access"],
+      "@kbn/profiling-data-access-plugin/*": ["x-pack/plugins/profiling_data_access/*"],
       "@kbn/profiling-plugin": ["x-pack/plugins/profiling"],
       "@kbn/profiling-plugin/*": ["x-pack/plugins/profiling/*"],
       "@kbn/random-sampling": ["x-pack/packages/kbn-random-sampling"],

x-pack/plugins/profiling/common/base64.ts

@@ -1,22 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-export const safeBase64Decoder = [
-  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0,
-  0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
-  25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,
-  45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0,
-];
-
-export const safeBase64Encoder =
-  'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234456789-_';
-
-/* eslint no-bitwise: ["error", { "allow": ["&"] }] */
-export function charCodeAt(input: string, i: number): number {
-  return safeBase64Decoder[input.charCodeAt(i) & 0x7f];
-}

x-pack/plugins/profiling/common/columnar_view_model.test.ts

@@ -7,12 +7,14 @@
 
 import { sum } from 'lodash';
 
-import { createCalleeTree } from './callee';
+import { createCalleeTree } from '@kbn/profiling-data-access-plugin/common/callee';
 import { createColumnarViewModel } from './columnar_view_model';
-import { createBaseFlameGraph, createFlameGraph } from './flamegraph';
-import { decodeStackTraceResponse } from './stack_traces';
-
-import { stackTraceFixtures } from './__fixtures__/stacktraces';
+import {
+  createBaseFlameGraph,
+  createFlameGraph,
+} from '@kbn/profiling-data-access-plugin/common/flamegraph';
+import { decodeStackTraceResponse } from '@kbn/profiling-data-access-plugin/common/stack_traces';
+import { stackTraceFixtures } from '@kbn/profiling-data-access-plugin/common/__fixtures__/stacktraces';
 
 describe('Columnar view model operations', () => {
   stackTraceFixtures.forEach(({ response, seconds, upsampledBy }) => {

x-pack/plugins/profiling/common/columnar_view_model.ts

@@ -6,8 +6,7 @@
  */
 
 import { ColumnarViewModel } from '@elastic/charts';
-
-import { ElasticFlameGraph } from './flamegraph';
+import { ElasticFlameGraph } from '@kbn/profiling-data-access-plugin/common/flamegraph';
 import { frameTypeToRGB, rgbToRGBA } from './frame_type_colors';
 
 function normalize(n: number, lower: number, upper: number): number {

x-pack/plugins/profiling/common/frame_type_colors.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FrameType } from './profiling';
+import { FrameType } from '@kbn/profiling-data-access-plugin/common/profiling';
 
 /*
  * Helper to calculate the color of a given block to be drawn. The desirable outcomes of this are:

x-pack/plugins/profiling/common/functions.test.ts

@@ -6,11 +6,9 @@
  */
 
 import { sum } from 'lodash';
-
 import { createTopNFunctions } from './functions';
-import { decodeStackTraceResponse } from './stack_traces';
-
-import { stackTraceFixtures } from './__fixtures__/stacktraces';
+import { decodeStackTraceResponse } from '@kbn/profiling-data-access-plugin/common/stack_traces';
+import { stackTraceFixtures } from '@kbn/profiling-data-access-plugin/common/__fixtures__/stacktraces';
 
 describe('TopN function operations', () => {
   stackTraceFixtures.forEach(({ response, seconds, upsampledBy }) => {

x-pack/plugins/profiling/common/functions.ts

@@ -6,7 +6,10 @@
  */
 import * as t from 'io-ts';
 import { sumBy } from 'lodash';
-import { createFrameGroupID, FrameGroupID } from './frame_group';
+import {
+  createFrameGroupID,
+  FrameGroupID,
+} from '@kbn/profiling-data-access-plugin/common/frame_group';
 import {
   createStackFrameMetadata,
   emptyExecutable,
@@ -19,7 +22,7 @@ import {
   StackFrameMetadata,
   StackTrace,
   StackTraceID,
-} from './profiling';
+} from '@kbn/profiling-data-access-plugin/common/profiling';
 
 interface TopNFunctionAndFrameGroup {
   Frame: StackFrameMetadata;

x-pack/plugins/profiling/common/index.ts

@@ -41,18 +41,6 @@ export function timeRangeFromRequest(request: any): [number, number] {
   return [timeFrom, timeTo];
 }
 
-// Converts from a Map object to a Record object since Map objects are not
-// serializable to JSON by default
-export function fromMapToRecord<K extends string, V>(m: Map<K, V>): Record<string, V> {
-  const output: Record<string, V> = {};
-
-  for (const [key, value] of m) {
-    output[key] = value;
-  }
-
-  return output;
-}
-
 export const NOT_AVAILABLE_LABEL = i18n.translate('xpack.profiling.notAvailableLabel', {
   defaultMessage: 'N/A',
 });

x-pack/plugins/profiling/common/run_length_encoding.test.ts

@@ -1,170 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { runLengthDecode, runLengthDecodeBase64Url, runLengthEncode } from './run_length_encoding';
-
-describe('Run-length encoding operations', () => {
-  test('run length is fully reversible', () => {
-    const tests: number[][] = [[], [0], [0, 1, 2, 3], [0, 1, 1, 2, 2, 2, 3, 3, 3, 3]];
-
-    for (const t of tests) {
-      expect(runLengthDecode(runLengthEncode(t))).toEqual(t);
-    }
-  });
-
-  test('runLengthDecode with optional parameter', () => {
-    const tests: Array<{
-      bytes: Buffer;
-      expected: number[];
-    }> = [
-      {
-        bytes: Buffer.from([0x5, 0x0, 0x2, 0x2]),
-        expected: [0, 0, 0, 0, 0, 2, 2],
-      },
-      {
-        bytes: Buffer.from([0x1, 0x8]),
-        expected: [8],
-      },
-    ];
-
-    for (const t of tests) {
-      expect(runLengthDecode(t.bytes, t.expected.length)).toEqual(t.expected);
-    }
-  });
-
-  test('runLengthDecode with larger output than available input', () => {
-    const bytes = Buffer.from([0x5, 0x0, 0x2, 0x2]);
-    const decoded = [0, 0, 0, 0, 0, 2, 2];
-    const expected = decoded.concat(Array(decoded.length).fill(0));
-
-    expect(runLengthDecode(bytes, expected.length)).toEqual(expected);
-  });
-
-  test('runLengthDecode without optional parameter', () => {
-    const tests: Array<{
-      bytes: Buffer;
-      expected: number[];
-    }> = [
-      {
-        bytes: Buffer.from([0x5, 0x0, 0x2, 0x2]),
-        expected: [0, 0, 0, 0, 0, 2, 2],
-      },
-      {
-        bytes: Buffer.from([0x1, 0x8]),
-        expected: [8],
-      },
-    ];
-
-    for (const t of tests) {
-      expect(runLengthDecode(t.bytes)).toEqual(t.expected);
-    }
-  });
-
-  test('runLengthDecode works for very long runs', () => {
-    const tests: Array<{
-      bytes: Buffer;
-      expected: number[];
-    }> = [
-      {
-        bytes: Buffer.from([0x5, 0x2, 0xff, 0x0]),
-        expected: [2, 2, 2, 2, 2].concat(Array(255).fill(0)),
-      },
-      {
-        bytes: Buffer.from([0xff, 0x2, 0x1, 0x2]),
-        expected: Array(256).fill(2),
-      },
-    ];
-
-    for (const t of tests) {
-      expect(runLengthDecode(t.bytes)).toEqual(t.expected);
-    }
-  });
-
-  test('runLengthEncode works for very long runs', () => {
-    const tests: Array<{
-      numbers: number[];
-      expected: Buffer;
-    }> = [
-      {
-        numbers: [2, 2, 2, 2, 2].concat(Array(255).fill(0)),
-        expected: Buffer.from([0x5, 0x2, 0xff, 0x0]),
-      },
-      {
-        numbers: Array(256).fill(2),
-        expected: Buffer.from([0xff, 0x2, 0x1, 0x2]),
-      },
-    ];
-
-    for (const t of tests) {
-      expect(runLengthEncode(t.numbers)).toEqual(t.expected);
-    }
-  });
-
-  test('runLengthDecodeBase64Url', () => {
-    const tests: Array<{
-      data: string;
-      expected: number[];
-    }> = [
-      {
-        data: 'CQM',
-        expected: [3, 3, 3, 3, 3, 3, 3, 3, 3],
-      },
-      {
-        data: 'AQkBCAEHAQYBBQEEAQMBAgEBAQA',
-        expected: [9, 8, 7, 6, 5, 4, 3, 2, 1, 0],
-      },
-      {
-        data: 'EgMHBA',
-        expected: Array(18).fill(3).concat(Array(7).fill(4)),
-      },
-      {
-        data: 'CAMfBQIDEAQ',
-        expected: Array(8)
-          .fill(3)
-          .concat(Array(31).fill(5))
-          .concat([3, 3])
-          .concat(Array(16).fill(4)),
-      },
-    ];
-
-    for (const t of tests) {
-      expect(runLengthDecodeBase64Url(t.data, t.data.length, t.expected.length)).toEqual(
-        t.expected
-      );
-    }
-  });
-
-  test('runLengthDecodeBase64Url with larger output than available input', () => {
-    const data = Buffer.from([0x5, 0x0, 0x3, 0x2]).toString('base64url');
-    const decoded = [0, 0, 0, 0, 0, 2, 2, 2];
-    const expected = decoded.concat(Array(decoded.length).fill(0));
-
-    expect(runLengthDecodeBase64Url(data, data.length, expected.length)).toEqual(expected);
-  });
-
-  test('runLengthDecodeBase64Url works for very long runs', () => {
-    const tests: Array<{
-      data: string;
-      expected: number[];
-    }> = [
-      {
-        data: Buffer.from([0x5, 0x2, 0xff, 0x0]).toString('base64url'),
-        expected: [2, 2, 2, 2, 2].concat(Array(255).fill(0)),
-      },
-      {
-        data: Buffer.from([0xff, 0x2, 0x1, 0x2]).toString('base64url'),
-        expected: Array(256).fill(2),
-      },
-    ];
-
-    for (const t of tests) {
-      expect(runLengthDecodeBase64Url(t.data, t.data.length, t.expected.length)).toEqual(
-        t.expected
-      );
-    }
-  });
-});

x-pack/plugins/profiling/common/run_length_encoding.ts

@@ -1,199 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { charCodeAt } from './base64';
-
-// runLengthEncode run-length encodes the input array.
-//
-// The input is a list of uint8s. The output is a binary stream of
-// 2-byte pairs (first byte is the length and the second byte is the
-// binary representation of the object) in reverse order.
-//
-// E.g. uint8 array [0, 0, 0, 0, 0, 2, 2, 2] is converted into the byte
-// array [5, 0, 3, 2].
-export function runLengthEncode(input: number[]): Buffer {
-  const output: number[] = [];
-
-  if (input.length === 0) {
-    return Buffer.from(output);
-  }
-
-  let count = 1;
-  let current = input[0];
-
-  for (let i = 1; i < input.length; i++) {
-    const next = input[i];
-
-    if (next === current && count < 255) {
-      count++;
-      continue;
-    }
-
-    output.push(count, current);
-
-    count = 1;
-    current = next;
-  }
-
-  output.push(count, current);
-
-  return Buffer.from(output);
-}
-
-function copyNumber(target: number[], value: number, offset: number, end: number) {
-  for (let i = offset; i < end; i++) {
-    target[i] = value;
-  }
-}
-
-// runLengthDecode decodes a run-length encoding for the input array.
-//
-// The input is a binary stream of 2-byte pairs (first byte is the length and the
-// second byte is the binary representation of the object). The output is a list of
-// uint8s.
-//
-// E.g. byte array [5, 0, 3, 2] is converted into an uint8 array like
-// [0, 0, 0, 0, 0, 2, 2, 2].
-export function runLengthDecode(input: Buffer, outputSize?: number): number[] {
-  let size;
-
-  if (typeof outputSize === 'undefined') {
-    size = 0;
-    for (let i = 0; i < input.length; i += 2) {
-      size += input[i];
-    }
-  } else {
-    size = outputSize;
-  }
-
-  const output: number[] = new Array(size);
-
-  let idx = 0;
-  for (let i = 0; i < input.length; i += 2) {
-    for (let j = 0; j < input[i]; j++) {
-      output[idx] = input[i + 1];
-      idx++;
-    }
-  }
-
-  // Due to truncation of the frame types for stacktraces longer than 255,
-  // the expected output size and the actual decoded size can be different.
-  // Ordinarily, these two values should be the same.
-  //
-  // We have decided to fill in the remainder of the output array with zeroes
-  // as a reasonable default. Without this step, the output array would have
-  // undefined values.
-  copyNumber(output, 0, idx, size);
-
-  return output;
-}
-
-// runLengthDecodeBase64Url decodes a run-length encoding for the
-// base64-encoded input string.
-//
-// The input is a base64-encoded string. The output is a list of uint8s.
-//
-// E.g. string 'BQADAg' is converted into an uint8 array like
-// [0, 0, 0, 0, 0, 2, 2, 2].
-//
-// The motivating intent for this method is to unpack a base64-encoded
-// run-length encoding without using intermediate storage.
-//
-// This method relies on these assumptions and details:
-// - array encoded using run-length and base64 always returns string of length
-//   0, 3, or 6 (mod 8)
-// - since original array is composed of uint8s, we ignore Unicode codepoints
-// - JavaScript bitwise operators operate on 32-bits so decoding must be done
-//   in 32-bit chunks
-
-/* eslint no-bitwise: ["error", { "allow": ["<<", ">>", ">>=", "&", "|"] }] */
-export function runLengthDecodeBase64Url(input: string, size: number, capacity: number): number[] {
-  const output = new Array<number>(capacity);
-  const multipleOf8 = Math.floor(size / 8);
-  const remainder = size % 8;
-
-  let n = 0;
-  let count = 0;
-  let value = 0;
-  let i = 0;
-  let j = 0;
-
-  for (i = 0; i < multipleOf8 * 8; i += 8) {
-    n =
-      (charCodeAt(input, i) << 26) |
-      (charCodeAt(input, i + 1) << 20) |
-      (charCodeAt(input, i + 2) << 14) |
-      (charCodeAt(input, i + 3) << 8) |
-      (charCodeAt(input, i + 4) << 2) |
-      (charCodeAt(input, i + 5) >> 4);
-
-    count = (n >> 24) & 0xff;
-    value = (n >> 16) & 0xff;
-
-    copyNumber(output, value, j, j + count);
-    j += count;
-
-    count = (n >> 8) & 0xff;
-    value = n & 0xff;
-
-    copyNumber(output, value, j, j + count);
-    j += count;
-
-    n =
-      ((charCodeAt(input, i + 5) & 0xf) << 12) |
-      (charCodeAt(input, i + 6) << 6) |
-      charCodeAt(input, i + 7);
-
-    count = (n >> 8) & 0xff;
-    value = n & 0xff;
-
-    copyNumber(output, value, j, j + count);
-    j += count;
-  }
-
-  if (remainder === 6) {
-    n =
-      (charCodeAt(input, i) << 26) |
-      (charCodeAt(input, i + 1) << 20) |
-      (charCodeAt(input, i + 2) << 14) |
-      (charCodeAt(input, i + 3) << 8) |
-      (charCodeAt(input, i + 4) << 2) |
-      (charCodeAt(input, i + 5) >> 4);
-
-    count = (n >> 24) & 0xff;
-    value = (n >> 16) & 0xff;
-
-    copyNumber(output, value, j, j + count);
-    j += count;
-
-    count = (n >> 8) & 0xff;
-    value = n & 0xff;
-
-    copyNumber(output, value, j, j + count);
-    j += count;
-  } else if (remainder === 3) {
-    n = (charCodeAt(input, i) << 12) | (charCodeAt(input, i + 1) << 6) | charCodeAt(input, i + 2);
-    n >>= 2;
-
-    count = (n >> 8) & 0xff;
-    value = n & 0xff;
-
-    copyNumber(output, value, j, j + count);
-    j += count;
-  }
-
-  // Due to truncation of the frame types for stacktraces longer than 255,
-  // the expected output size and the actual decoded size can be different.
-  // Ordinarily, these two values should be the same.
-  //
-  // We have decided to fill in the remainder of the output array with zeroes
-  // as a reasonable default. Without this step, the output array would have
-  // undefined values.
-  copyNumber(output, 0, j, capacity);
-
-  return output;
-}

x-pack/plugins/profiling/common/topn.ts

@@ -9,9 +9,9 @@ import { euiPaletteColorBlind } from '@elastic/eui';
 import { InferSearchResponseOf } from '@kbn/es-types';
 import { i18n } from '@kbn/i18n';
 import { orderBy } from 'lodash';
-import { ProfilingESField } from './elasticsearch';
+import { ProfilingESField } from '@kbn/profiling-data-access-plugin/common/elasticsearch';
+import { StackFrameMetadata } from '@kbn/profiling-data-access-plugin/common/profiling';
 import { createUniformBucketsForTimeRange } from './histogram';
-import { StackFrameMetadata } from './profiling';
 
 export const OTHER_BUCKET_LABEL = i18n.translate('xpack.profiling.topn.otherBucketLabel', {
   defaultMessage: 'Other',

x-pack/plugins/profiling/kibana.jsonc

@@ -23,12 +23,13 @@
       "observabilityShared",
       "observabilityAIAssistant",
       "unifiedSearch",
-      "share"
+      "share",
+      "profilingDataAccess"
     ],
     "requiredBundles": [
       "kibanaReact",
       "kibanaUtils",
-      "observabilityAIAssistant",
+      "observabilityAIAssistant"
     ]
   }
 }

x-pack/plugins/profiling/public/components/flamegraph/index.tsx

@@ -19,7 +19,7 @@ import { EuiFlexGroup, EuiFlexItem, useEuiTheme } from '@elastic/eui';
 import { Maybe } from '@kbn/observability-plugin/common/typings';
 import React, { useEffect, useMemo, useState } from 'react';
 import { useUiTracker } from '@kbn/observability-shared-plugin/public';
-import { ElasticFlameGraph } from '../../../common/flamegraph';
+import { ElasticFlameGraph } from '@kbn/profiling-data-access-plugin/common/flamegraph';
 import { getFlamegraphModel } from '../../utils/get_flamegraph_model';
 import { FlameGraphLegend } from './flame_graph_legend';
 import { FrameInformationWindow } from '../frame_information_window';

x-pack/plugins/profiling/public/components/frame_information_window/get_information_rows.ts

@@ -6,8 +6,8 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import { describeFrameType } from '@kbn/profiling-data-access-plugin/common/profiling';
 import { NOT_AVAILABLE_LABEL } from '../../../common';
-import { describeFrameType } from '../../../common/profiling';
 
 export function getInformationRows({
   fileID,

x-pack/plugins/profiling/public/components/frame_information_window/index.tsx

@@ -13,7 +13,10 @@ import {
   useObservabilityAIAssistant,
 } from '@kbn/observability-ai-assistant-plugin/public';
 import React, { useMemo } from 'react';
-import { FrameSymbolStatus, getFrameSymbolStatus } from '../../../common/profiling';
+import {
+  FrameSymbolStatus,
+  getFrameSymbolStatus,
+} from '@kbn/profiling-data-access-plugin/common/profiling';
 import { FrameInformationPanel } from './frame_information_panel';
 import { getImpactRows } from './get_impact_rows';
 import { getInformationRows } from './get_information_rows';

x-pack/plugins/profiling/public/components/frame_information_window/missing_symbols_callout.stories.tsx

@@ -8,7 +8,7 @@
 import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import { Meta } from '@storybook/react';
 import React from 'react';
-import { FrameType } from '../../../common/profiling';
+import { FrameType } from '@kbn/profiling-data-access-plugin/common/profiling';
 import { MockProfilingDependenciesStorybook } from '../contexts/profiling_dependencies/mock_profiling_dependencies_storybook';
 import { MissingSymbolsCallout } from './missing_symbols_callout';
 

x-pack/plugins/profiling/public/components/frame_information_window/missing_symbols_callout.tsx

@@ -9,7 +9,7 @@ import { EuiButton, EuiCallOut, EuiLink } from '@elastic/eui';
 import React from 'react';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { i18n } from '@kbn/i18n';
-import { FrameType, getLanguageType } from '../../../common/profiling';
+import { FrameType, getLanguageType } from '@kbn/profiling-data-access-plugin/common/profiling';
 import { PROFILING_FEEDBACK_LINK } from '../profiling_app_page_template';
 import { useProfilingDependencies } from '../contexts/profiling_dependencies/use_profiling_dependencies';
 import { useProfilingRouter } from '../../hooks/use_profiling_router';

x-pack/plugins/profiling/public/components/stack_frame_summary/index.tsx

@@ -6,7 +6,11 @@
  */
 import { EuiFlexGroup, EuiFlexItem, EuiLink, EuiText } from '@elastic/eui';
 import React from 'react';
-import { getCalleeFunction, getCalleeSource, StackFrameMetadata } from '../../../common/profiling';
+import {
+  getCalleeFunction,
+  getCalleeSource,
+  StackFrameMetadata,
+} from '@kbn/profiling-data-access-plugin/common/profiling';
 
 interface Props {
   frame: StackFrameMetadata;

x-pack/plugins/profiling/public/components/subchart.tsx

@@ -32,7 +32,7 @@ import {
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import React from 'react';
-import { StackFrameMetadata } from '../../common/profiling';
+import { StackFrameMetadata } from '@kbn/profiling-data-access-plugin/common/profiling';
 import { CountPerTime, OTHER_BUCKET_LABEL, TopNSample } from '../../common/topn';
 import { useKibanaTimeZoneSetting } from '../hooks/use_kibana_timezone_setting';
 import { useProfilingChartsTheme } from '../hooks/use_profiling_charts_theme';

x-pack/plugins/profiling/public/components/topn_functions/utils.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 import { keyBy } from 'lodash';
+import { StackFrameMetadata } from '@kbn/profiling-data-access-plugin/common/profiling';
 import { TopNFunctions } from '../../../common/functions';
-import { StackFrameMetadata } from '../../../common/profiling';
 import { calculateImpactEstimates } from '../../../common/calculate_impact_estimates';
 
 export function getColorLabel(percent: number) {

x-pack/plugins/profiling/public/routing/index.tsx

@@ -9,8 +9,11 @@ import { toNumberRt } from '@kbn/io-ts-utils';
 import { createRouter, Outlet } from '@kbn/typed-react-router-config';
 import * as t from 'io-ts';
 import React from 'react';
+import {
+  StackTracesDisplayOption,
+  TopNType,
+} from '@kbn/profiling-data-access-plugin/common/stack_traces';
 import { TopNFunctionSortField, topNFunctionSortFieldRt } from '../../common/functions';
-import { StackTracesDisplayOption, TopNType } from '../../common/stack_traces';
 import {
   indexLifecyclePhaseRt,
   IndexLifecyclePhaseSelectOption,

x-pack/plugins/profiling/public/services.ts

@@ -5,8 +5,12 @@
  * 2.0.
  */
 import { HttpFetchQuery } from '@kbn/core/public';
+import {
+  BaseFlameGraph,
+  createFlameGraph,
+  ElasticFlameGraph,
+} from '@kbn/profiling-data-access-plugin/common/flamegraph';
 import { getRoutePaths } from '../common';
-import { BaseFlameGraph, createFlameGraph, ElasticFlameGraph } from '../common/flamegraph';
 import { TopNFunctions } from '../common/functions';
 import type {
   IndexLifecyclePhaseSelectOption,

x-pack/plugins/profiling/public/utils/get_flamegraph_model/index.ts

@@ -8,10 +8,10 @@ import { ColumnarViewModel } from '@elastic/charts';
 import { i18n } from '@kbn/i18n';
 import d3 from 'd3';
 import { compact, range, sum, uniqueId } from 'lodash';
+import { describeFrameType, FrameType } from '@kbn/profiling-data-access-plugin/common/profiling';
+import { ElasticFlameGraph } from '@kbn/profiling-data-access-plugin/common/flamegraph';
 import { createColumnarViewModel } from '../../../common/columnar_view_model';
-import { ElasticFlameGraph } from '../../../common/flamegraph';
 import { FRAME_TYPE_COLOR_MAP, rgbToRGBA } from '../../../common/frame_type_colors';
-import { describeFrameType, FrameType } from '../../../common/profiling';
 import { ComparisonMode } from '../../components/normalization_menu';
 import { getInterpolationValue } from './get_interpolation_value';
 

x-pack/plugins/profiling/public/views/stack_traces_view/get_stack_traces_tabs.ts

@@ -8,7 +8,7 @@
 import { EuiPageHeaderContentProps } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { TypeOf } from '@kbn/typed-react-router-config';
-import { TopNType } from '../../../common/stack_traces';
+import { TopNType } from '@kbn/profiling-data-access-plugin/common/stack_traces';
 import { StatefulProfilingRouter } from '../../hooks/use_profiling_router';
 import { ProfilingRoutes } from '../../routing';
 

x-pack/plugins/profiling/public/views/stack_traces_view/index.tsx

@@ -7,7 +7,10 @@
 import { EuiButton, EuiButtonGroup, EuiFlexGroup, EuiFlexItem, EuiPanel } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import React from 'react';
-import { StackTracesDisplayOption, TopNType } from '../../../common/stack_traces';
+import {
+  StackTracesDisplayOption,
+  TopNType,
+} from '@kbn/profiling-data-access-plugin/common/stack_traces';
 import { groupSamplesByCategory, TopNResponse } from '../../../common/topn';
 import { useProfilingParams } from '../../hooks/use_profiling_params';
 import { useProfilingRouter } from '../../hooks/use_profiling_router';

x-pack/plugins/profiling/public/views/stack_traces_view/utils.test.ts

@@ -4,7 +4,10 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { StackTracesDisplayOption, TopNType } from '../../../common/stack_traces';
+import {
+  StackTracesDisplayOption,
+  TopNType,
+} from '@kbn/profiling-data-access-plugin/common/stack_traces';
 import { getTracesViewRouteParams } from './utils';
 
 describe('stack traces view utils', () => {

x-pack/plugins/profiling/public/views/stack_traces_view/utils.ts

@@ -6,7 +6,10 @@
  */
 
 import { TypeOf } from '@kbn/typed-react-router-config';
-import { getFieldNameForTopNType, TopNType } from '../../../common/stack_traces';
+import {
+  getFieldNameForTopNType,
+  TopNType,
+} from '@kbn/profiling-data-access-plugin/common/stack_traces';
 import { ProfilingRoutes } from '../../routing';
 
 export function getTracesViewRouteParams({

x-pack/plugins/profiling/public/views/storage_explorer/summary.tsx

@@ -9,7 +9,10 @@ import { EuiFlexGroup, EuiFlexItem, EuiLink, EuiPanel, EuiStat, EuiText } from '
 import { i18n } from '@kbn/i18n';
 import { asDynamicBytes } from '@kbn/observability-plugin/common';
 import React from 'react';
-import { StackTracesDisplayOption, TopNType } from '../../../common/stack_traces';
+import {
+  StackTracesDisplayOption,
+  TopNType,
+} from '@kbn/profiling-data-access-plugin/common/stack_traces';
 import { StorageExplorerSummaryAPIResponse } from '../../../common/storage_explorer';
 import { useProfilingDependencies } from '../../components/contexts/profiling_dependencies/use_profiling_dependencies';
 import { LabelWithHint } from '../../components/label_with_hint';

x-pack/plugins/profiling/server/routes/flamechart.ts

@@ -6,21 +6,17 @@
  */
 
 import { schema } from '@kbn/config-schema';
-
 import { RouteRegisterParameters } from '.';
 import { getRoutePaths } from '../../common';
-import { createCalleeTree } from '../../common/callee';
 import { handleRouteHandlerError } from '../utils/handle_route_error_handler';
-import { createBaseFlameGraph } from '../../common/flamegraph';
-import { withProfilingSpan } from '../utils/with_profiling_span';
 import { getClient } from './compat';
-import { createCommonFilter } from './query';
-import { searchStackTraces } from './search_stacktraces';
 
 export function registerFlameChartSearchRoute({
   router,
   logger,
-  services: { createProfilingEsClient },
+  dependencies: {
+    start: { profilingDataAccess },
+  },
 }: RouteRegisterParameters) {
   const paths = getRoutePaths();
   router.get(
@@ -37,39 +33,15 @@ export function registerFlameChartSearchRoute({
     },
     async (context, request, response) => {
       const { timeFrom, timeTo, kuery } = request.query;
-      const targetSampleSize = 20000; // minimum number of samples to get statistically sound results
 
       try {
         const esClient = await getClient(context);
-        const profilingElasticsearchClient = createProfilingEsClient({ request, esClient });
-        const filter = createCommonFilter({
-          timeFrom,
-          timeTo,
+        const flamegraph = await profilingDataAccess.services.fetchFlamechartData({
+          esClient,
+          rangeFrom: timeFrom,
+          rangeTo: timeTo,
           kuery,
         });
-        const totalSeconds = timeTo - timeFrom;
-
-        const { events, stackTraces, executables, stackFrames, totalFrames, samplingRate } =
-          await searchStackTraces({
-            client: profilingElasticsearchClient,
-            filter,
-            sampleSize: targetSampleSize,
-          });
-
-        const flamegraph = await withProfilingSpan('create_flamegraph', async () => {
-          const tree = createCalleeTree(
-            events,
-            stackTraces,
-            stackFrames,
-            executables,
-            totalFrames,
-            samplingRate
-          );
-
-          const fg = createBaseFlameGraph(tree, samplingRate, totalSeconds);
-
-          return fg;
-        });
 
         return response.ok({ body: flamegraph });
       } catch (error) {

x-pack/plugins/profiling/server/routes/query.ts

@@ -7,7 +7,7 @@
 
 import { QueryDslBoolQuery } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { kqlQuery } from '@kbn/observability-plugin/server';
-import { ProfilingESField } from '../../common/elasticsearch';
+import { ProfilingESField } from '@kbn/profiling-data-access-plugin/common/elasticsearch';
 
 export interface ProjectTimeQuery {
   bool: QueryDslBoolQuery;

x-pack/plugins/profiling/server/routes/search_stacktraces.ts

@@ -4,8 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
-import { decodeStackTraceResponse } from '../../common/stack_traces';
+import { decodeStackTraceResponse } from '@kbn/profiling-data-access-plugin/common/stack_traces';
 import { ProfilingESClient } from '../utils/create_profiling_es_client';
 import { ProjectTimeQuery } from './query';
 

x-pack/plugins/profiling/server/routes/stacktrace.test.ts

@@ -1,88 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { createStackFrameID, StackTrace } from '../../common/profiling';
-import { runLengthEncode } from '../../common/run_length_encoding';
-import { decodeStackTrace, EncodedStackTrace } from './stacktrace';
-
-enum fileID {
-  A = 'aQpJmTLWydNvOapSFZOwKg',
-  B = 'hz_u-HGyrN6qeIk6UIJeCA',
-  C = 'AJ8qrcXSoJbl_haPhlc4og',
-  D = 'lHZiv7a58px6Gumcpo-6yA',
-  E = 'fkbxUTZgljnk71ZMnqJnyA',
-  F = 'gnEsgxvvEODj6iFYMQWYlA',
-}
-
-enum addressOrLine {
-  A = 515512,
-  B = 26278522,
-  C = 6712518,
-  D = 105806025,
-  E = 111,
-  F = 106182663,
-  G = 100965370,
-}
-
-const frameID: Record<string, string> = {
-  A: createStackFrameID(fileID.A, addressOrLine.A),
-  B: createStackFrameID(fileID.B, addressOrLine.B),
-  C: createStackFrameID(fileID.C, addressOrLine.C),
-  D: createStackFrameID(fileID.D, addressOrLine.D),
-  E: createStackFrameID(fileID.E, addressOrLine.E),
-  F: createStackFrameID(fileID.F, addressOrLine.F),
-  G: createStackFrameID(fileID.F, addressOrLine.G),
-};
-
-const frameTypeA = [0, 0, 0];
-const frameTypeB = [8, 8, 8, 8];
-
-describe('Stack trace operations', () => {
-  test('decodeStackTrace', () => {
-    const tests: Array<{
-      original: EncodedStackTrace;
-      expected: StackTrace;
-    }> = [
-      {
-        original: {
-          Stacktrace: {
-            frame: {
-              ids: frameID.A + frameID.B + frameID.C,
-              types: runLengthEncode(frameTypeA).toString('base64url'),
-            },
-          },
-        } as EncodedStackTrace,
-        expected: {
-          FrameIDs: [frameID.A, frameID.B, frameID.C],
-          FileIDs: [fileID.A, fileID.B, fileID.C],
-          AddressOrLines: [addressOrLine.A, addressOrLine.B, addressOrLine.C],
-          Types: frameTypeA,
-        } as StackTrace,
-      },
-      {
-        original: {
-          Stacktrace: {
-            frame: {
-              ids: frameID.D + frameID.E + frameID.F + frameID.G,
-              types: runLengthEncode(frameTypeB).toString('base64url'),
-            },
-          },
-        } as EncodedStackTrace,
-        expected: {
-          FrameIDs: [frameID.D, frameID.E, frameID.F, frameID.G],
-          FileIDs: [fileID.D, fileID.E, fileID.F, fileID.F],
-          AddressOrLines: [addressOrLine.D, addressOrLine.E, addressOrLine.F, addressOrLine.G],
-          Types: frameTypeB,
-        } as StackTrace,
-      },
-    ];
-
-    for (const t of tests) {
-      expect(decodeStackTrace(t.original)).toEqual(t.expected);
-    }
-  });
-});

x-pack/plugins/profiling/server/routes/stacktrace.ts

@@ -1,72 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { DedotObject, ProfilingESField } from '../../common/elasticsearch';
-import {
-  getAddressFromStackFrameID,
-  getFileIDFromStackFrameID,
-  StackTrace,
-} from '../../common/profiling';
-import { runLengthDecodeBase64Url } from '../../common/run_length_encoding';
-
-const BASE64_FRAME_ID_LENGTH = 32;
-
-export type EncodedStackTrace = DedotObject<{
-  // This field is a base64-encoded byte string. The string represents a
-  // serialized list of frame IDs in which the order of frames are
-  // reversed to allow for prefix compression (leaf frame last). Each
-  // frame ID is composed of two concatenated values: a 16-byte file ID
-  // and an 8-byte address or line number (depending on the context of
-  // the downstream reader).
-  //
-  //         Frame ID #1               Frame ID #2
-  // +----------------+--------+----------------+--------+----
-  // |     File ID    |  Addr  |     File ID    |  Addr  |
-  // +----------------+--------+----------------+--------+----
-  [ProfilingESField.StacktraceFrameIDs]: string;
-
-  // This field is a run-length encoding of a list of uint8s. The order is
-  // reversed from the original input.
-  [ProfilingESField.StacktraceFrameTypes]: string;
-}>;
-
-// decodeStackTrace unpacks an encoded stack trace from Elasticsearch
-export function decodeStackTrace(input: EncodedStackTrace): StackTrace {
-  const inputFrameIDs = input.Stacktrace.frame.ids;
-  const inputFrameTypes = input.Stacktrace.frame.types;
-  const countsFrameIDs = inputFrameIDs.length / BASE64_FRAME_ID_LENGTH;
-
-  const fileIDs: string[] = new Array(countsFrameIDs);
-  const frameIDs: string[] = new Array(countsFrameIDs);
-  const addressOrLines: number[] = new Array(countsFrameIDs);
-
-  // Step 1: Convert the base64-encoded frameID list into two separate
-  // lists (frame IDs and file IDs), both of which are also base64-encoded.
-  //
-  // To get the frame ID, we grab the next 32 bytes.
-  //
-  // To get the file ID, we grab the first 22 bytes of the frame ID.
-  // However, since the file ID is base64-encoded using 21.33 bytes
-  // (16 * 4 / 3), then the 22 bytes have an extra 4 bits from the
-  // address (see diagram in definition of EncodedStackTrace).
-  for (let i = 0, pos = 0; i < countsFrameIDs; i++, pos += BASE64_FRAME_ID_LENGTH) {
-    const frameID = inputFrameIDs.slice(pos, pos + BASE64_FRAME_ID_LENGTH);
-    frameIDs[i] = frameID;
-    fileIDs[i] = getFileIDFromStackFrameID(frameID);
-    addressOrLines[i] = getAddressFromStackFrameID(frameID);
-  }
-
-  // Step 2: Convert the run-length byte encoding into a list of uint8s.
-  const typeIDs = runLengthDecodeBase64Url(inputFrameTypes, inputFrameTypes.length, countsFrameIDs);
-
-  return {
-    AddressOrLines: addressOrLines,
-    FileIDs: fileIDs,
-    FrameIDs: frameIDs,
-    Types: typeIDs,
-  } as StackTrace;
-}

x-pack/plugins/profiling/server/routes/storage_explorer/get_host_breakdown_size_timeseries.ts

@@ -6,7 +6,7 @@
  */
 
 import { kqlQuery, termQuery } from '@kbn/observability-plugin/server';
-import { ProfilingESField } from '../../../common/elasticsearch';
+import { ProfilingESField } from '@kbn/profiling-data-access-plugin/common/elasticsearch';
 import { computeBucketWidthFromTimeRangeAndBucketCount } from '../../../common/histogram';
 import {
   IndexLifecyclePhaseSelectOption,

x-pack/plugins/profiling/server/routes/storage_explorer/get_host_details.ts

@@ -6,7 +6,7 @@
  */
 
 import { kqlQuery, termQuery } from '@kbn/observability-plugin/server';
-import { ProfilingESField } from '../../../common/elasticsearch';
+import { ProfilingESField } from '@kbn/profiling-data-access-plugin/common/elasticsearch';
 import {
   IndexLifecyclePhaseSelectOption,
   indexLifeCyclePhaseToDataTier,

x-pack/plugins/profiling/server/routes/storage_explorer/get_profiling_hosts_details_by_id.ts

@@ -6,7 +6,7 @@
  */
 import { kqlQuery } from '@kbn/observability-plugin/server';
 import { keyBy } from 'lodash';
-import { ProfilingESField } from '../../../common/elasticsearch';
+import { ProfilingESField } from '@kbn/profiling-data-access-plugin/common/elasticsearch';
 import { ProfilingESClient } from '../../utils/create_profiling_es_client';
 
 interface HostDetails {

x-pack/plugins/profiling/server/routes/topn.test.ts

@@ -8,7 +8,7 @@
 import { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types';
 import { coreMock } from '@kbn/core/server/mocks';
 import { loggerMock } from '@kbn/logging-mocks';
-import { ProfilingESField } from '../../common/elasticsearch';
+import { ProfilingESField } from '@kbn/profiling-data-access-plugin/common/elasticsearch';
 import { ProfilingESClient } from '../utils/create_profiling_es_client';
 import { topNElasticSearchQuery } from './topn';
 

x-pack/plugins/profiling/server/routes/topn.ts

@@ -7,12 +7,15 @@
 
 import { schema } from '@kbn/config-schema';
 import type { Logger } from '@kbn/core/server';
-import { RouteRegisterParameters } from '.';
+import { ProfilingESField } from '@kbn/profiling-data-access-plugin/common/elasticsearch';
+import { groupStackFrameMetadataByStackTrace } from '@kbn/profiling-data-access-plugin/common/profiling';
+import {
+  getFieldNameForTopNType,
+  TopNType,
+} from '@kbn/profiling-data-access-plugin/common/stack_traces';
 import { getRoutePaths, INDEX_EVENTS } from '../../common';
-import { ProfilingESField } from '../../common/elasticsearch';
+import { RouteRegisterParameters } from '.';
 import { computeBucketWidthFromTimeRangeAndBucketCount } from '../../common/histogram';
-import { groupStackFrameMetadataByStackTrace } from '../../common/profiling';
-import { getFieldNameForTopNType, TopNType } from '../../common/stack_traces';
 import { createTopNSamples, getTopNAggregationRequest, TopNResponse } from '../../common/topn';
 import { handleRouteHandlerError } from '../utils/handle_route_error_handler';
 import { ProfilingESClient } from '../utils/create_profiling_es_client';

x-pack/plugins/profiling/server/types.ts

@@ -12,6 +12,10 @@ import { SpacesPluginStart, SpacesPluginSetup } from '@kbn/spaces-plugin/server'
 import { CloudSetup, CloudStart } from '@kbn/cloud-plugin/server';
 import { FleetSetupContract, FleetStartContract } from '@kbn/fleet-plugin/server';
 import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
+import {
+  ProfilingDataAccessPluginSetup,
+  ProfilingDataAccessPluginStart,
+} from '@kbn/profiling-data-access-plugin/server';
 
 export interface ProfilingPluginSetupDeps {
   observability: ObservabilityPluginSetup;
@@ -20,6 +24,7 @@ export interface ProfilingPluginSetupDeps {
   fleet: FleetSetupContract;
   spaces?: SpacesPluginSetup;
   usageCollection?: UsageCollectionSetup;
+  profilingDataAccess: ProfilingDataAccessPluginSetup;
 }
 
 export interface ProfilingPluginStartDeps {
@@ -28,6 +33,7 @@ export interface ProfilingPluginStartDeps {
   cloud: CloudStart;
   fleet: FleetStartContract;
   spaces?: SpacesPluginStart;
+  profilingDataAccess: ProfilingDataAccessPluginStart;
 }
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface

x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts

@@ -10,8 +10,11 @@ import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import type { KibanaRequest } from '@kbn/core/server';
 import { unwrapEsResponse } from '@kbn/observability-plugin/server';
 import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import {
+  ProfilingStatusResponse,
+  StackTraceResponse,
+} from '@kbn/profiling-data-access-plugin/common/stack_traces';
 import { withProfilingSpan } from './with_profiling_span';
-import { ProfilingStatusResponse, StackTraceResponse } from '../../common/stack_traces';
 
 export function cancelEsRequestOnAbort<T extends Promise<any>>(
   promise: T,

x-pack/plugins/profiling/tsconfig.json

@@ -47,7 +47,8 @@
     "@kbn/licensing-plugin",
     "@kbn/utility-types",
     "@kbn/usage-collection-plugin",
-    "@kbn/observability-ai-assistant-plugin"
+    "@kbn/observability-ai-assistant-plugin",
+    "@kbn/profiling-data-access-plugin"
     // add references to other TypeScript projects the plugin depends on
 
     // requiredPlugins from ./kibana.json

x-pack/plugins/profiling_data_access/.i18nrc.json

@@ -0,0 +1,7 @@
+{
+  "prefix": "profiling",
+  "paths": {
+    "profiling": "."
+  },
+  "translations": []
+}

x-pack/plugins/profiling_data_access/common/profiling.test.ts

@@ -6,26 +6,15 @@
  */
 
 import {
-  createStackFrameID,
   createStackFrameMetadata,
   FrameSymbolStatus,
   FrameType,
-  getAddressFromStackFrameID,
   getCalleeFunction,
   getCalleeSource,
-  getFileIDFromStackFrameID,
   getFrameSymbolStatus,
   getLanguageType,
 } from './profiling';
 
-describe('Stack frame operations', () => {
-  test('decode stack frame ID', () => {
-    const frameID = createStackFrameID('ABCDEFGHIJKLMNOPQRSTUw', 123456789);
-    expect(getAddressFromStackFrameID(frameID)).toEqual(123456789);
-    expect(getFileIDFromStackFrameID(frameID)).toEqual('ABCDEFGHIJKLMNOPQRSTUw');
-  });
-});
-
 describe('Stack frame metadata operations', () => {
   test('metadata has executable and function names', () => {
     const metadata = createStackFrameMetadata({

x-pack/plugins/profiling_data_access/common/profiling.ts

@@ -5,50 +5,10 @@
  * 2.0.
  */
 
-import { charCodeAt, safeBase64Encoder } from './base64';
-
 export type StackTraceID = string;
 export type StackFrameID = string;
 export type FileID = string;
 
-export function createStackFrameID(fileID: FileID, addressOrLine: number): StackFrameID {
-  const buf = Buffer.alloc(24);
-  Buffer.from(fileID, 'base64url').copy(buf);
-  buf.writeBigUInt64BE(BigInt(addressOrLine), 16);
-  return buf.toString('base64url');
-}
-
-/* eslint no-bitwise: ["error", { "allow": ["&"] }] */
-export function getFileIDFromStackFrameID(frameID: StackFrameID): FileID {
-  return frameID.slice(0, 21) + safeBase64Encoder[frameID.charCodeAt(21) & 0x30];
-}
-
-/* eslint no-bitwise: ["error", { "allow": ["<<=", "&"] }] */
-export function getAddressFromStackFrameID(frameID: StackFrameID): number {
-  let address = charCodeAt(frameID, 21) & 0xf;
-  address <<= 6;
-  address += charCodeAt(frameID, 22);
-  address <<= 6;
-  address += charCodeAt(frameID, 23);
-  address <<= 6;
-  address += charCodeAt(frameID, 24);
-  address <<= 6;
-  address += charCodeAt(frameID, 25);
-  address <<= 6;
-  address += charCodeAt(frameID, 26);
-  address <<= 6;
-  address += charCodeAt(frameID, 27);
-  address <<= 6;
-  address += charCodeAt(frameID, 28);
-  address <<= 6;
-  address += charCodeAt(frameID, 29);
-  address <<= 6;
-  address += charCodeAt(frameID, 30);
-  address <<= 6;
-  address += charCodeAt(frameID, 31);
-  return address;
-}
-
 export enum FrameType {
   Unsymbolized = 0,
   Python,

x-pack/plugins/profiling_data_access/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+const path = require('path');
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: path.resolve(__dirname, '../../..'),
+  roots: ['<rootDir>/x-pack/plugins/profiling_data_access'],
+};

x-pack/plugins/profiling_data_access/kibana.jsonc

@@ -0,0 +1,14 @@
+{
+  "type": "plugin",
+  "id": "@kbn/profiling-data-access-plugin",
+  "owner": "@elastic/profiling-ui",
+  "plugin": {
+    "id": "profilingDataAccess",
+    "server": true,
+    "browser": false,
+    "configPath": ["xpack", "profiling"],
+    "requiredPlugins": ["data"],
+    "optionalPlugins": [],
+    "requiredBundles": []
+  }
+}

x-pack/plugins/profiling_data_access/server/index.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { schema, TypeOf } from '@kbn/config-schema';
+
+import type { PluginInitializerContext } from '@kbn/core/server';
+import { ProfilingDataAccessPlugin } from './plugin';
+import type { ProfilingDataAccessPluginSetup, ProfilingDataAccessPluginStart } from './plugin';
+
+const configSchema = schema.object({
+  elasticsearch: schema.conditional(
+    schema.contextRef('dist'),
+    schema.literal(true),
+    schema.never(),
+    schema.maybe(
+      schema.object({
+        hosts: schema.string(),
+        username: schema.string(),
+        password: schema.string(),
+      })
+    )
+  ),
+});
+
+export type ProfilingConfig = TypeOf<typeof configSchema>;
+
+export type { ProfilingDataAccessPluginSetup, ProfilingDataAccessPluginStart };
+
+export function plugin(initializerContext: PluginInitializerContext) {
+  return new ProfilingDataAccessPlugin(initializerContext);
+}

x-pack/plugins/profiling_data_access/server/plugin.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server';
+import { ProfilingConfig } from '.';
+import { registerServices } from './services/register_services';
+import { createProfilingEsClient } from './utils/create_profiling_es_client';
+
+export type ProfilingDataAccessPluginSetup = ReturnType<ProfilingDataAccessPlugin['setup']>;
+export type ProfilingDataAccessPluginStart = ReturnType<ProfilingDataAccessPlugin['start']>;
+
+export class ProfilingDataAccessPlugin implements Plugin {
+  constructor(private readonly initializerContext: PluginInitializerContext<ProfilingConfig>) {}
+  public setup(core: CoreSetup) {}
+
+  public start(core: CoreStart) {
+    const config = this.initializerContext.config.get();
+
+    const profilingSpecificEsClient = config.elasticsearch
+      ? core.elasticsearch.createClient('profiling', {
+          hosts: [config.elasticsearch.hosts],
+          username: config.elasticsearch.username,
+          password: config.elasticsearch.password,
+        })
+      : undefined;
+
+    const services = registerServices({
+      createProfilingEsClient: ({ esClient: defaultEsClient, useDefaultAuth = false }) => {
+        const esClient =
+          profilingSpecificEsClient && !useDefaultAuth
+            ? profilingSpecificEsClient.asInternalUser
+            : defaultEsClient;
+
+        return createProfilingEsClient({ esClient });
+      },
+    });
+
+    // called after all plugins are set up
+    return {
+      services,
+    };
+  }
+}

x-pack/plugins/profiling_data_access/server/services/fetch_flamechart/index.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+import { RegisterServicesParams } from '../register_services';
+import { withProfilingSpan } from '../../utils/with_profiling_span';
+import { searchStackTraces } from '../search_stack_traces';
+import { createCalleeTree } from '../../../common/callee';
+import { createBaseFlameGraph } from '../../../common/flamegraph';
+
+interface FetchFlamechartParams {
+  esClient: ElasticsearchClient;
+  rangeFrom: number;
+  rangeTo: number;
+  kuery: string;
+}
+
+export function createFetchFlamechart({ createProfilingEsClient }: RegisterServicesParams) {
+  return async ({ esClient, rangeFrom, rangeTo, kuery }: FetchFlamechartParams) => {
+    const profilingEsClient = createProfilingEsClient({ esClient });
+    const targetSampleSize = 20000; // minimum number of samples to get statistically sound results
+
+    const totalSeconds = rangeTo - rangeFrom;
+
+    const { events, stackTraces, executables, stackFrames, totalFrames, samplingRate } =
+      await searchStackTraces({
+        client: profilingEsClient,
+        rangeFrom,
+        rangeTo,
+        kuery,
+        sampleSize: targetSampleSize,
+      });
+
+    const flamegraph = await withProfilingSpan('create_flamegraph', async () => {
+      const tree = createCalleeTree(
+        events,
+        stackTraces,
+        stackFrames,
+        executables,
+        totalFrames,
+        samplingRate
+      );
+
+      const fg = createBaseFlameGraph(tree, samplingRate, totalSeconds);
+
+      return fg;
+    });
+
+    return flamegraph;
+  };
+}

x-pack/plugins/profiling_data_access/server/services/register_services.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+import { createFetchFlamechart } from './fetch_flamechart';
+import { ProfilingESClient } from '../utils/create_profiling_es_client';
+
+export interface RegisterServicesParams {
+  createProfilingEsClient: (params: {
+    esClient: ElasticsearchClient;
+    useDefaultAuth?: boolean;
+  }) => ProfilingESClient;
+}
+
+export function registerServices(params: RegisterServicesParams) {
+  return { fetchFlamechartData: createFetchFlamechart(params) };
+}

x-pack/plugins/profiling_data_access/server/services/search_stack_traces/index.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
+import { decodeStackTraceResponse } from '../../../common/stack_traces';
+import { ProfilingESClient } from '../../utils/create_profiling_es_client';
+
+export async function searchStackTraces({
+  client,
+  sampleSize,
+  rangeFrom,
+  rangeTo,
+  kuery,
+}: {
+  client: ProfilingESClient;
+  sampleSize: number;
+  rangeFrom: number;
+  rangeTo: number;
+  kuery: string;
+}) {
+  const response = await client.profilingStacktraces({
+    query: {
+      bool: {
+        filter: [
+          ...kqlQuery(kuery),
+          {
+            range: {
+              ['@timestamp']: {
+                gte: String(rangeFrom),
+                lt: String(rangeTo),
+                format: 'epoch_second',
+                boost: 1.0,
+              },
+            },
+          },
+        ],
+      },
+    },
+    sampleSize,
+  });
+
+  return decodeStackTraceResponse(response);
+}
+
+function kqlQuery(kql?: string): estypes.QueryDslQueryContainer[] {
+  if (!kql) {
+    return [];
+  }
+
+  const ast = fromKueryExpression(kql);
+  return [toElasticsearchQuery(ast)];
+}

x-pack/plugins/profiling_data_access/server/utils/create_profiling_es_client.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { ElasticsearchClient } from '@kbn/core/server';
+import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
+import { unwrapEsResponse } from '@kbn/observability-plugin/server';
+import { ProfilingStatusResponse, StackTraceResponse } from '../../common/stack_traces';
+import { withProfilingSpan } from './with_profiling_span';
+
+export interface ProfilingESClient {
+  search<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
+    operationName: string,
+    searchRequest: TSearchRequest
+  ): Promise<InferSearchResponseOf<TDocument, TSearchRequest>>;
+  profilingStacktraces({}: {
+    query: QueryDslQueryContainer;
+    sampleSize: number;
+  }): Promise<StackTraceResponse>;
+  profilingStatus(): Promise<ProfilingStatusResponse>;
+  getEsClient(): ElasticsearchClient;
+}
+
+export function createProfilingEsClient({
+  esClient,
+}: {
+  esClient: ElasticsearchClient;
+}): ProfilingESClient {
+  return {
+    search<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
+      operationName: string,
+      searchRequest: TSearchRequest
+    ): Promise<InferSearchResponseOf<TDocument, TSearchRequest>> {
+      const controller = new AbortController();
+
+      const promise = withProfilingSpan(operationName, () => {
+        return esClient.search(searchRequest, {
+          signal: controller.signal,
+          meta: true,
+        }) as unknown as Promise<{
+          body: InferSearchResponseOf<TDocument, TSearchRequest>;
+        }>;
+      });
+
+      return unwrapEsResponse(promise);
+    },
+    profilingStacktraces({ query, sampleSize }) {
+      const controller = new AbortController();
+      const promise = withProfilingSpan('_profiling/stacktraces', () => {
+        return esClient.transport.request(
+          {
+            method: 'POST',
+            path: encodeURI('/_profiling/stacktraces'),
+            body: {
+              query,
+              sample_size: sampleSize,
+            },
+          },
+          {
+            signal: controller.signal,
+            meta: true,
+          }
+        );
+      });
+
+      return unwrapEsResponse(promise) as Promise<StackTraceResponse>;
+    },
+    profilingStatus() {
+      const controller = new AbortController();
+
+      const promise = withProfilingSpan('_profiling/status', () => {
+        return esClient.transport.request(
+          {
+            method: 'GET',
+            path: encodeURI('/_profiling/status'),
+          },
+          {
+            signal: controller.signal,
+            meta: true,
+          }
+        );
+      });
+
+      return unwrapEsResponse(promise) as Promise<ProfilingStatusResponse>;
+    },
+    getEsClient() {
+      return esClient;
+    },
+  };
+}

x-pack/plugins/profiling_data_access/server/utils/with_profiling_span.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { withSpan, SpanOptions, parseSpanOptions } from '@kbn/apm-utils';
+
+export function withProfilingSpan<T>(
+  optionsOrName: SpanOptions | string,
+  cb: () => Promise<T>
+): Promise<T> {
+  const options = parseSpanOptions(optionsOrName);
+
+  const optionsWithDefaults = {
+    ...(options.intercept ? {} : { type: 'plugin:profiling' }),
+    ...options,
+    labels: {
+      plugin: 'profiling',
+      ...options.labels,
+    },
+  };
+
+  return withSpan(optionsWithDefaults, cb);
+}

x-pack/plugins/profiling_data_access/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types"
+  },
+  "include": [
+    "server/**/*",
+    "common/**/*.ts",
+    "common/**/*.json",
+    "jest.config.js"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/config-schema",
+    "@kbn/core",
+    "@kbn/es-query",
+    "@kbn/es-types",
+    "@kbn/observability-plugin",
+    "@kbn/apm-utils"
+  ]
+}

yarn.lock

@@ -5093,6 +5093,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/profiling-data-access-plugin@link:x-pack/plugins/profiling_data_access":
+  version "0.0.0"
+  uid ""
+
 "@kbn/profiling-plugin@link:x-pack/plugins/profiling":
   version "0.0.0"
   uid ""