[packages] add kbn-performance-testing-dataset-extractor (#131631)

packages/kbn-performance-testing-dataset-extractor/BUILD.bazel

@@ -0,0 +1,122 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
+
+PKG_DIRNAME = "kbn-performance-testing-dataset-extractor"
+PKG_REQUIRE_NAME = "@kbn/performance-testing-dataset-extractor"
+
+SOURCE_FILES = glob(
+  [
+    "src/**/*.ts",
+  ],
+  exclude = [
+    "**/*.test.*",
+  ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+]
+
+# In this array place runtime dependencies, including other packages and NPM packages
+# which must be available for this code to run.
+#
+#  To reference other packages use:
+#    "//repo/relative/path/to/package"
+#    eg. "//packages/kbn-utils"
+#
+#  To reference a NPM package use:
+#    "@npm//name-of-package"
+#    eg. "@npm//lodash"
+RUNTIME_DEPS = [
+  "//packages/kbn-dev-utils",
+  "//packages/kbn-utils",
+  "//packages/kbn-tooling-log",
+  "@npm//@elastic/elasticsearch",
+]
+
+# In this array place dependencies necessary to build the types, which will include the
+# :npm_module_types target of other packages and packages from NPM, including @types/*
+# packages.
+#
+#  To reference the types for another package use:
+#    "//repo/relative/path/to/package:npm_module_types"
+#    eg. "//packages/kbn-utils:npm_module_types"
+#
+#  References to NPM packages work the same as RUNTIME_DEPS
+TYPES_DEPS = [
+  "//packages/kbn-dev-utils:npm_module_types",
+  "//packages/kbn-utils:npm_module_types",
+  "//packages/kbn-tooling-log:npm_module_types",
+  "@npm//@elastic/elasticsearch",
+  "@npm//@types/node",
+  "@npm//@types/jest",
+]
+
+jsts_transpiler(
+  name = "target_node",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+)
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+    "//:tsconfig.base.json",
+    "//:tsconfig.bazel.json",
+  ],
+)
+
+ts_project(
+  name = "tsc_types",
+  args = ['--pretty'],
+  srcs = SRCS,
+  deps = TYPES_DEPS,
+  declaration = True,
+  emit_declaration_only = True,
+  out_dir = "target_types",
+  root_dir = "src",
+  tsconfig = ":tsconfig",
+)
+
+js_library(
+  name = PKG_DIRNAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  deps = RUNTIME_DEPS + [":target_node"],
+  package_name = PKG_REQUIRE_NAME,
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [":" + PKG_DIRNAME],
+)
+
+filegroup(
+  name = "build",
+  srcs = [":npm_module"],
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm_types(
+  name = "npm_module_types",
+  srcs = SRCS,
+  deps = [":tsc_types"],
+  package_name = PKG_REQUIRE_NAME,
+  tsconfig = ":tsconfig",
+  visibility = ["//visibility:public"],
+)
+
+filegroup(
+  name = "build_types",
+  srcs = [":npm_module_types"],
+  visibility = ["//visibility:public"],
+)

packages/kbn-performance-testing-dataset-extractor/README.md

@@ -0,0 +1,14 @@
+# @kbn/performance-testing-dataset-extractor
+
+A library to convert APM traces into JSON format for performance testing.
+
+## Usage
+
+```
+    node scripts/extract_performance_testing_dataset \
+        --journeyName "<_source.labels.journeyName>" \
+        --buildId "<_source.labels.testBuildId>" \
+        --es-url "<ES baseURL>" \
+        --es-username "<ES username>" \
+        --es-password "<ES password>"
+```

packages/kbn-performance-testing-dataset-extractor/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-performance-testing-dataset-extractor'],
+};

packages/kbn-performance-testing-dataset-extractor/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@kbn/performance-testing-dataset-extractor",
+  "description": "A library to convert APM traces into JSON format for performance testing.",
+  "private": true,
+  "version": "1.0.0",
+  "main": "./target_node/index.js",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "kibana": {
+    "devOnly": true
+  }
+}

packages/kbn-performance-testing-dataset-extractor/src/cli.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/** ***********************************************************
+ *
+ *  Run `node scripts/extract_performance_testing_dataset --help` for usage information
+ *
+ *************************************************************/
+
+import { run, createFlagError } from '@kbn/dev-utils';
+import { extractor } from './extractor';
+
+export async function runExtractor() {
+  run(
+    async ({ log, flags }) => {
+      const baseURL = flags['es-url'];
+      if (baseURL && typeof baseURL !== 'string') {
+        throw createFlagError('--es-url must be a string');
+      }
+      if (!baseURL) {
+        throw createFlagError('--es-url must be defined');
+      }
+
+      const username = flags['es-username'];
+      if (username && typeof username !== 'string') {
+        throw createFlagError('--es-username must be a string');
+      }
+      if (!username) {
+        throw createFlagError('--es-username must be defined');
+      }
+
+      const password = flags['es-password'];
+      if (password && typeof password !== 'string') {
+        throw createFlagError('--es-password must be a string');
+      }
+      if (!password) {
+        throw createFlagError('--es-password must be defined');
+      }
+
+      const journeyName = flags.journeyName;
+      if (journeyName && typeof journeyName !== 'string') {
+        throw createFlagError('--journeyName must be a string');
+      }
+      if (!journeyName) {
+        throw createFlagError('--journeyName must be defined');
+      }
+
+      const buildId = flags.buildId;
+      if (buildId && typeof buildId !== 'string') {
+        throw createFlagError('--buildId must be a string');
+      }
+      if (!buildId) {
+        throw createFlagError('--buildId must be defined');
+      }
+
+      return extractor({
+        param: { journeyName, buildId },
+        client: { baseURL, username, password },
+        log,
+      });
+    },
+    {
+      description: `CLI to fetch and normalize APM traces for journey scalability testing`,
+      flags: {
+        string: ['journeyName', 'buildId', 'es-url', 'es-username', 'es-password'],
+        help: `
+          --journeyName      Single user performance journey name, stored in APM-based document as label: 'labels.journeyName'
+          --buildId          BUILDKITE_JOB_ID or uuid generated locally, stored in APM-based document as label: 'labels.testBuildId'
+          --es-url           url for Elasticsearch (APM cluster)
+          --es-username      username for Elasticsearch (APM cluster)
+          --es-password      password for Elasticsearch (APM cluster)
+        `,
+      },
+    }
+  );
+}

packages/kbn-performance-testing-dataset-extractor/src/es_client.ts

@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+
+interface ClientOptions {
+  node: string;
+  username: string;
+  password: string;
+}
+
+interface Labels {
+  journeyName: string;
+  maxUsersCount: string;
+}
+
+interface Request {
+  method: string;
+  headers: string;
+  body?: { original: string };
+}
+
+interface Response {
+  status_code: number;
+}
+
+interface Transaction {
+  id: string;
+  name: string;
+  type: string;
+}
+
+export interface Document {
+  labels: Labels;
+  character: string;
+  quote: string;
+  service: { version: string };
+  processor: string;
+  trace: { id: string };
+  '@timestamp': string;
+  environment: string;
+  url: { path: string };
+  http: {
+    request: Request;
+    response: Response;
+  };
+  transaction: Transaction;
+}
+
+export function initClient(options: ClientOptions) {
+  const client = new Client({
+    node: options.node,
+    auth: {
+      username: options.username,
+      password: options.password,
+    },
+  });
+
+  return {
+    async getTransactions(buildId: string, journeyName: string) {
+      const result = await client.search<Document>({
+        body: {
+          track_total_hits: true,
+          sort: [
+            {
+              '@timestamp': {
+                order: 'desc',
+                unmapped_type: 'boolean',
+              },
+            },
+          ],
+          size: 10000,
+          stored_fields: ['*'],
+          _source: true,
+          query: {
+            bool: {
+              must: [],
+              filter: [
+                {
+                  bool: {
+                    filter: [
+                      {
+                        bool: {
+                          should: [
+                            {
+                              match_phrase: {
+                                'transaction.type': 'request',
+                              },
+                            },
+                          ],
+                          minimum_should_match: 1,
+                        },
+                      },
+                      {
+                        bool: {
+                          should: [
+                            {
+                              match_phrase: {
+                                'processor.event': 'transaction',
+                              },
+                            },
+                          ],
+                          minimum_should_match: 1,
+                        },
+                      },
+                      {
+                        bool: {
+                          should: [
+                            {
+                              match_phrase: {
+                                'labels.testBuildId': buildId,
+                              },
+                            },
+                          ],
+                          minimum_should_match: 1,
+                        },
+                      },
+                      {
+                        bool: {
+                          should: [
+                            {
+                              match_phrase: {
+                                'labels.journeyName': journeyName,
+                              },
+                            },
+                          ],
+                          minimum_should_match: 1,
+                        },
+                      },
+                    ],
+                  },
+                },
+              ],
+              should: [],
+              must_not: [],
+            },
+          },
+        },
+      });
+      return result?.hits?.hits;
+    },
+  };
+}

packages/kbn-performance-testing-dataset-extractor/src/extractor.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import fs from 'fs/promises';
+import { existsSync } from 'fs';
+import path from 'path';
+import { ToolingLog } from '@kbn/tooling-log';
+import { initClient, Document } from './es_client';
+
+interface CLIParams {
+  param: {
+    journeyName: string;
+    buildId: string;
+  };
+  client: {
+    baseURL: string;
+    username: string;
+    password: string;
+  };
+  log: ToolingLog;
+}
+
+export const extractor = async ({ param, client, log }: CLIParams) => {
+  const authOptions = {
+    node: client.baseURL,
+    username: client.username,
+    password: client.password,
+  };
+  const esClient = initClient(authOptions);
+  const hits = await esClient.getTransactions(param.buildId, param.journeyName);
+  if (!hits || hits.length === 0) {
+    log.warning(`
+      No transactions found with 'labels.testBuildId=${param.buildId}' and 'labels.journeyName=${param.journeyName}'
+      \nOutput file won't be generated
+    `);
+    return;
+  }
+
+  const source = hits[0]!._source as Document;
+  const journeyName = source.labels.journeyName || 'Unknown Journey';
+  const kibanaVersion = source.service.version;
+  const maxUsersCount = source.labels.maxUsersCount || '0';
+
+  const data = hits
+    .map((hit) => hit!._source as Document)
+    .map((hit) => {
+      return {
+        processor: hit.processor,
+        traceId: hit.trace.id,
+        timestamp: hit['@timestamp'],
+        environment: hit.environment,
+        request: {
+          url: { path: hit.url.path },
+          headers: hit.http.request.headers,
+          method: hit.http.request.method,
+          body: hit.http.request.body ? JSON.parse(hit.http.request.body.original) : '',
+        },
+        response: { statusCode: hit.http.response.status_code },
+        transaction: {
+          id: hit.transaction.id,
+          name: hit.transaction.name,
+          type: hit.transaction.type,
+        },
+      };
+    });
+
+  const output = {
+    journeyName,
+    kibanaVersion,
+    maxUsersCount,
+    traceItems: data,
+  };
+
+  const outputDir = path.resolve('target/scalability_traces');
+  const fileName = `${output.journeyName.replace(/ /g, '')}-${param.buildId}.json`;
+  const filePath = path.resolve(outputDir, fileName);
+
+  log.info(`Found ${hits.length} transactions, output file: ${filePath}`);
+  if (!existsSync(outputDir)) {
+    await fs.mkdir(outputDir, { recursive: true });
+  }
+  await fs.writeFile(filePath, JSON.stringify(output, null, 2), 'utf8');
+};

packages/kbn-performance-testing-dataset-extractor/src/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { extractor } from './extractor';
+export * from './cli';

packages/kbn-performance-testing-dataset-extractor/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.bazel.json",
+  "compilerOptions": {
+    "declaration": true,
+    "emitDeclarationOnly": true,
+    "outDir": "target_types",
+    "rootDir": "src",
+    "stripInternal": false,
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "src/**/*"
+  ]
+}