[kbn-performance-testing-dataset-extractor] use FTR config as cli argument (#132676)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-performance-testing-dataset-extractor/BUILD.bazel

@@ -36,8 +36,9 @@ NPM_MODULE_EXTRA_FILES = [
 #    "@npm//name-of-package"
 #    eg. "@npm//lodash"
 RUNTIME_DEPS = [
-  "//packages/kbn-dev-utils",
+  "//packages/kbn-dev-cli-errors",
-  "//packages/kbn-utils",
+  "//packages/kbn-dev-cli-runner",
+  "//packages/kbn-test",
   "//packages/kbn-tooling-log",
   "@npm//@elastic/elasticsearch",
 ]
@@ -52,8 +53,9 @@ RUNTIME_DEPS = [
 #
 #  References to NPM packages work the same as RUNTIME_DEPS
 TYPES_DEPS = [
-  "//packages/kbn-dev-utils:npm_module_types",
+  "//packages/kbn-dev-cli-errors:npm_module_types",
-  "//packages/kbn-utils:npm_module_types",
+  "//packages/kbn-dev-cli-runner:npm_module_types",
+  "//packages/kbn-test:npm_module_types",
   "//packages/kbn-tooling-log:npm_module_types",
   "@npm//@elastic/elasticsearch",
   "@npm//@types/node",

packages/kbn-performance-testing-dataset-extractor/src/cli.ts

@@ -14,7 +14,13 @@
 
 import { run } from '@kbn/dev-cli-runner';
 import { createFlagError } from '@kbn/dev-cli-errors';
-import { extractor } from './extractor';
+import { EsVersion, readConfigFile } from '@kbn/test';
+import path from 'path';
+import { extractor, ScalabilitySetup } from './extractor';
+
+interface Vars {
+  [key: string]: string;
+}
 
 export async function runExtractor() {
   run(
@@ -43,14 +49,47 @@ export async function runExtractor() {
         throw createFlagError('--es-password must be defined');
       }
 
-      const journeyName = flags.journeyName;
+      const configPath = flags.config;
-      if (journeyName && typeof journeyName !== 'string') {
+      if (typeof configPath !== 'string') {
-        throw createFlagError('--journeyName must be a string');
+        throw createFlagError('--config must be a string');
       }
-      if (!journeyName) {
+      const config = await readConfigFile(log, EsVersion.getDefault(), path.resolve(configPath));
-        throw createFlagError('--journeyName must be defined');
+
+      const scalabilitySetup: ScalabilitySetup = config.get('scalabilitySetup');
+
+      if (!scalabilitySetup) {
+        log.error(`'scalabilitySetup' must be defined in config file!`);
+        return;
       }
 
+      const env = config.get(`kbnTestServer.env`);
+      if (
+        typeof env !== 'object' ||
+        typeof env.ELASTIC_APM_GLOBAL_LABELS !== 'string' ||
+        !env.ELASTIC_APM_GLOBAL_LABELS.includes('journeyName=')
+      ) {
+        log.error(
+          `'journeyName' must be defined in config file:
+
+      env: {
+        ...config.kbnTestServer.env,
+        ELASTIC_APM_GLOBAL_LABELS: Object.entries({
+          journeyName: <journey name>,
+        })
+      },`
+        );
+        return;
+      }
+
+      const envVars: Vars = env.ELASTIC_APM_GLOBAL_LABELS.split(',').reduce(
+        (acc: Vars, pair: string) => {
+          const [key, value] = pair.split('=');
+          return { ...acc, [key]: value };
+        },
+        {}
+      );
+      const journeyName = envVars.journeyName;
+
       const buildId = flags.buildId;
       if (buildId && typeof buildId !== 'string') {
         throw createFlagError('--buildId must be a string');
@@ -60,7 +99,7 @@ export async function runExtractor() {
       }
 
       return extractor({
-        param: { journeyName, buildId },
+        param: { journeyName, scalabilitySetup, buildId },
         client: { baseURL, username, password },
         log,
       });
@@ -68,9 +107,9 @@ export async function runExtractor() {
     {
       description: `CLI to fetch and normalize APM traces for journey scalability testing`,
       flags: {
-        string: ['journeyName', 'buildId', 'es-url', 'es-username', 'es-password'],
+        string: ['config', 'buildId', 'es-url', 'es-username', 'es-password'],
         help: `
-          --journeyName      Single user performance journey name, stored in APM-based document as label: 'labels.journeyName'
+          --config           path to an FTR config file that sets scalabilitySetup and journeyName (stored as 'labels.journeyName' in APM-based document)
           --buildId          BUILDKITE_JOB_ID or uuid generated locally, stored in APM-based document as label: 'labels.testBuildId'
           --es-url           url for Elasticsearch (APM cluster)
           --es-username      username for Elasticsearch (APM cluster)

packages/kbn-performance-testing-dataset-extractor/src/extractor.ts

@@ -15,6 +15,7 @@ import { initClient, Document } from './es_client';
 interface CLIParams {
   param: {
     journeyName: string;
+    scalabilitySetup: ScalabilitySetup;
     buildId: string;
   };
   client: {
@@ -25,26 +26,38 @@ interface CLIParams {
   log: ToolingLog;
 }
 
+interface Stage {
+  action: string;
+  minUsersCount?: number;
+  maxUsersCount: number;
+  duration: string;
+}
+
+export interface ScalabilitySetup {
+  warmup: { stages: Stage[] };
+  test: { stages: Stage[] };
+  maxDuration: string;
+}
+
 export const extractor = async ({ param, client, log }: CLIParams) => {
   const authOptions = {
     node: client.baseURL,
     username: client.username,
     password: client.password,
   };
+  const { journeyName, scalabilitySetup, buildId } = param;
+  log.info(
+    `Searching transactions with 'labels.testBuildId=${buildId}' and 'labels.journeyName=${journeyName}'`
+  );
   const esClient = initClient(authOptions);
-  const hits = await esClient.getTransactions(param.buildId, param.journeyName);
+  const hits = await esClient.getTransactions(buildId, journeyName);
   if (!hits || hits.length === 0) {
-    log.warning(`
+    log.warning(`No transactions found. Output file won't be generated.`);
-      No transactions found with 'labels.testBuildId=${param.buildId}' and 'labels.journeyName=${param.journeyName}'
-      \nOutput file won't be generated
-    `);
     return;
   }
 
   const source = hits[0]!._source as Document;
-  const journeyName = source.labels.journeyName || 'Unknown Journey';
   const kibanaVersion = source.service.version;
-  const maxUsersCount = source.labels.maxUsersCount || '0';
 
   const data = hits
     .map((hit) => hit!._source as Document)
@@ -72,12 +85,12 @@ export const extractor = async ({ param, client, log }: CLIParams) => {
   const output = {
     journeyName,
     kibanaVersion,
-    maxUsersCount,
+    scalabilitySetup,
     traceItems: data,
   };
 
   const outputDir = path.resolve('target/scalability_traces');
-  const fileName = `${output.journeyName.replace(/ /g, '')}-${param.buildId}.json`;
+  const fileName = `${output.journeyName.replace(/ /g, '')}-${buildId}.json`;
   const filePath = path.resolve(outputDir, fileName);
 
   log.info(`Found ${hits.length} transactions, output file: ${filePath}`);