[kbn-performance-testing-dataset-extractor] filter out transactions with static resources (#135222)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-performance-testing-dataset-extractor/src/cli.ts

@@ -98,8 +98,10 @@ export async function runExtractor() {
         throw createFlagError('--buildId must be defined');
       }
 
+      const withoutStaticResources = !!flags['without-static-resources'] || false;
+
       return extractor({
-        param: { journeyName, scalabilitySetup, buildId },
+        param: { journeyName, scalabilitySetup, buildId, withoutStaticResources },
         client: { baseURL, username, password },
         log,
       });
@@ -108,12 +110,14 @@ export async function runExtractor() {
       description: `CLI to fetch and normalize APM traces for journey scalability testing`,
       flags: {
         string: ['config', 'buildId', 'es-url', 'es-username', 'es-password'],
+        boolean: ['without-static-resources'],
         help: `
-          --config           path to an FTR config file that sets scalabilitySetup and journeyName (stored as 'labels.journeyName' in APM-based document)
+          --config <config_path>     path to an FTR config file that sets scalabilitySetup and journeyName (stored as 'labels.journeyName' in APM-based document)
-          --buildId          BUILDKITE_JOB_ID or uuid generated locally, stored in APM-based document as label: 'labels.testBuildId'
+          --buildId <buildId>        BUILDKITE_JOB_ID or uuid generated locally, stored in APM-based document as label: 'labels.testBuildId'
-          --es-url           url for Elasticsearch (APM cluster)
+          --es-url <url>             url for Elasticsearch (APM cluster)
-          --es-username      username for Elasticsearch (APM cluster)
+          --es-username <username>   username for Elasticsearch (APM cluster)
-          --es-password      password for Elasticsearch (APM cluster)
+          --es-password <password>   password for Elasticsearch (APM cluster)
+          --without-static-resources filters out traces with url path matching static resources pattern
         `,
       },
     }

packages/kbn-performance-testing-dataset-extractor/src/extractor.ts

@@ -15,12 +15,14 @@ import { SearchHit } from '@elastic/elasticsearch/lib/api/types';
 import { initClient, Document } from './es_client';
 
 const DATE_FORMAT = `YYYY-MM-DD'T'HH:mm:ss.SSS'Z'`;
+const STATIC_RESOURCES_PATTERN = /\.(css|ico|js|json|jpeg|jpg|gif|png|otf|ttf|woff|woff2)$/;
 
 interface CLIParams {
   param: {
     journeyName: string;
     scalabilitySetup: ScalabilitySetup;
     buildId: string;
+    withoutStaticResources: boolean;
   };
   client: {
     baseURL: string;
@@ -61,13 +63,47 @@ const calculateTransactionTimeRage = (hit: SearchHit<Document>) => {
   return { startTime, endTime };
 };
 
+const getTraceItems = (
+  hits: Array<SearchHit<Document>>,
+  withoutStaticResources: boolean,
+  log: ToolingLog
+) => {
+  const data = hits
+    .map((hit) => hit!._source as Document)
+    .map((hit) => {
+      const payload = hit.http.request?.body?.original;
+      return {
+        processor: hit.processor,
+        traceId: hit.trace.id,
+        timestamp: hit['@timestamp'],
+        environment: hit.environment,
+        request: {
+          url: { path: hit.url.path },
+          headers: hit.http.request.headers,
+          method: hit.http.request.method,
+          body: payload ? parsePayload(payload, hit.trace.id, log) : undefined,
+        },
+        response: { statusCode: hit.http.response.status_code },
+        transaction: {
+          id: hit.transaction.id,
+          name: hit.transaction.name,
+          type: hit.transaction.type,
+        },
+      };
+    });
+
+  return withoutStaticResources
+    ? data.filter((item) => !STATIC_RESOURCES_PATTERN.test(item.request.url.path))
+    : data;
+};
+
 export const extractor = async ({ param, client, log }: CLIParams) => {
   const authOptions = {
     node: client.baseURL,
     username: client.username,
     password: client.password,
   };
-  const { journeyName, scalabilitySetup, buildId } = param;
+  const { journeyName, scalabilitySetup, buildId, withoutStaticResources } = param;
   log.info(
     `Searching transactions with 'labels.testBuildId=${buildId}' and 'labels.journeyName=${journeyName}'`
   );
@@ -97,42 +133,18 @@ export const extractor = async ({ param, client, log }: CLIParams) => {
   const source = hits[0]!._source as Document;
   const kibanaVersion = source.service.version;
 
-  const data = hits
-    .map((hit) => hit!._source as Document)
-    .map((hit) => {
-      const payload = hit.http.request?.body?.original;
-      return {
-        processor: hit.processor,
-        traceId: hit.trace.id,
-        timestamp: hit['@timestamp'],
-        environment: hit.environment,
-        request: {
-          url: { path: hit.url.path },
-          headers: hit.http.request.headers,
-          method: hit.http.request.method,
-          body: payload ? parsePayload(payload, hit.trace.id, log) : undefined,
-        },
-        response: { statusCode: hit.http.response.status_code },
-        transaction: {
-          id: hit.transaction.id,
-          name: hit.transaction.name,
-          type: hit.transaction.type,
-        },
-      };
-    });
-
   const output = {
     journeyName,
     kibanaVersion,
     scalabilitySetup,
-    traceItems: data,
+    traceItems: getTraceItems(hits, withoutStaticResources, log),
   };
 
   const outputDir = path.resolve('target/scalability_traces');
   const fileName = `${output.journeyName.replace(/ /g, '')}-${buildId}.json`;
   const filePath = path.resolve(outputDir, fileName);
 
-  log.info(`Found ${hits.length} transactions, output file: ${filePath}`);
+  log.info(`Found ${output.traceItems.length} transactions, output file: ${filePath}`);
   if (!existsSync(outputDir)) {
     await fs.mkdir(outputDir, { recursive: true });
   }