[performance] re-enable scalability dataset extraction step on CI (#134719)

* add scalability setup for login and promotion_tracking_dashboard journeys, re-enable step on ci

* update query to get single ftr transaction

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/pipelines/performance/daily.yml

@@ -19,11 +19,11 @@ steps:
     depends_on: build
     key: tests
 
-  # - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
-  #   command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
-  #   agents:
-  #     queue: n2-2
-  #   depends_on: tests
+  - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
+    command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
+    agents:
+      queue: n2-2
+    depends_on: tests
 
   - label: ':chart_with_upwards_trend: Report performance metrics to ci-stats'
     command: .buildkite/scripts/steps/functional/report_performance_metrics.sh

.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh

@@ -13,14 +13,14 @@ GCS_BUCKET="gs://kibana-performance/scalability-tests"
 .buildkite/scripts/bootstrap.sh
 
 echo "--- Extract APM metrics"
-journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover")
+scalabilityJourneys=("login" "promotion_tracking_dashboard")
 
-for i in "${journeys[@]}"; do
+for i in "${scalabilityJourneys[@]}"; do
     JOURNEY_NAME="${i}"
     echo "Looking for JOURNEY=${JOURNEY_NAME} and BUILD_ID=${BUILD_ID} in APM traces"
 
     node scripts/extract_performance_testing_dataset \
-        --journeyName "${JOURNEY_NAME}" \
+        --config "x-pack/test/performance/journeys/${i}/config.ts" \ \
         --buildId "${BUILD_ID}" \
         --es-url "${ES_SERVER_URL}" \
         --es-username "${USER_FROM_VAULT}" \
@@ -39,4 +39,4 @@ echo "${BUILDKITE_COMMIT}" > "${BUILD_ID}/KIBANA_COMMIT_HASH"
 gsutil -m cp -r "${BUILD_ID}" "${GCS_BUCKET}"
 echo "--- Update reference to the latest CI build"
 echo "${BUILD_ID}" > LATEST
-gsutil cp LATEST "${GCS_BUCKET}"
+gsutil cp LATEST "${GCS_BUCKET}"

packages/kbn-performance-testing-dataset-extractor/src/es_client.ts

@@ -114,6 +114,7 @@ export function initClient(options: ClientOptions) {
         { field: 'processor.event', value: 'transaction' },
         { field: 'labels.testBuildId', value: buildId },
         { field: 'labels.journeyName', value: journeyName },
+        { field: 'labels.performancePhase', value: 'TEST' },
       ];
       const queryFilters = filters.map((filter) => addBooleanFilter(filter));
       return await this.getTransactions(queryFilters);
@@ -125,7 +126,7 @@ export function initClient(options: ClientOptions) {
           sort: [
             {
               '@timestamp': {
-                order: 'desc',
+                order: 'asc',
                 unmapped_type: 'boolean',
               },
             },

x-pack/test/performance/journeys/login/config.ts

@@ -14,6 +14,33 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
 
   const config = {
     testFiles,
+    scalabilitySetup: {
+      warmup: {
+        stages: [
+          {
+            action: 'constantConcurrentUsers',
+            maxUsersCount: 10,
+            duration: '30s',
+          },
+          {
+            action: 'rampConcurrentUsers',
+            minUsersCount: 10,
+            maxUsersCount: 50,
+            duration: '2m',
+          },
+        ],
+      },
+      test: {
+        stages: [
+          {
+            action: 'constantConcurrentUsers',
+            maxUsersCount: 50,
+            duration: '5m',
+          },
+        ],
+      },
+      maxDuration: '10m',
+    },
     ...performanceConfig.getAll(),
   };
 

x-pack/test/performance/journeys/promotion_tracking_dashboard/config.ts

@@ -15,6 +15,33 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const config = {
     testFiles,
     ...performanceConfig.getAll(),
+    scalabilitySetup: {
+      warmup: {
+        stages: [
+          {
+            action: 'constantConcurrentUsers',
+            maxUsersCount: 10,
+            duration: '30s',
+          },
+          {
+            action: 'rampConcurrentUsers',
+            minUsersCount: 10,
+            maxUsersCount: 50,
+            duration: '2m',
+          },
+        ],
+      },
+      test: {
+        stages: [
+          {
+            action: 'constantConcurrentUsers',
+            maxUsersCount: 50,
+            duration: '5m',
+          },
+        ],
+      },
+      maxDuration: '10m',
+    },
   };
 
   const apmGlobalLabels = {