performance tests journey based configs (#132119)

* restructure performance tests for per journey config

* add cleanup hook to performance service for browser instance cleanup

* only enable APM in performance configs when TEST_PERFORMANCE_PHASE is set

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Spencer <spencer@elastic.co>

.buildkite/ftr_configs.yml

@@ -246,3 +246,9 @@ enabled:
   - x-pack/test/ui_capabilities/spaces_only/config.ts
   - x-pack/test/upgrade_assistant_integration/config.js
   - x-pack/test/usage_collection/config.ts
+  - x-pack/test/performance/journeys/ecommerce_dashboard/config.ts
+  - x-pack/test/performance/journeys/flight_dashboard/config.ts
+  - x-pack/test/performance/journeys/login/config.ts
+  - x-pack/test/performance/journeys/many_fields_discover/config.ts
+  - x-pack/test/performance/journeys/promotion_tracking_dashboard/config.ts
+  - x-pack/test/performance/journeys/web_logs_dashboard/config.ts

.buildkite/scripts/steps/functional/performance_playwright.sh

@@ -24,24 +24,20 @@ for i in "${journeys[@]}"; do
     echo "JOURNEY[${i}] is running"
 
     export TEST_PERFORMANCE_PHASE=WARMUP
-    export ELASTIC_APM_ACTIVE=false
     export JOURNEY_NAME="${i}"
 
     checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: WARMUP)" \
       node scripts/functional_tests \
-      --config x-pack/test/performance/config.playwright.ts \
-      --include "x-pack/test/performance/tests/playwright/${i}.ts" \
+      --config "x-pack/test/performance/journeys/${i}/config.ts" \
       --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
       --debug \
       --bail
 
     export TEST_PERFORMANCE_PHASE=TEST
-    export ELASTIC_APM_ACTIVE=true
 
     checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: TEST)" \
       node scripts/functional_tests \
-      --config x-pack/test/performance/config.playwright.ts \
-      --include "x-pack/test/performance/tests/playwright/${i}.ts" \
+      --config "x-pack/test/performance/journeys/${i}/config.ts" \
       --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
       --debug \
       --bail

.eslintrc.js

@@ -657,6 +657,7 @@ module.exports = {
         'x-pack/test/*/*config.*ts',
         'x-pack/test/saved_object_api_integration/*/apis/**/*',
         'x-pack/test/ui_capabilities/*/tests/**/*',
+        'x-pack/test/performance/**/*.ts',
       ],
       rules: {
         'import/no-default-export': 'off',

x-pack/test/performance/journeys/base.config.ts

@@ -7,17 +7,15 @@
 import uuid from 'uuid';
 import { FtrConfigProviderContext } from '@kbn/test';
 
-import { services } from './services';
-import { pageObjects } from './page_objects';
+import { services } from '../services';
+import { pageObjects } from '../page_objects';
 
 // These "secret" values are intentionally written in the source. We would make the APM server accept anonymous traffic if we could
 const APM_SERVER_URL = 'https://kibana-ops-e2e-perf.apm.us-central1.gcp.cloud.es.io:443';
 const APM_PUBLIC_TOKEN = 'CTs9y3cvcfq13bQqsB';
 
 export default async function ({ readConfigFile, log }: FtrConfigProviderContext) {
-  const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
-
-  const testFiles = [require.resolve('./tests/playwright')];
+  const functionalConfig = await readConfigFile(require.resolve('../../functional/config.base.js'));
 
   const testBuildId = process.env.BUILDKITE_BUILD_ID ?? `local-${uuid()}`;
   const testJobId = process.env.BUILDKITE_JOB_ID ?? `local-${uuid()}`;
@@ -26,7 +24,6 @@ export default async function ({ readConfigFile, log }: FtrConfigProviderContext
   log.info(` 👷‍♀️ BUILD ID ${testBuildId}\n 👷 JOB ID ${testJobId}\n 👷‍♂️ EXECUTION ID:${executionId}`);
 
   return {
-    testFiles,
     services,
     pageObjects,
     servicesRequiredForTestAnalysis: ['performance'],
@@ -41,7 +38,7 @@ export default async function ({ readConfigFile, log }: FtrConfigProviderContext
       ...functionalConfig.get('kbnTestServer'),
       serverArgs: [...functionalConfig.get('kbnTestServer.serverArgs')],
       env: {
-        ELASTIC_APM_ACTIVE: process.env.ELASTIC_APM_ACTIVE,
+        ELASTIC_APM_ACTIVE: process.env.TEST_PERFORMANCE_PHASE ? 'true' : 'false',
         ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: 'false',
         ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development',
         ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '1.0',
@@ -58,15 +55,10 @@ export default async function ({ readConfigFile, log }: FtrConfigProviderContext
         // for a body with larger size, we might need to reconfigure the APM server to increase the limit.
         // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#long-field-max-length
         ELASTIC_APM_LONG_FIELD_MAX_LENGTH: 300_000,
-        ELASTIC_APM_GLOBAL_LABELS: Object.entries({
-          ftrConfig: `x-pack/test/performance/tests/config.playwright`,
-          performancePhase: process.env.TEST_PERFORMANCE_PHASE,
-          journeyName: process.env.JOURNEY_NAME,
+        ELASTIC_APM_GLOBAL_LABELS: {
           testJobId,
           testBuildId,
-        })
-          .filter(([, v]) => !!v)
-          .reduce((acc, [k, v]) => (acc ? `${acc},${k}=${v}` : `${k}=${v}`), ''),
+        },
       },
       // delay shutdown by 15 seconds to ensure that APM can report the data it collects during test execution
       delayShutdown: 15_000,

x-pack/test/performance/journeys/ecommerce_dashboard/config.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { FtrConfigProviderContext } from '@kbn/test';
+import { serializeApmGlobalLabels } from '../../utils';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const performanceConfig = await readConfigFile(require.resolve('../base.config'));
+
+  const testFiles = [require.resolve('./ecommerce_dashboard')];
+
+  const config = {
+    testFiles,
+    ...performanceConfig.getAll(),
+  };
+
+  const apmGlobalLabels = {
+    ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS,
+    ftrConfig: `x-pack/test/performance/tests/journeys/ecommerce_dashboard/config.ts`,
+    performancePhase: process.env.TEST_PERFORMANCE_PHASE,
+    journeyName: 'ecommerce_dashboard',
+  };
+
+  return {
+    ...config,
+    kbnTestServer: {
+      ...config.kbnTestServer,
+      env: {
+        ...config.kbnTestServer.env,
+        ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels),
+      },
+    },
+  };
+}

x-pack/test/performance/journeys/ecommerce_dashboard/ecommerce_dashboard.ts

@@ -8,7 +8,7 @@ import { Page } from 'playwright';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { StepCtx } from '../../services/performance';
 
-export default function ecommerceDashboard({ getService }: FtrProviderContext) {
+export default function ({ getService }: FtrProviderContext) {
   describe('ecommerce_dashboard', () => {
     it('ecommerce_dashboard', async () => {
       const performance = getService('performance');

x-pack/test/performance/journeys/flight_dashboard/config.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { FtrConfigProviderContext } from '@kbn/test';
+import { serializeApmGlobalLabels } from '../../utils';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const performanceConfig = await readConfigFile(require.resolve('../base.config'));
+
+  const testFiles = [require.resolve('./flight_dashboard')];
+
+  const config = {
+    testFiles,
+    ...performanceConfig.getAll(),
+  };
+
+  const apmGlobalLabels = {
+    ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS,
+    ftrConfig: `x-pack/test/performance/tests/journeys/flight_dashboard/config.ts`,
+    performancePhase: process.env.TEST_PERFORMANCE_PHASE,
+    journeyName: 'flight_dashboard',
+  };
+
+  return {
+    ...config,
+    kbnTestServer: {
+      ...config.kbnTestServer,
+      env: {
+        ...config.kbnTestServer.env,
+        ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels),
+      },
+    },
+  };
+}

x-pack/test/performance/journeys/flight_dashboard/flight_dashboard.ts

@@ -7,7 +7,7 @@
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { StepCtx } from '../../services/performance';
 
-export default function flightDashboard({ getService }: FtrProviderContext) {
+export default function ({ getService }: FtrProviderContext) {
   describe('flight_dashboard', () => {
     it('flight_dashboard', async () => {
       const performance = getService('performance');

x-pack/test/performance/journeys/login/config.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { FtrConfigProviderContext } from '@kbn/test';
+import { serializeApmGlobalLabels } from '../../utils';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const performanceConfig = await readConfigFile(require.resolve('../base.config'));
+
+  const testFiles = [require.resolve('./login')];
+
+  const config = {
+    testFiles,
+    ...performanceConfig.getAll(),
+  };
+
+  const apmGlobalLabels = {
+    ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS,
+    ftrConfig: `x-pack/test/performance/tests/journeys/login/config.ts`,
+    performancePhase: process.env.TEST_PERFORMANCE_PHASE,
+    journeyName: 'login',
+  };
+
+  return {
+    ...config,
+    kbnTestServer: {
+      ...config.kbnTestServer,
+      env: {
+        ...config.kbnTestServer.env,
+        ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels),
+      },
+    },
+  };
+}

x-pack/test/performance/journeys/login/login.ts

@@ -7,7 +7,7 @@
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { StepCtx } from '../../services/performance';
 
-export default function ecommerceDashboard({ getService }: FtrProviderContext) {
+export default function ({ getService }: FtrProviderContext) {
   describe('login', () => {
     it('login', async () => {
       const inputDelays = getService('inputDelays');

x-pack/test/performance/journeys/many_fields_discover/config.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { FtrConfigProviderContext } from '@kbn/test';
+import { serializeApmGlobalLabels } from '../../utils';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const performanceConfig = await readConfigFile(require.resolve('../base.config'));
+
+  const testFiles = [require.resolve('./many_fields_discover')];
+
+  const config = {
+    testFiles,
+    ...performanceConfig.getAll(),
+  };
+
+  const apmGlobalLabels = {
+    ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS,
+    ftrConfig: `x-pack/test/performance/tests/journeys/many_fields_discover/config.ts`,
+    performancePhase: process.env.TEST_PERFORMANCE_PHASE,
+    journeyName: 'many_fields_discover',
+  };
+
+  return {
+    ...config,
+    kbnTestServer: {
+      ...config.kbnTestServer,
+      env: {
+        ...config.kbnTestServer.env,
+        ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels),
+      },
+    },
+  };
+}

x-pack/test/performance/journeys/many_fields_discover/many_fields_discover.ts

@@ -7,7 +7,7 @@
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { StepCtx } from '../../services/performance';
 
-export default function manyFieldsDiscover({ getService }: FtrProviderContext) {
+export default function ({ getService }: FtrProviderContext) {
   // FAILING: https://github.com/elastic/kibana/issues/130287
   describe.skip('many_fields_discover', () => {
     const performance = getService('performance');

x-pack/test/performance/journeys/promotion_tracking_dashboard/config.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { FtrConfigProviderContext } from '@kbn/test';
+import { serializeApmGlobalLabels } from '../../utils';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const performanceConfig = await readConfigFile(require.resolve('../base.config'));
+
+  const testFiles = [require.resolve('./promotion_tracking_dashboard')];
+
+  const config = {
+    testFiles,
+    ...performanceConfig.getAll(),
+  };
+
+  const apmGlobalLabels = {
+    ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS,
+    ftrConfig: `x-pack/test/performance/tests/journeys/promotion_tracking_dashboard/config.ts`,
+    performancePhase: process.env.TEST_PERFORMANCE_PHASE,
+    journeyName: 'promotion_tracking_dashboard',
+  };
+
+  return {
+    ...config,
+    kbnTestServer: {
+      ...config.kbnTestServer,
+      env: {
+        ...config.kbnTestServer.env,
+        ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels),
+      },
+    },
+  };
+}

x-pack/test/performance/journeys/promotion_tracking_dashboard/promotion_tracking_dashboard.ts

@@ -7,7 +7,7 @@
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { StepCtx } from '../../services/performance';
 
-export default function promotionTrackingDashboard({ getService }: FtrProviderContext) {
+export default function ({ getService }: FtrProviderContext) {
   describe('promotion_tracking_dashboard', () => {
     const performance = getService('performance');
     const esArchiver = getService('esArchiver');

x-pack/test/performance/journeys/web_logs_dashboard/config.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { FtrConfigProviderContext } from '@kbn/test';
+import { serializeApmGlobalLabels } from '../../utils';
+
+export default async function webLogsDashboard({ readConfigFile, log }: FtrConfigProviderContext) {
+  const performanceConfig = await readConfigFile(require.resolve('../base.config'));
+
+  const testFiles = [require.resolve('./web_logs_dashboard')];
+
+  const config = {
+    testFiles,
+    ...performanceConfig.getAll(),
+  };
+
+  const apmGlobalLabels = {
+    ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS,
+    ftrConfig: `x-pack/test/performance/tests/journeys/web_logs_dashboard/config.ts`,
+    performancePhase: process.env.TEST_PERFORMANCE_PHASE,
+    journeyName: 'web_logs_dashboard',
+  };
+
+  return {
+    ...config,
+    kbnTestServer: {
+      ...config.kbnTestServer,
+      env: {
+        ...config.kbnTestServer.env,
+        ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels),
+      },
+    },
+  };
+}

x-pack/test/performance/journeys/web_logs_dashboard/web_logs_dashboard.ts

@@ -7,7 +7,7 @@
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { StepCtx } from '../../services/performance';
 
-export default function weblogDashboard({ getService }: FtrProviderContext) {
+export default function ({ getService }: FtrProviderContext) {
   describe('weblogs_dashboard', () => {
     it('weblogs_dashboard', async () => {
       const performance = getService('performance');

x-pack/test/performance/services/performance.ts

@@ -37,6 +37,10 @@ export class PerformanceTestingService extends FtrService {
 
   constructor(ctx: FtrProviderContext) {
     super(ctx);
+
+    ctx.getService('lifecycle').cleanup.add(async () => {
+      await this.shutdownBrowser();
+    });
   }
 
   private getKibanaUrl() {
@@ -173,7 +177,7 @@ export class PerformanceTestingService extends FtrService {
     }
   }
 
-  public async shutdownBrowser() {
+  private async shutdownBrowser() {
     if (this.browser) {
       await (await this.getBrowserInstance()).close();
     }

x-pack/test/performance/tests/playwright/index.ts

@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-import { FtrProviderContext } from '../../ftr_provider_context';
-
-export default function ({ loadTestFile, getService }: FtrProviderContext) {
-  const performance = getService('performance');
-
-  describe('Performance tests', () => {
-    loadTestFile(require.resolve('./login'));
-    loadTestFile(require.resolve('./ecommerce_dashboard'));
-    loadTestFile(require.resolve('./flight_dashboard'));
-    loadTestFile(require.resolve('./web_logs_dashboard'));
-    loadTestFile(require.resolve('./promotion_tracking_dashboard'));
-    loadTestFile(require.resolve('./many_fields_discover'));
-
-    after(async () => {
-      await performance.shutdownBrowser();
-    });
-  });
-}

x-pack/test/performance/utils.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export function serializeApmGlobalLabels(obj: any) {
+  return Object.entries(obj)
+    .filter(([, v]) => !!v)
+    .reduce((acc, [k, v]) => (acc ? `${acc},${k}=${v}` : `${k}=${v}`), '');
+}