[perf-testing] send detailed APM data from report generation (#107592) (#108257)

Co-authored-by: spalger <spalger@users.noreply.github.com>
# Conflicts:
#	x-pack/scripts/functional_tests.js

packages/kbn-apm-config-loader/src/config.ts

@@ -122,6 +122,19 @@ export class ApmConfiguration {
       config.transactionSampleRate = parseFloat(process.env.ELASTIC_APM_TRANSACTION_SAMPLE_RATE);
     }
 
+    if (process.env.ELASTIC_APM_SERVER_URL) {
+      config.serverUrl = process.env.ELASTIC_APM_SERVER_URL;
+    }
+
+    if (process.env.ELASTIC_APM_GLOBAL_LABELS) {
+      config.globalLabels = Object.fromEntries(
+        process.env.ELASTIC_APM_GLOBAL_LABELS.split(',').map((p) => {
+          const [key, ...val] = p.split('=');
+          return [key, val.join('=')];
+        })
+      );
+    }
+
     return config;
   }
 

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -200,6 +200,8 @@ export const schema = Joi.object()
               .default(/Kibana is now available/),
           })
           .default(),
+        env: Joi.object().unknown().default(),
+        delayShutdown: Joi.number(),
       })
       .default(),
 

packages/kbn-test/src/functional_tests/lib/run_kibana_server.js

@@ -29,6 +29,7 @@ function extendNodeOptions(installDir) {
 export async function runKibanaServer({ procs, config, options }) {
   const { installDir } = options;
   const runOptions = config.get('kbnTestServer.runOptions');
+  const env = config.get('kbnTestServer.env');
 
   await procs.run('kibana', {
     cmd: getKibanaCmd(installDir),
@@ -36,6 +37,7 @@ export async function runKibanaServer({ procs, config, options }) {
     env: {
       FORCE_COLOR: 1,
       ...process.env,
+      ...env,
       ...extendNodeOptions(installDir),
     },
     cwd: installDir || KIBANA_ROOT,

packages/kbn-test/src/functional_tests/tasks.js

@@ -100,6 +100,12 @@ export async function runTests(options) {
         await runFtr({ configPath, options: opts });
       } finally {
         try {
+          const delay = config.get('kbnTestServer.delayShutdown');
+          if (typeof delay === 'number') {
+            log.info('Delaying shutdown of Kibana for', delay, 'ms');
+            await new Promise((r) => setTimeout(r, delay));
+          }
+
           await procs.stop('kibana');
         } finally {
           if (es) {

vars/workers.groovy

@@ -99,8 +99,10 @@ def base(Map params, Closure closure) {
     withEnv([
       "CI=true",
       "HOME=${env.JENKINS_HOME}",
+      "PR_NUMBER=${env.ghprbPullId ?: ''}",
       "PR_SOURCE_BRANCH=${env.ghprbSourceBranch ?: ''}",
       "PR_TARGET_BRANCH=${env.ghprbTargetBranch ?: ''}",
+      "PR_MERGE_BASE=${checkoutInfo.mergeBase ?: ''}",
       "PR_AUTHOR=${env.ghprbPullAuthorLogin ?: ''}",
       "TEST_BROWSER_HEADLESS=1",
       "GIT_COMMIT=${checkoutInfo.commit}",

x-pack/scripts/functional_tests.js

@@ -90,4 +90,5 @@ require('@kbn/test').runTestsCli([
   require.resolve('../test/usage_collection/config.ts'),
   require.resolve('../test/fleet_functional/config.ts'),
   require.resolve('../test/examples/config.ts'),
+  require.resolve('../test/performance/config.ts'),
 ]);

x-pack/test/performance/config.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+import { services } from './services';
+import { pageObjects } from './page_objects';
+
+// These "secret" values are intentionally written in the source. We would make the APM server accept annonymous traffic if we could
+const APM_SERVER_URL = 'https://2fad4006bf784bb8a54e52f4a5862609.apm.us-west1.gcp.cloud.es.io:443';
+const APM_PUBLIC_TOKEN = 'Q5q5rWQEw6tKeirBpw';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
+
+  return {
+    testFiles: [require.resolve('./tests/index.ts')],
+    services,
+    pageObjects,
+    servers: functionalConfig.get('servers'),
+    esTestCluster: functionalConfig.get('esTestCluster'),
+    apps: functionalConfig.get('apps'),
+    screenshots: functionalConfig.get('screenshots'),
+    junit: {
+      reportName: 'Performance Tests',
+    },
+    kbnTestServer: {
+      ...functionalConfig.get('kbnTestServer'),
+      env: {
+        ELASTIC_APM_ACTIVE: 'true',
+        ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development',
+        ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '1.0',
+        ELASTIC_APM_SERVER_URL: APM_SERVER_URL,
+        ELASTIC_APM_SECRET_TOKEN: APM_PUBLIC_TOKEN,
+        ELASTIC_APM_GLOBAL_LABELS: Object.entries({
+          ftrConfig: `x-pack/test/performance`,
+          jenkinsJobName: process.env.JOB_NAME,
+          jenkinsBuildNumber: process.env.BUILD_NUMBER,
+          prId: process.env.PR_NUMBER,
+          branch: process.env.GIT_BRANCH,
+          commit: process.env.GIT_COMMIT,
+          mergeBase: process.env.PR_MERGE_BASE,
+          targetBranch: process.env.PR_TARGET_BRANCH,
+        })
+          .filter(([, v]) => !!v)
+          .reduce((acc, [k, v]) => (acc ? `${acc},${k}=${v}` : `${k}=${v}`), ''),
+      },
+      // delay shutdown by 15 seconds to ensure that APM can report the data it collects during test execution
+      delayShutdown: 15_000,
+    },
+  };
+}

x-pack/test/performance/es_archives/reporting_dashboard/mappings.json

@@ -0,0 +1,41 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+    },
+    "index": "foo",
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "group": {
+          "ignore_above": 256,
+          "type": "keyword"
+        },
+        "randomInt": {
+          "type": "integer"
+        },
+        "geo": {
+          "properties": {
+            "country_code": {
+              "type": "keyword"
+            },
+            "country_name": {
+              "type": "keyword"
+            },
+            "point": {
+              "type": "geo_point"
+            }
+          }
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}

x-pack/test/performance/ftr_provider_context.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GenericFtrProviderContext, GenericFtrService } from '@kbn/test';
+
+import { pageObjects } from './page_objects';
+import { services } from './services';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;
+export class FtrService extends GenericFtrService<FtrProviderContext> {}

x-pack/test/performance/page_objects.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from '../functional/page_objects';

x-pack/test/performance/services.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from '../functional/services';

x-pack/test/performance/tests/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('performance', function () {
+    this.tags('ciGroup8');
+
+    loadTestFile(require.resolve('./reporting_dashboard'));
+  });
+}

x-pack/test/performance/tests/reporting_dashboard.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function ({ getService, getPageObject }: FtrProviderContext) {
+  const retry = getService('retry');
+  const es = getService('es');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+  const common = getPageObject('common');
+  const dashboard = getPageObject('dashboard');
+  const reporting = getPageObject('reporting');
+
+  describe('reporting dashbaord', () => {
+    before(async () => {
+      await kibanaServer.importExport.load(
+        'x-pack/test/performance/kbn_archives/reporting_dashboard'
+      );
+      await esArchiver.loadIfNeeded('x-pack/test/performance/es_archives/reporting_dashboard');
+    });
+
+    after(async () => {
+      await kibanaServer.importExport.unload(
+        'x-pack/test/performance/kbn_archives/reporting_dashboard'
+      );
+      await esArchiver.unload('x-pack/test/performance/es_archives/reporting_dashboard');
+      await es.deleteByQuery({
+        index: '.reporting-*',
+        refresh: true,
+        body: { query: { match_all: {} } },
+      });
+    });
+
+    it('downloaded PDF has OK status', async function () {
+      this.timeout(180000);
+
+      await common.navigateToApp('dashboards');
+      await retry.waitFor('dashboard landing page', async () => {
+        return await dashboard.onDashboardLandingPage();
+      });
+      await dashboard.loadSavedDashboard('dashboard');
+      await reporting.openPdfReportingPanel();
+      await reporting.clickGenerateReportButton();
+
+      await reporting.getReportURL(60000);
+    });
+  });
+}