Enable APM for ES when running performance journeys (#155195)

Enables APM instrumentation for Elasticsearch when running performance
journeys, for a more complete understanding of where time is being
spent:

<img width="1418" alt="CleanShot 2023-04-18 at 20 30 32@2x"
src="https://user-images.githubusercontent.com/352732/232870884-151aed23-347b-485e-8490-4b56b6beaa33.png">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-journeys/index.ts

@@ -7,6 +7,7 @@
  */
 
 export { JourneyConfig } from './journey/journey_config';
+export { JOURNEY_APM_CONFIG } from './journey/journey_apm_config';
 export type {
   ScalabilityAction,
   ScalabilitySetup,

packages/kbn-journeys/journey/journey_apm_config.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// These "secret" values are intentionally written in the source. We would make the APM server accept anonymous traffic if we could
+const APM_SERVER_URL = 'https://kibana-ops-e2e-perf.apm.us-central1.gcp.cloud.es.io:443';
+const APM_PUBLIC_TOKEN = 'CTs9y3cvcfq13bQqsB';
+
+export const JOURNEY_APM_CONFIG = {
+  serverUrl: APM_SERVER_URL,
+  secretToken: APM_PUBLIC_TOKEN,
+  active: 'true',
+  contextPropagationOnly: 'false',
+  environment: process.env.CI ? 'ci' : 'development',
+  transactionSampleRate: '1.0',
+  // capture request body for both errors and request transactions
+  // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-body
+  captureBody: 'all',
+  // capture request headers
+  // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-headers
+  captureRequestHeaders: true,
+  // request body with bigger size will be trimmed.
+  // 300_000 is the default of the APM server.
+  // for a body with larger size, we might need to reconfigure the APM server to increase the limit.
+  // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#long-field-max-length
+  longFieldMaxLength: 300_000,
+  globalLabels: {
+    performancePhase: process.env.TEST_PERFORMANCE_PHASE,
+    branch: process.env.BUILDKITE_BRANCH,
+    gitRev: process.env.BUILDKITE_COMMIT,
+    ciBuildName: process.env.BUILDKITE_PIPELINE_SLUG,
+  },
+};

packages/kbn-journeys/journey/journey_ftr_config.ts

@@ -15,10 +15,7 @@ import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
 
 import { AnyStep } from './journey';
 import { JourneyConfig } from './journey_config';
-
-// These "secret" values are intentionally written in the source. We would make the APM server accept anonymous traffic if we could
-const APM_SERVER_URL = 'https://kibana-ops-e2e-perf.apm.us-central1.gcp.cloud.es.io:443';
-const APM_PUBLIC_TOKEN = 'CTs9y3cvcfq13bQqsB';
+import { JOURNEY_APM_CONFIG } from './journey_apm_config';
 
 export function makeFtrConfigProvider(
   config: JourneyConfig<any>,
@@ -92,33 +89,22 @@ export function makeFtrConfigProvider(
         ],
 
         env: {
-          ELASTIC_APM_ACTIVE: 'true',
-          ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: 'false',
-          ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development',
-          ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '1.0',
-          ELASTIC_APM_SERVER_URL: APM_SERVER_URL,
-          ELASTIC_APM_SECRET_TOKEN: APM_PUBLIC_TOKEN,
-          // capture request body for both errors and request transactions
-          // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-body
-          ELASTIC_APM_CAPTURE_BODY: 'all',
-          // capture request headers
-          // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-headers
-          ELASTIC_APM_CAPTURE_HEADERS: true,
-          // request body with bigger size will be trimmed.
-          // 300_000 is the default of the APM server.
-          // for a body with larger size, we might need to reconfigure the APM server to increase the limit.
-          // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#long-field-max-length
-          ELASTIC_APM_LONG_FIELD_MAX_LENGTH: 300_000,
+          ELASTIC_APM_ACTIVE: JOURNEY_APM_CONFIG.active,
+          ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: JOURNEY_APM_CONFIG.contextPropagationOnly,
+          ELASTIC_APM_ENVIRONMENT: JOURNEY_APM_CONFIG.environment,
+          ELASTIC_APM_TRANSACTION_SAMPLE_RATE: JOURNEY_APM_CONFIG.transactionSampleRate,
+          ELASTIC_APM_SERVER_URL: JOURNEY_APM_CONFIG.serverUrl,
+          ELASTIC_APM_SECRET_TOKEN: JOURNEY_APM_CONFIG.secretToken,
+          ELASTIC_APM_CAPTURE_BODY: JOURNEY_APM_CONFIG.captureBody,
+          ELASTIC_APM_CAPTURE_HEADERS: JOURNEY_APM_CONFIG.captureRequestHeaders,
+          ELASTIC_APM_LONG_FIELD_MAX_LENGTH: JOURNEY_APM_CONFIG.longFieldMaxLength,
           ELASTIC_APM_GLOBAL_LABELS: Object.entries({
             ...config.getExtraApmLabels(),
             testJobId,
             testBuildId,
             journeyName: config.getName(),
             ftrConfig: config.getRepoRelPath(),
-            performancePhase: process.env.TEST_PERFORMANCE_PHASE,
-            branch: process.env.BUILDKITE_BRANCH,
-            gitRev: process.env.BUILDKITE_COMMIT,
-            ciBuildName: process.env.BUILDKITE_PIPELINE_SLUG,
+            ...JOURNEY_APM_CONFIG.globalLabels,
           })
             .flatMap(([key, value]) => (value == null ? [] : `${key}=${value}`))
             .join(','),

src/dev/performance/run_performance_cli.ts

@@ -11,6 +11,7 @@ import { run } from '@kbn/dev-cli-runner';
 import { REPO_ROOT } from '@kbn/repo-info';
 import fs from 'fs';
 import path from 'path';
+import { JOURNEY_APM_CONFIG } from '@kbn/journeys';
 
 const JOURNEY_BASE_PATH = 'x-pack/performance/journeys';
 
@@ -103,7 +104,25 @@ run(
       process.stdout.write(`--- Starting ES\n`);
       await procRunner.run('es', {
         cmd: 'node',
-        args: ['scripts/es', 'snapshot', '--license=trial'],
+        args: [
+          'scripts/es',
+          'snapshot',
+          '--license=trial',
+          ...(JOURNEY_APM_CONFIG.active
+            ? [
+                '-E',
+                'tracing.apm.enabled=true',
+                '-E',
+                'tracing.apm.agent.transaction_sample_rate=1.0',
+                '-E',
+                `tracing.apm.agent.server_url=${JOURNEY_APM_CONFIG.serverUrl}`,
+                '-E',
+                `tracing.apm.agent.secret_token=${JOURNEY_APM_CONFIG.secretToken}`,
+                '-E',
+                `tracing.apm.agent.environment=${JOURNEY_APM_CONFIG.environment}`,
+              ]
+            : []),
+        ],
         cwd: REPO_ROOT,
         wait: /kbn\/es setup complete/,
       });

src/dev/tsconfig.json

@@ -38,5 +38,6 @@
     "@kbn/repo-file-maps",
     "@kbn/get-repo-files",
     "@kbn/import-locator",
+    "@kbn/journeys",
   ]
 }