Debugging with apm - fixes and tutorial (#127892) (#128256)

* fixes + tutorial

* cors config

* omit secretToken so its not sent to FE
Add config tests

* lint

* empty

* swallow errors when parsing configs

* read config test adjustment

* apm docs review

* new line

* doc review

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
(cherry picked from commit c97bfc8e3a)

Co-authored-by: Liza Katz <liza.katz@elastic.co>

dev_docs/tutorials/debugging.mdx

@@ -61,3 +61,49 @@ logging:
     - name: elasticsearch.query
       level: debug
 ```
+
+## Debugging Kibana with APM
+
+Kibana is integrated with APM's node and RUM agents. 
+To learn more about how APM works and what it reports, refer to the [documentation](https://www.elastic.co/guide/en/apm/guide/current/index.html).
+
+We currently track the following types of transactions from Kibana: 
+
+Frontend (APM RUM): 
+ * `http-request`- tracks all outgoing API requests
+ * `page-load` - tracks the inidial loading time of kibana
+ * `app-change` - tracks application changes
+
+Backend (APM Node): 
+ * `request` - tracks all incoming API requests
+ * `kibana-platform` - tracks server initiation phases (preboot, setup and start)
+ * `task-manager` - tracks the operation of the task manager, including claiming pending tasks and marking them as running
+ * `task-run` - tracks the execution of individual tasks
+
+### Enabling APM on a local environment
+
+In some cases, it is beneficial to enable APM on a local development environment to get an initial undesrtanding of a feature's performance during manual or automatic tests.
+
+ 1. Create a secondary monitoring deployment to collect APM data. The easiest option is to use [Elastic Cloud](https://cloud.elastic.co/deployments) to create a new deployment.
+ 2. Open Kibana, go to `Integrations` and enable the Elastic APM integration.
+ 3. Scroll down and copy the server URL and secret token. You may also find them in your cloud console under APM & Fleet.
+ 4. Create or open `config\kibana.dev.yml` on your local development environment.
+ 5. Add the following settings: 
+    ```
+    elastic.apm.active: true
+    elastic.apm.serverUrl: <serverUrl>
+    elastic.apm.secretToken: <secretToken>
+    ```
+ 6. Once you run kibana and start using it, two new services (kibana, kibana-frontend) should appear under the APM UI on the APM deployment.
+  ![APM UI](./apm_ui.png)
+
+### Enabling APM via environment variables
+
+It is possible to enable APM via environment variables as well.
+They take precedence over any values defined in `kibana.yml` or `kibana.dev.yml`
+
+Set the following environment variables to enable APM: 
+
+ * ELASTIC_APM_ACTIVE
+ * ELASTIC_APM_SERVER_URL
+ * ELASTIC_APM_SECRET_TOKEN

packages/kbn-apm-config-loader/src/config.test.mocks.ts

@@ -17,13 +17,6 @@ export const packageMock = {
 };
 jest.doMock(join(mockedRootDir, 'package.json'), () => packageMock.raw, { virtual: true });
 
-export const devConfigMock = {
-  raw: {} as any,
-};
-jest.doMock(join(mockedRootDir, 'config', 'apm.dev.js'), () => devConfigMock.raw, {
-  virtual: true,
-});
-
 export const gitRevExecMock = jest.fn();
 jest.doMock('child_process', () => ({
   ...childProcessModule,
@@ -48,7 +41,6 @@ jest.doMock('fs', () => ({
 
 export const resetAllMocks = () => {
   packageMock.raw = {};
-  devConfigMock.raw = {};
   gitRevExecMock.mockReset();
   readUuidFileMock.mockReset();
   jest.resetModules();

packages/kbn-apm-config-loader/src/config.test.ts

@@ -5,23 +5,21 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-import type { Labels } from 'elastic-apm-node';
+import type { AgentConfigOptions, Labels } from 'elastic-apm-node';
 import {
   packageMock,
   mockedRootDir,
   gitRevExecMock,
-  devConfigMock,
   readUuidFileMock,
   resetAllMocks,
 } from './config.test.mocks';
 
-import { ApmConfiguration } from './config';
+import { ApmConfiguration, CENTRALIZED_SERVICE_BASE_CONFIG } from './config';
 
 describe('ApmConfiguration', () => {
   beforeEach(() => {
     // start with an empty env to avoid CI from spoiling snapshots, env is unique for each jest file
     process.env = {};
-    devConfigMock.raw = {};
     packageMock.raw = {
       version: '8.0.0',
       build: {
@@ -150,82 +148,58 @@ describe('ApmConfiguration', () => {
     );
   });
 
-  it('loads the configuration from the dev config is present', () => {
-    devConfigMock.raw = {
-      active: true,
-      serverUrl: 'https://dev-url.co',
-    };
-    const config = new ApmConfiguration(mockedRootDir, {}, false);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        active: true,
-        serverUrl: 'https://dev-url.co',
-      })
-    );
-  });
+  describe('env vars', () => {
+    beforeEach(() => {
+      delete process.env.ELASTIC_APM_ENVIRONMENT;
+      delete process.env.ELASTIC_APM_SECRET_TOKEN;
+      delete process.env.ELASTIC_APM_SERVER_URL;
+      delete process.env.NODE_ENV;
+    });
 
-  it('does not load the configuration from the dev config in distributable', () => {
-    devConfigMock.raw = {
-      active: false,
-    };
-    const config = new ApmConfiguration(mockedRootDir, {}, true);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        active: true,
-      })
-    );
-  });
+    it('correctly sets environment by reading env vars', () => {
+      let config = new ApmConfiguration(mockedRootDir, {}, false);
+      expect(config.getConfig('serviceName')).toEqual(
+        expect.objectContaining({
+          environment: 'development',
+        })
+      );
 
-  it('overwrites the standard config file with the dev config', () => {
-    const kibanaConfig = {
-      elastic: {
-        apm: {
-          active: true,
-          serverUrl: 'https://url',
-          secretToken: 'secret',
-        },
-      },
-    };
-    devConfigMock.raw = {
-      active: true,
-      serverUrl: 'https://dev-url.co',
-    };
-    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, false);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        active: true,
-        serverUrl: 'https://dev-url.co',
-        secretToken: 'secret',
-      })
-    );
-  });
+      process.env.NODE_ENV = 'production';
+      config = new ApmConfiguration(mockedRootDir, {}, false);
+      expect(config.getConfig('serviceName')).toEqual(
+        expect.objectContaining({
+          environment: 'production',
+        })
+      );
 
-  it('correctly sets environment by reading env vars', () => {
-    delete process.env.ELASTIC_APM_ENVIRONMENT;
-    delete process.env.NODE_ENV;
+      process.env.ELASTIC_APM_ENVIRONMENT = 'ci';
+      config = new ApmConfiguration(mockedRootDir, {}, false);
+      expect(config.getConfig('serviceName')).toEqual(
+        expect.objectContaining({
+          environment: 'ci',
+        })
+      );
+    });
 
-    let config = new ApmConfiguration(mockedRootDir, {}, false);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        environment: 'development',
-      })
-    );
+    it('uses default config if serverUrl is not set', () => {
+      process.env.ELASTIC_APM_SECRET_TOKEN = 'banana';
+      const config = new ApmConfiguration(mockedRootDir, {}, false);
+      const serverConfig = config.getConfig('serviceName');
+      expect(serverConfig).toHaveProperty(
+        'secretToken',
+        (CENTRALIZED_SERVICE_BASE_CONFIG as AgentConfigOptions).secretToken
+      );
+      expect(serverConfig).toHaveProperty('serverUrl', CENTRALIZED_SERVICE_BASE_CONFIG.serverUrl);
+    });
 
-    process.env.NODE_ENV = 'production';
-    config = new ApmConfiguration(mockedRootDir, {}, false);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        environment: 'production',
-      })
-    );
-
-    process.env.ELASTIC_APM_ENVIRONMENT = 'ci';
-    config = new ApmConfiguration(mockedRootDir, {}, false);
-    expect(config.getConfig('serviceName')).toEqual(
-      expect.objectContaining({
-        environment: 'ci',
-      })
-    );
+    it('uses env vars config if serverUrl is set', () => {
+      process.env.ELASTIC_APM_SECRET_TOKEN = 'banana';
+      process.env.ELASTIC_APM_SERVER_URL = 'http://banana.com/';
+      const config = new ApmConfiguration(mockedRootDir, {}, false);
+      const serverConfig = config.getConfig('serviceName');
+      expect(serverConfig).toHaveProperty('secretToken', process.env.ELASTIC_APM_SECRET_TOKEN);
+      expect(serverConfig).toHaveProperty('serverUrl', process.env.ELASTIC_APM_SERVER_URL);
+    });
   });
 
   describe('contextPropagationOnly', () => {

packages/kbn-apm-config-loader/src/config.ts

@@ -24,7 +24,7 @@ const DEFAULT_CONFIG: AgentConfigOptions = {
   globalLabels: {},
 };
 
-const CENTRALIZED_SERVICE_BASE_CONFIG: AgentConfigOptions | RUMAgentConfigOptions = {
+export const CENTRALIZED_SERVICE_BASE_CONFIG: AgentConfigOptions | RUMAgentConfigOptions = {
   serverUrl: 'https://kibana-cloud-apm.apm.us-east-1.aws.found.io',
 
   // The secretToken below is intended to be hardcoded in this file even though
@@ -136,6 +136,10 @@ export class ApmConfiguration {
       config.serverUrl = process.env.ELASTIC_APM_SERVER_URL;
     }
 
+    if (process.env.ELASTIC_APM_SECRET_TOKEN) {
+      config.secretToken = process.env.ELASTIC_APM_SECRET_TOKEN;
+    }
+
     if (process.env.ELASTIC_APM_GLOBAL_LABELS) {
       config.globalLabels = Object.fromEntries(
         process.env.ELASTIC_APM_GLOBAL_LABELS.split(',').map((p) => {
@@ -156,23 +160,6 @@ export class ApmConfiguration {
     return this.rawKibanaConfig?.elastic?.apm ?? {};
   }
 
-  /**
-   * Get the configuration from the apm.dev.js file, supersedes config
-   * from the --config file, disabled when running the distributable
-   */
-  private getDevConfig(): AgentConfigOptions {
-    if (this.isDistributable) {
-      return {};
-    }
-
-    try {
-      const apmDevConfigPath = join(this.rootDir, 'config', 'apm.dev.js');
-      return require(apmDevConfigPath);
-    } catch (e) {
-      return {};
-    }
-  }
-
   /**
    * Determine the Kibana UUID, initialized the value of `globalLabels.kibana_uuid`
    * when the UUID can be determined.
@@ -266,12 +253,7 @@ export class ApmConfiguration {
    * Reads APM configuration from different sources and merges them together.
    */
   private getConfigFromAllSources(): AgentConfigOptions {
-    const config = merge(
-      {},
-      this.getConfigFromKibanaConfig(),
-      this.getDevConfig(),
-      this.getConfigFromEnv()
-    );
+    const config = merge({}, this.getConfigFromKibanaConfig(), this.getConfigFromEnv());
 
     if (config.active === false && config.contextPropagationOnly !== false) {
       throw new Error(

packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap

@@ -74,6 +74,28 @@ Object {
 }
 `;
 
+exports[`reads yaml files from file system and parses to json, even if one is missing 1`] = `
+Object {
+  "abc": Object {
+    "def": "test",
+    "qwe": 1,
+    "zyx": Object {
+      "val": 1,
+    },
+  },
+  "bar": true,
+  "empty_arr": Array [],
+  "foo": 1,
+  "pom": Object {
+    "bom": 3,
+  },
+  "xyz": Array [
+    "1",
+    "2",
+  ],
+}
+`;
+
 exports[`returns a deep object 1`] = `
 Object {
   "pid": Object {

packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts

@@ -23,6 +23,10 @@ describe('getConfigurationFilePaths', () => {
   });
 
   it('fallbacks to `getConfigPath` value', () => {
-    expect(getConfigurationFilePaths([])).toEqual([getConfigPath()]);
+    const path = getConfigPath();
+    expect(getConfigurationFilePaths([])).toEqual([
+      path,
+      path.replace('kibana.yml', 'kibana.dev.yml'),
+    ]);
   });
 });

packages/kbn-apm-config-loader/src/utils/get_config_file_paths.ts

@@ -22,5 +22,9 @@ export const getConfigurationFilePaths = (argv: string[]): string[] => {
   if (rawPaths.length) {
     return rawPaths.map((path) => resolve(process.cwd(), path));
   }
-  return [getConfigPath()];
+
+  const configPath = getConfigPath();
+
+  // Pick up settings from dev.yml as well
+  return [configPath, configPath.replace('kibana.yml', 'kibana.dev.yml')];
 };

packages/kbn-apm-config-loader/src/utils/read_config.test.ts

@@ -29,6 +29,12 @@ test('reads and merges multiple yaml files from file system and parses to json',
   expect(config).toMatchSnapshot();
 });
 
+test('reads yaml files from file system and parses to json, even if one is missing', () => {
+  const config = getConfigFromFiles([fixtureFile('one.yml'), fixtureFile('boo.yml')]);
+
+  expect(config).toMatchSnapshot();
+});
+
 test('should inject an environment variable value when setting a value with ${ENV_VAR}', () => {
   process.env.KBN_ENV_VAR1 = 'val1';
   process.env.KBN_ENV_VAR2 = 'val2';
@@ -61,8 +67,9 @@ describe('different cwd()', () => {
     expect(config).toMatchSnapshot();
   });
 
-  test('fails to load relative paths, not found because of the cwd', () => {
+  test('ignores errors loading relative paths', () => {
     const relativePath = relative(resolve(__dirname, '..', '..'), fixtureFile('one.yml'));
-    expect(() => getConfigFromFiles([relativePath])).toThrowError(/ENOENT/);
+    const config = getConfigFromFiles([relativePath]);
+    expect(config).toStrictEqual({});
   });
 });

packages/kbn-apm-config-loader/src/utils/read_config.ts

@@ -13,7 +13,13 @@ import { set } from '@elastic/safer-lodash-set';
 import { isPlainObject } from 'lodash';
 import { ensureDeepObject } from './ensure_deep_object';
 
-const readYaml = (path: string) => safeLoad(readFileSync(path, 'utf8'));
+const readYaml = (path: string) => {
+  try {
+    return safeLoad(readFileSync(path, 'utf8'));
+  } catch (e) {
+    /* tslint:disable:no-empty */
+  }
+};
 
 function replaceEnvVarRefs(val: string) {
   return val.replace(/\$\{(\w+)\}/g, (match, envVarName) => {

src/core/server/http_resources/get_apm_config.test.ts

@@ -56,6 +56,16 @@ describe('getApmConfig', () => {
     );
   });
 
+  it('omits secret token', () => {
+    getConfigurationMock.mockReturnValue({
+      ...defaultApmConfig,
+      secretToken: 'smurfs',
+    });
+    const config = getApmConfig('/some-other-path');
+
+    expect(config).not.toHaveProperty('secretToken');
+  });
+
   it('enhance the configuration with values from the current server-side transaction', () => {
     agentMock.currentTransaction = {
       sampled: 'sampled',

src/core/server/http_resources/get_apm_config.ts

@@ -6,11 +6,19 @@
  * Side Public License, v 1.
  */
 
-import agent from 'elastic-apm-node';
+import agent, { AgentConfigOptions } from 'elastic-apm-node';
 import { getConfiguration, shouldInstrumentClient } from '@kbn/apm-config-loader';
 
+const OMIT_APM_CONFIG: Array<keyof AgentConfigOptions> = ['secretToken'];
+
 export const getApmConfig = (requestPath: string) => {
-  const baseConfig = getConfiguration('kibana-frontend');
+  const baseConfig = getConfiguration('kibana-frontend') || {};
+
+  // Omit configs not used by RUM agent.
+  OMIT_APM_CONFIG.forEach((config) => {
+    delete baseConfig[config];
+  });
+
   if (!shouldInstrumentClient(baseConfig)) {
     return null;
   }