diff --git a/dev_docs/tutorials/apm_ui.png b/dev_docs/tutorials/apm_ui.png new file mode 100644 index 000000000000..889fb6695e57 Binary files /dev/null and b/dev_docs/tutorials/apm_ui.png differ diff --git a/dev_docs/tutorials/debugging.mdx b/dev_docs/tutorials/debugging.mdx index 598c6119910c..d5d86a902530 100644 --- a/dev_docs/tutorials/debugging.mdx +++ b/dev_docs/tutorials/debugging.mdx @@ -61,3 +61,49 @@ logging: - name: elasticsearch.query level: debug ``` + +## Debugging Kibana with APM + +Kibana is integrated with APM's node and RUM agents. +To learn more about how APM works and what it reports, refer to the [documentation](https://www.elastic.co/guide/en/apm/guide/current/index.html). + +We currently track the following types of transactions from Kibana: + +Frontend (APM RUM): + * `http-request`- tracks all outgoing API requests + * `page-load` - tracks the inidial loading time of kibana + * `app-change` - tracks application changes + +Backend (APM Node): + * `request` - tracks all incoming API requests + * `kibana-platform` - tracks server initiation phases (preboot, setup and start) + * `task-manager` - tracks the operation of the task manager, including claiming pending tasks and marking them as running + * `task-run` - tracks the execution of individual tasks + +### Enabling APM on a local environment + +In some cases, it is beneficial to enable APM on a local development environment to get an initial undesrtanding of a feature's performance during manual or automatic tests. + + 1. Create a secondary monitoring deployment to collect APM data. The easiest option is to use [Elastic Cloud](https://cloud.elastic.co/deployments) to create a new deployment. + 2. Open Kibana, go to `Integrations` and enable the Elastic APM integration. + 3. Scroll down and copy the server URL and secret token. You may also find them in your cloud console under APM & Fleet. + 4. Create or open `config\kibana.dev.yml` on your local development environment. + 5. Add the following settings: + ``` + elastic.apm.active: true + elastic.apm.serverUrl: + elastic.apm.secretToken: + ``` + 6. Once you run kibana and start using it, two new services (kibana, kibana-frontend) should appear under the APM UI on the APM deployment. + ![APM UI](./apm_ui.png) + +### Enabling APM via environment variables + +It is possible to enable APM via environment variables as well. +They take precedence over any values defined in `kibana.yml` or `kibana.dev.yml` + +Set the following environment variables to enable APM: + + * ELASTIC_APM_ACTIVE + * ELASTIC_APM_SERVER_URL + * ELASTIC_APM_SECRET_TOKEN diff --git a/packages/kbn-apm-config-loader/src/config.test.mocks.ts b/packages/kbn-apm-config-loader/src/config.test.mocks.ts index 4e148cbd3252..040d26deeba5 100644 --- a/packages/kbn-apm-config-loader/src/config.test.mocks.ts +++ b/packages/kbn-apm-config-loader/src/config.test.mocks.ts @@ -17,13 +17,6 @@ export const packageMock = { }; jest.doMock(join(mockedRootDir, 'package.json'), () => packageMock.raw, { virtual: true }); -export const devConfigMock = { - raw: {} as any, -}; -jest.doMock(join(mockedRootDir, 'config', 'apm.dev.js'), () => devConfigMock.raw, { - virtual: true, -}); - export const gitRevExecMock = jest.fn(); jest.doMock('child_process', () => ({ ...childProcessModule, @@ -48,7 +41,6 @@ jest.doMock('fs', () => ({ export const resetAllMocks = () => { packageMock.raw = {}; - devConfigMock.raw = {}; gitRevExecMock.mockReset(); readUuidFileMock.mockReset(); jest.resetModules(); diff --git a/packages/kbn-apm-config-loader/src/config.test.ts b/packages/kbn-apm-config-loader/src/config.test.ts index 265362fa8f16..5bc723a27590 100644 --- a/packages/kbn-apm-config-loader/src/config.test.ts +++ b/packages/kbn-apm-config-loader/src/config.test.ts @@ -5,23 +5,21 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import type { Labels } from 'elastic-apm-node'; +import type { AgentConfigOptions, Labels } from 'elastic-apm-node'; import { packageMock, mockedRootDir, gitRevExecMock, - devConfigMock, readUuidFileMock, resetAllMocks, } from './config.test.mocks'; -import { ApmConfiguration } from './config'; +import { ApmConfiguration, CENTRALIZED_SERVICE_BASE_CONFIG } from './config'; describe('ApmConfiguration', () => { beforeEach(() => { // start with an empty env to avoid CI from spoiling snapshots, env is unique for each jest file process.env = {}; - devConfigMock.raw = {}; packageMock.raw = { version: '8.0.0', build: { @@ -150,82 +148,58 @@ describe('ApmConfiguration', () => { ); }); - it('loads the configuration from the dev config is present', () => { - devConfigMock.raw = { - active: true, - serverUrl: 'https://dev-url.co', - }; - const config = new ApmConfiguration(mockedRootDir, {}, false); - expect(config.getConfig('serviceName')).toEqual( - expect.objectContaining({ - active: true, - serverUrl: 'https://dev-url.co', - }) - ); - }); + describe('env vars', () => { + beforeEach(() => { + delete process.env.ELASTIC_APM_ENVIRONMENT; + delete process.env.ELASTIC_APM_SECRET_TOKEN; + delete process.env.ELASTIC_APM_SERVER_URL; + delete process.env.NODE_ENV; + }); - it('does not load the configuration from the dev config in distributable', () => { - devConfigMock.raw = { - active: false, - }; - const config = new ApmConfiguration(mockedRootDir, {}, true); - expect(config.getConfig('serviceName')).toEqual( - expect.objectContaining({ - active: true, - }) - ); - }); + it('correctly sets environment by reading env vars', () => { + let config = new ApmConfiguration(mockedRootDir, {}, false); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + environment: 'development', + }) + ); - it('overwrites the standard config file with the dev config', () => { - const kibanaConfig = { - elastic: { - apm: { - active: true, - serverUrl: 'https://url', - secretToken: 'secret', - }, - }, - }; - devConfigMock.raw = { - active: true, - serverUrl: 'https://dev-url.co', - }; - const config = new ApmConfiguration(mockedRootDir, kibanaConfig, false); - expect(config.getConfig('serviceName')).toEqual( - expect.objectContaining({ - active: true, - serverUrl: 'https://dev-url.co', - secretToken: 'secret', - }) - ); - }); + process.env.NODE_ENV = 'production'; + config = new ApmConfiguration(mockedRootDir, {}, false); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + environment: 'production', + }) + ); - it('correctly sets environment by reading env vars', () => { - delete process.env.ELASTIC_APM_ENVIRONMENT; - delete process.env.NODE_ENV; + process.env.ELASTIC_APM_ENVIRONMENT = 'ci'; + config = new ApmConfiguration(mockedRootDir, {}, false); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + environment: 'ci', + }) + ); + }); - let config = new ApmConfiguration(mockedRootDir, {}, false); - expect(config.getConfig('serviceName')).toEqual( - expect.objectContaining({ - environment: 'development', - }) - ); + it('uses default config if serverUrl is not set', () => { + process.env.ELASTIC_APM_SECRET_TOKEN = 'banana'; + const config = new ApmConfiguration(mockedRootDir, {}, false); + const serverConfig = config.getConfig('serviceName'); + expect(serverConfig).toHaveProperty( + 'secretToken', + (CENTRALIZED_SERVICE_BASE_CONFIG as AgentConfigOptions).secretToken + ); + expect(serverConfig).toHaveProperty('serverUrl', CENTRALIZED_SERVICE_BASE_CONFIG.serverUrl); + }); - process.env.NODE_ENV = 'production'; - config = new ApmConfiguration(mockedRootDir, {}, false); - expect(config.getConfig('serviceName')).toEqual( - expect.objectContaining({ - environment: 'production', - }) - ); - - process.env.ELASTIC_APM_ENVIRONMENT = 'ci'; - config = new ApmConfiguration(mockedRootDir, {}, false); - expect(config.getConfig('serviceName')).toEqual( - expect.objectContaining({ - environment: 'ci', - }) - ); + it('uses env vars config if serverUrl is set', () => { + process.env.ELASTIC_APM_SECRET_TOKEN = 'banana'; + process.env.ELASTIC_APM_SERVER_URL = 'http://banana.com/'; + const config = new ApmConfiguration(mockedRootDir, {}, false); + const serverConfig = config.getConfig('serviceName'); + expect(serverConfig).toHaveProperty('secretToken', process.env.ELASTIC_APM_SECRET_TOKEN); + expect(serverConfig).toHaveProperty('serverUrl', process.env.ELASTIC_APM_SERVER_URL); + }); }); describe('contextPropagationOnly', () => { diff --git a/packages/kbn-apm-config-loader/src/config.ts b/packages/kbn-apm-config-loader/src/config.ts index 303ec931eeda..1094b9662014 100644 --- a/packages/kbn-apm-config-loader/src/config.ts +++ b/packages/kbn-apm-config-loader/src/config.ts @@ -24,7 +24,7 @@ const DEFAULT_CONFIG: AgentConfigOptions = { globalLabels: {}, }; -const CENTRALIZED_SERVICE_BASE_CONFIG: AgentConfigOptions | RUMAgentConfigOptions = { +export const CENTRALIZED_SERVICE_BASE_CONFIG: AgentConfigOptions | RUMAgentConfigOptions = { serverUrl: 'https://kibana-cloud-apm.apm.us-east-1.aws.found.io', // The secretToken below is intended to be hardcoded in this file even though @@ -136,6 +136,10 @@ export class ApmConfiguration { config.serverUrl = process.env.ELASTIC_APM_SERVER_URL; } + if (process.env.ELASTIC_APM_SECRET_TOKEN) { + config.secretToken = process.env.ELASTIC_APM_SECRET_TOKEN; + } + if (process.env.ELASTIC_APM_GLOBAL_LABELS) { config.globalLabels = Object.fromEntries( process.env.ELASTIC_APM_GLOBAL_LABELS.split(',').map((p) => { @@ -156,23 +160,6 @@ export class ApmConfiguration { return this.rawKibanaConfig?.elastic?.apm ?? {}; } - /** - * Get the configuration from the apm.dev.js file, supersedes config - * from the --config file, disabled when running the distributable - */ - private getDevConfig(): AgentConfigOptions { - if (this.isDistributable) { - return {}; - } - - try { - const apmDevConfigPath = join(this.rootDir, 'config', 'apm.dev.js'); - return require(apmDevConfigPath); - } catch (e) { - return {}; - } - } - /** * Determine the Kibana UUID, initialized the value of `globalLabels.kibana_uuid` * when the UUID can be determined. @@ -266,12 +253,7 @@ export class ApmConfiguration { * Reads APM configuration from different sources and merges them together. */ private getConfigFromAllSources(): AgentConfigOptions { - const config = merge( - {}, - this.getConfigFromKibanaConfig(), - this.getDevConfig(), - this.getConfigFromEnv() - ); + const config = merge({}, this.getConfigFromKibanaConfig(), this.getConfigFromEnv()); if (config.active === false && config.contextPropagationOnly !== false) { throw new Error( diff --git a/packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap b/packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap index afdce4e76d3f..ee7b6751e2c6 100644 --- a/packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap +++ b/packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap @@ -74,6 +74,28 @@ Object { } `; +exports[`reads yaml files from file system and parses to json, even if one is missing 1`] = ` +Object { + "abc": Object { + "def": "test", + "qwe": 1, + "zyx": Object { + "val": 1, + }, + }, + "bar": true, + "empty_arr": Array [], + "foo": 1, + "pom": Object { + "bom": 3, + }, + "xyz": Array [ + "1", + "2", + ], +} +`; + exports[`returns a deep object 1`] = ` Object { "pid": Object { diff --git a/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts b/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts index 3b361a1c6958..41ba8b07b66b 100644 --- a/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts +++ b/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts @@ -23,6 +23,10 @@ describe('getConfigurationFilePaths', () => { }); it('fallbacks to `getConfigPath` value', () => { - expect(getConfigurationFilePaths([])).toEqual([getConfigPath()]); + const path = getConfigPath(); + expect(getConfigurationFilePaths([])).toEqual([ + path, + path.replace('kibana.yml', 'kibana.dev.yml'), + ]); }); }); diff --git a/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.ts b/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.ts index dc5a55935a90..d2cb7fb75258 100644 --- a/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.ts +++ b/packages/kbn-apm-config-loader/src/utils/get_config_file_paths.ts @@ -22,5 +22,9 @@ export const getConfigurationFilePaths = (argv: string[]): string[] => { if (rawPaths.length) { return rawPaths.map((path) => resolve(process.cwd(), path)); } - return [getConfigPath()]; + + const configPath = getConfigPath(); + + // Pick up settings from dev.yml as well + return [configPath, configPath.replace('kibana.yml', 'kibana.dev.yml')]; }; diff --git a/packages/kbn-apm-config-loader/src/utils/read_config.test.ts b/packages/kbn-apm-config-loader/src/utils/read_config.test.ts index 2838738c0ab6..157cebf122fa 100644 --- a/packages/kbn-apm-config-loader/src/utils/read_config.test.ts +++ b/packages/kbn-apm-config-loader/src/utils/read_config.test.ts @@ -29,6 +29,12 @@ test('reads and merges multiple yaml files from file system and parses to json', expect(config).toMatchSnapshot(); }); +test('reads yaml files from file system and parses to json, even if one is missing', () => { + const config = getConfigFromFiles([fixtureFile('one.yml'), fixtureFile('boo.yml')]); + + expect(config).toMatchSnapshot(); +}); + test('should inject an environment variable value when setting a value with ${ENV_VAR}', () => { process.env.KBN_ENV_VAR1 = 'val1'; process.env.KBN_ENV_VAR2 = 'val2'; @@ -61,8 +67,9 @@ describe('different cwd()', () => { expect(config).toMatchSnapshot(); }); - test('fails to load relative paths, not found because of the cwd', () => { + test('ignores errors loading relative paths', () => { const relativePath = relative(resolve(__dirname, '..', '..'), fixtureFile('one.yml')); - expect(() => getConfigFromFiles([relativePath])).toThrowError(/ENOENT/); + const config = getConfigFromFiles([relativePath]); + expect(config).toStrictEqual({}); }); }); diff --git a/packages/kbn-apm-config-loader/src/utils/read_config.ts b/packages/kbn-apm-config-loader/src/utils/read_config.ts index 4371c7983515..0d6fce88b053 100644 --- a/packages/kbn-apm-config-loader/src/utils/read_config.ts +++ b/packages/kbn-apm-config-loader/src/utils/read_config.ts @@ -13,7 +13,13 @@ import { set } from '@elastic/safer-lodash-set'; import { isPlainObject } from 'lodash'; import { ensureDeepObject } from './ensure_deep_object'; -const readYaml = (path: string) => safeLoad(readFileSync(path, 'utf8')); +const readYaml = (path: string) => { + try { + return safeLoad(readFileSync(path, 'utf8')); + } catch (e) { + /* tslint:disable:no-empty */ + } +}; function replaceEnvVarRefs(val: string) { return val.replace(/\$\{(\w+)\}/g, (match, envVarName) => { diff --git a/src/core/server/http_resources/get_apm_config.test.ts b/src/core/server/http_resources/get_apm_config.test.ts index 9552a91da97b..5e4e7b9b9525 100644 --- a/src/core/server/http_resources/get_apm_config.test.ts +++ b/src/core/server/http_resources/get_apm_config.test.ts @@ -56,6 +56,16 @@ describe('getApmConfig', () => { ); }); + it('omits secret token', () => { + getConfigurationMock.mockReturnValue({ + ...defaultApmConfig, + secretToken: 'smurfs', + }); + const config = getApmConfig('/some-other-path'); + + expect(config).not.toHaveProperty('secretToken'); + }); + it('enhance the configuration with values from the current server-side transaction', () => { agentMock.currentTransaction = { sampled: 'sampled', diff --git a/src/core/server/http_resources/get_apm_config.ts b/src/core/server/http_resources/get_apm_config.ts index 3e7be65f9665..881efe261d79 100644 --- a/src/core/server/http_resources/get_apm_config.ts +++ b/src/core/server/http_resources/get_apm_config.ts @@ -6,11 +6,19 @@ * Side Public License, v 1. */ -import agent from 'elastic-apm-node'; +import agent, { AgentConfigOptions } from 'elastic-apm-node'; import { getConfiguration, shouldInstrumentClient } from '@kbn/apm-config-loader'; +const OMIT_APM_CONFIG: Array = ['secretToken']; + export const getApmConfig = (requestPath: string) => { - const baseConfig = getConfiguration('kibana-frontend'); + const baseConfig = getConfiguration('kibana-frontend') || {}; + + // Omit configs not used by RUM agent. + OMIT_APM_CONFIG.forEach((config) => { + delete baseConfig[config]; + }); + if (!shouldInstrumentClient(baseConfig)) { return null; }