Add the @kbn/apm-config-loader package (#77855) (#78585)

* first shot of the apm configuration loader

* revert changes to kibana config

* remove test files for now

* remove `?.` usages

* use lazy config init to avoid crashing integration test runner

* loader improvements

* add config value override via cli args

* add tests for utils package

* add prod/dev config handling + loader tests

* add tests for config

* address josh's comments

* nit on doc

package.json

@@ -124,6 +124,7 @@
     "@hapi/good-squeeze": "5.2.1",
     "@hapi/wreck": "^15.0.2",
     "@kbn/analytics": "1.0.0",
+    "@kbn/apm-config-loader": "1.0.0",
     "@kbn/babel-preset": "1.0.0",
     "@kbn/config": "1.0.0",
     "@kbn/config-schema": "1.0.0",

packages/kbn-apm-config-loader/README.md

@@ -0,0 +1,13 @@
+# @kbn/apm-config-loader
+
+Configuration loader for the APM instrumentation script.
+
+This module is only meant to be used by the APM instrumentation script (`src/apm.js`)
+to load the required configuration options from the `kibana.yaml` configuration file with
+default values.
+
+### Why not just use @kbn-config?
+
+`@kbn/config` is the recommended way to load and read the kibana configuration file,
+however in the specific case of APM, we want to only need the minimal dependencies
+before loading `elastic-apm-node` to avoid losing instrumentation on the already loaded modules.

packages/kbn-apm-config-loader/__fixtures__/config.yml

@@ -0,0 +1,11 @@
+pid:
+  enabled: true
+  file: '/var/run/kibana.pid'
+  obj: { val: 3 }
+  arr: [1]
+  empty_obj: {}
+  empty_arr: []
+obj: { val: 3 }
+arr: [1, 2]
+empty_obj: {}
+empty_arr: []

packages/kbn-apm-config-loader/__fixtures__/config_flat.yml

@@ -0,0 +1,6 @@
+pid.enabled: true
+pid.file: '/var/run/kibana.pid'
+pid.obj: { val: 3 }
+pid.arr: [1, 2]
+pid.empty_obj: {}
+pid.empty_arr: []

packages/kbn-apm-config-loader/__fixtures__/en_var_ref_config.yml

@@ -0,0 +1,5 @@
+foo: 1
+bar: "pre-${KBN_ENV_VAR1}-mid-${KBN_ENV_VAR2}-post"
+
+elasticsearch:
+  requestHeadersWhitelist: ["${KBN_ENV_VAR1}", "${KBN_ENV_VAR2}"]

packages/kbn-apm-config-loader/__fixtures__/one.yml

@@ -0,0 +1,9 @@
+foo: 1
+bar: true
+xyz: ['1', '2']
+empty_arr: []
+abc:
+  def: test
+  qwe: 1
+  zyx: { val: 1 }
+pom.bom: 3

packages/kbn-apm-config-loader/__fixtures__/two.yml

@@ -0,0 +1,10 @@
+foo: 2
+baz: bonkers
+xyz: ['3', '4']
+arr: [1]
+empty_arr: []
+abc:
+  ghi: test2
+  qwe: 2
+  zyx: {}
+pom.mob: 4

packages/kbn-apm-config-loader/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@kbn/apm-config-loader",
+  "main": "./target/index.js",
+  "types": "./target/index.d.ts",
+  "version": "1.0.0",
+  "license": "Apache-2.0",
+  "private": true,
+  "scripts": {
+    "build": "tsc",
+    "kbn:bootstrap": "yarn build",
+    "kbn:watch": "yarn build --watch"
+  },
+  "dependencies": {
+    "@elastic/safer-lodash-set": "0.0.0",
+    "@kbn/utils": "1.0.0",
+    "js-yaml": "3.13.1",
+    "lodash": "^4.17.20"
+  },
+  "devDependencies": {
+    "typescript": "4.0.2",
+    "tsd": "^0.7.4"
+  }
+}

packages/kbn-apm-config-loader/src/config.test.mocks.ts

@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { join } from 'path';
+const childProcessModule = jest.requireActual('child_process');
+const fsModule = jest.requireActual('fs');
+
+export const mockedRootDir = '/root';
+
+export const packageMock = {
+  raw: {} as any,
+};
+jest.doMock(join(mockedRootDir, 'package.json'), () => packageMock.raw, { virtual: true });
+
+export const devConfigMock = {
+  raw: {} as any,
+};
+jest.doMock(join(mockedRootDir, 'config', 'apm.dev.js'), () => devConfigMock.raw, {
+  virtual: true,
+});
+
+export const gitRevExecMock = jest.fn();
+jest.doMock('child_process', () => ({
+  ...childProcessModule,
+  execSync: (command: string, options: any) => {
+    if (command.startsWith('git rev-parse')) {
+      return gitRevExecMock(command, options);
+    }
+    return childProcessModule.execSync(command, options);
+  },
+}));
+
+export const readUuidFileMock = jest.fn();
+jest.doMock('fs', () => ({
+  ...fsModule,
+  readFileSync: (path: string, options: any) => {
+    if (path.endsWith('uuid')) {
+      return readUuidFileMock(path, options);
+    }
+    return fsModule.readFileSync(path, options);
+  },
+}));
+
+export const resetAllMocks = () => {
+  packageMock.raw = {};
+  devConfigMock.raw = {};
+  gitRevExecMock.mockReset();
+  readUuidFileMock.mockReset();
+  jest.resetModules();
+};

packages/kbn-apm-config-loader/src/config.test.ts

@@ -0,0 +1,158 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+  packageMock,
+  mockedRootDir,
+  gitRevExecMock,
+  devConfigMock,
+  readUuidFileMock,
+  resetAllMocks,
+} from './config.test.mocks';
+
+import { ApmConfiguration } from './config';
+
+describe('ApmConfiguration', () => {
+  beforeEach(() => {
+    packageMock.raw = {
+      version: '8.0.0',
+      build: {
+        sha: 'sha',
+      },
+    };
+  });
+
+  afterEach(() => {
+    resetAllMocks();
+  });
+
+  it('sets the correct service name', () => {
+    packageMock.raw = {
+      version: '9.2.1',
+    };
+    const config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('myservice').serviceName).toBe('myservice-9_2_1');
+  });
+
+  it('sets the git revision from `git rev-parse` command in non distribution mode', () => {
+    gitRevExecMock.mockReturnValue('some-git-rev');
+    const config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName').globalLabels.git_rev).toBe('some-git-rev');
+  });
+
+  it('sets the git revision from `pkg.build.sha` in distribution mode', () => {
+    gitRevExecMock.mockReturnValue('dev-sha');
+    packageMock.raw = {
+      version: '9.2.1',
+      build: {
+        sha: 'distribution-sha',
+      },
+    };
+    const config = new ApmConfiguration(mockedRootDir, {}, true);
+    expect(config.getConfig('serviceName').globalLabels.git_rev).toBe('distribution-sha');
+  });
+
+  it('reads the kibana uuid from the uuid file', () => {
+    readUuidFileMock.mockReturnValue('instance-uuid');
+    const config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName').globalLabels.kibana_uuid).toBe('instance-uuid');
+  });
+
+  it('uses the uuid from the kibana config if present', () => {
+    readUuidFileMock.mockReturnValue('uuid-from-file');
+    const kibanaConfig = {
+      server: {
+        uuid: 'uuid-from-config',
+      },
+    };
+    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, false);
+    expect(config.getConfig('serviceName').globalLabels.kibana_uuid).toBe('uuid-from-config');
+  });
+
+  it('uses the correct default config depending on the `isDistributable` parameter', () => {
+    let config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        serverUrl: expect.any(String),
+        secretToken: expect.any(String),
+      })
+    );
+
+    config = new ApmConfiguration(mockedRootDir, {}, true);
+    expect(Object.keys(config.getConfig('serviceName'))).not.toContain('serverUrl');
+  });
+
+  it('loads the configuration from the kibana config file', () => {
+    const kibanaConfig = {
+      elastic: {
+        apm: {
+          active: true,
+          serverUrl: 'https://url',
+          secretToken: 'secret',
+        },
+      },
+    };
+    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, true);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        active: true,
+        serverUrl: 'https://url',
+        secretToken: 'secret',
+      })
+    );
+  });
+
+  it('loads the configuration from the dev config is present', () => {
+    devConfigMock.raw = {
+      active: true,
+      serverUrl: 'https://dev-url.co',
+    };
+    const config = new ApmConfiguration(mockedRootDir, {}, true);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        active: true,
+        serverUrl: 'https://dev-url.co',
+      })
+    );
+  });
+
+  it('respect the precedence of the dev config', () => {
+    const kibanaConfig = {
+      elastic: {
+        apm: {
+          active: true,
+          serverUrl: 'https://url',
+          secretToken: 'secret',
+        },
+      },
+    };
+    devConfigMock.raw = {
+      active: true,
+      serverUrl: 'https://dev-url.co',
+    };
+    const config = new ApmConfiguration(mockedRootDir, kibanaConfig, true);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        active: true,
+        serverUrl: 'https://dev-url.co',
+        secretToken: 'secret',
+      })
+    );
+  });
+});

packages/kbn-apm-config-loader/src/config.ts

@@ -0,0 +1,139 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { join } from 'path';
+import { merge, get } from 'lodash';
+import { execSync } from 'child_process';
+// deep import to avoid loading the whole package
+import { getDataPath } from '@kbn/utils/target/path';
+import { readFileSync } from 'fs';
+import { ApmAgentConfig } from './types';
+
+const getDefaultConfig = (isDistributable: boolean): ApmAgentConfig => {
+  if (isDistributable) {
+    return {
+      active: false,
+      globalLabels: {},
+    };
+  }
+  return {
+    active: false,
+    serverUrl: 'https://f1542b814f674090afd914960583265f.apm.us-central1.gcp.cloud.es.io:443',
+    // The secretToken below is intended to be hardcoded in this file even though
+    // it makes it public. This is not a security/privacy issue. Normally we'd
+    // instead disable the need for a secretToken in the APM Server config where
+    // the data is transmitted to, but due to how it's being hosted, it's easier,
+    // for now, to simply leave it in.
+    secretToken: 'R0Gjg46pE9K9wGestd',
+    globalLabels: {},
+    breakdownMetrics: true,
+    centralConfig: false,
+    logUncaughtExceptions: true,
+  };
+};
+
+export class ApmConfiguration {
+  private baseConfig?: any;
+  private kibanaVersion: string;
+  private pkgBuild: Record<string, any>;
+
+  constructor(
+    private readonly rootDir: string,
+    private readonly rawKibanaConfig: Record<string, any>,
+    private readonly isDistributable: boolean
+  ) {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const { version, build } = require(join(this.rootDir, 'package.json'));
+    this.kibanaVersion = version.replace(/\./g, '_');
+    this.pkgBuild = build;
+  }
+
+  public getConfig(serviceName: string): ApmAgentConfig {
+    return {
+      ...this.getBaseConfig(),
+      serviceName: `${serviceName}-${this.kibanaVersion}`,
+    };
+  }
+
+  private getBaseConfig() {
+    if (!this.baseConfig) {
+      const apmConfig = merge(
+        getDefaultConfig(this.isDistributable),
+        this.getConfigFromKibanaConfig(),
+        this.getDevConfig()
+      );
+
+      const rev = this.getGitRev();
+      if (rev !== null) {
+        apmConfig.globalLabels.git_rev = rev;
+      }
+
+      const uuid = this.getKibanaUuid();
+      if (uuid) {
+        apmConfig.globalLabels.kibana_uuid = uuid;
+      }
+      this.baseConfig = apmConfig;
+    }
+
+    return this.baseConfig;
+  }
+
+  private getConfigFromKibanaConfig(): ApmAgentConfig {
+    return get(this.rawKibanaConfig, 'elastic.apm', {});
+  }
+
+  private getKibanaUuid() {
+    // try to access the `server.uuid` value from the config file first.
+    // if not manually defined, we will then read the value from the `{DATA_FOLDER}/uuid` file.
+    // note that as the file is created by the platform AFTER apm init, the file
+    // will not be present at first startup, but there is nothing we can really do about that.
+    if (get(this.rawKibanaConfig, 'server.uuid')) {
+      return this.rawKibanaConfig.server.uuid;
+    }
+
+    const dataPath: string = get(this.rawKibanaConfig, 'path.data') || getDataPath();
+    try {
+      const filename = join(dataPath, 'uuid');
+      return readFileSync(filename, 'utf-8');
+    } catch (e) {} // eslint-disable-line no-empty
+  }
+
+  private getDevConfig(): ApmAgentConfig {
+    try {
+      const apmDevConfigPath = join(this.rootDir, 'config', 'apm.dev.js');
+      return require(apmDevConfigPath);
+    } catch (e) {
+      return {};
+    }
+  }
+
+  private getGitRev() {
+    if (this.isDistributable) {
+      return this.pkgBuild.sha;
+    }
+    try {
+      return execSync('git rev-parse --short HEAD', {
+        encoding: 'utf-8' as BufferEncoding,
+        stdio: ['ignore', 'pipe', 'ignore'],
+      }).trim();
+    } catch (e) {
+      return null;
+    }
+  }
+}

packages/kbn-apm-config-loader/src/config_loader.test.mocks.ts

@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const getConfigurationFilePathsMock = jest.fn();
+jest.doMock('./utils/get_config_file_paths', () => ({
+  getConfigurationFilePaths: getConfigurationFilePathsMock,
+}));
+
+export const getConfigFromFilesMock = jest.fn();
+jest.doMock('./utils/read_config', () => ({
+  getConfigFromFiles: getConfigFromFilesMock,
+}));
+
+export const applyConfigOverridesMock = jest.fn();
+jest.doMock('./utils/apply_config_overrides', () => ({
+  applyConfigOverrides: applyConfigOverridesMock,
+}));
+
+export const ApmConfigurationMock = jest.fn();
+jest.doMock('./config', () => ({
+  ApmConfiguration: ApmConfigurationMock,
+}));
+
+export const resetAllMocks = () => {
+  getConfigurationFilePathsMock.mockReset();
+  getConfigFromFilesMock.mockReset();
+  applyConfigOverridesMock.mockReset();
+  ApmConfigurationMock.mockReset();
+};

packages/kbn-apm-config-loader/src/config_loader.test.ts

@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+  ApmConfigurationMock,
+  applyConfigOverridesMock,
+  getConfigFromFilesMock,
+  getConfigurationFilePathsMock,
+  resetAllMocks,
+} from './config_loader.test.mocks';
+
+import { loadConfiguration } from './config_loader';
+
+describe('loadConfiguration', () => {
+  const argv = ['some', 'arbitrary', 'args'];
+  const rootDir = '/root/dir';
+  const isDistributable = false;
+
+  afterEach(() => {
+    resetAllMocks();
+  });
+
+  it('calls `getConfigurationFilePaths` with the correct arguments', () => {
+    loadConfiguration(argv, rootDir, isDistributable);
+    expect(getConfigurationFilePathsMock).toHaveBeenCalledTimes(1);
+    expect(getConfigurationFilePathsMock).toHaveBeenCalledWith(argv);
+  });
+
+  it('calls `getConfigFromFiles` with the correct arguments', () => {
+    const configPaths = ['/path/to/config', '/path/to/other/config'];
+    getConfigurationFilePathsMock.mockReturnValue(configPaths);
+
+    loadConfiguration(argv, rootDir, isDistributable);
+    expect(getConfigFromFilesMock).toHaveBeenCalledTimes(1);
+    expect(getConfigFromFilesMock).toHaveBeenCalledWith(configPaths);
+  });
+
+  it('calls `applyConfigOverrides` with the correct arguments', () => {
+    const config = { server: { uuid: 'uuid' } };
+    getConfigFromFilesMock.mockReturnValue(config);
+
+    loadConfiguration(argv, rootDir, isDistributable);
+    expect(applyConfigOverridesMock).toHaveBeenCalledTimes(1);
+    expect(applyConfigOverridesMock).toHaveBeenCalledWith(config, argv);
+  });
+
+  it('creates and return an `ApmConfiguration` instance', () => {
+    const apmInstance = { apmInstance: true };
+    ApmConfigurationMock.mockImplementation(() => apmInstance);
+
+    const config = { server: { uuid: 'uuid' } };
+    getConfigFromFilesMock.mockReturnValue(config);
+
+    const instance = loadConfiguration(argv, rootDir, isDistributable);
+    expect(ApmConfigurationMock).toHaveBeenCalledTimes(1);
+    expect(ApmConfigurationMock).toHaveBeenCalledWith(rootDir, config, isDistributable);
+    expect(instance).toBe(apmInstance);
+  });
+});

packages/kbn-apm-config-loader/src/config_loader.ts

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { getConfigurationFilePaths, getConfigFromFiles, applyConfigOverrides } from './utils';
+import { ApmConfiguration } from './config';
+
+/**
+ * Load the APM configuration.
+ *
+ * @param argv the `process.argv` arguments
+ * @param rootDir The root directory of kibana (where the sources and the `package.json` file are)
+ * @param production true for production builds, false otherwise
+ */
+export const loadConfiguration = (
+  argv: string[],
+  rootDir: string,
+  isDistributable: boolean
+): ApmConfiguration => {
+  const configPaths = getConfigurationFilePaths(argv);
+  const rawConfiguration = getConfigFromFiles(configPaths);
+  applyConfigOverrides(rawConfiguration, argv);
+  return new ApmConfiguration(rootDir, rawConfiguration, isDistributable);
+};

packages/kbn-apm-config-loader/src/index.ts

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { loadConfiguration } from './config_loader';
+export type { ApmConfiguration } from './config';
+export type { ApmAgentConfig } from './types';

packages/kbn-apm-config-loader/src/types.ts

@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// There is an (incomplete) `AgentConfigOptions` type declared in node_modules/elastic-apm-node/index.d.ts
+// but it's not exported, and using ts tricks to retrieve the type via Parameters<ApmAgent['start']>[0]
+// causes errors in the generated .d.ts file because of esModuleInterop and the fact that the apm module
+// is just exporting an instance of the `ApmAgent` type.
+export type ApmAgentConfig = Record<string, any>;

packages/kbn-apm-config-loader/src/utils/__snapshots__/read_config.test.ts.snap

@@ -0,0 +1,108 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`different cwd() resolves relative files based on the cwd 1`] = `
+Object {
+  "abc": Object {
+    "def": "test",
+    "qwe": 1,
+    "zyx": Object {
+      "val": 1,
+    },
+  },
+  "bar": true,
+  "empty_arr": Array [],
+  "foo": 1,
+  "pom": Object {
+    "bom": 3,
+  },
+  "xyz": Array [
+    "1",
+    "2",
+  ],
+}
+`;
+
+exports[`reads and merges multiple yaml files from file system and parses to json 1`] = `
+Object {
+  "abc": Object {
+    "def": "test",
+    "ghi": "test2",
+    "qwe": 2,
+    "zyx": Object {},
+  },
+  "arr": Array [
+    1,
+  ],
+  "bar": true,
+  "baz": "bonkers",
+  "empty_arr": Array [],
+  "foo": 2,
+  "pom": Object {
+    "bom": 3,
+    "mob": 4,
+  },
+  "xyz": Array [
+    "3",
+    "4",
+  ],
+}
+`;
+
+exports[`reads single yaml from file system and parses to json 1`] = `
+Object {
+  "arr": Array [
+    1,
+    2,
+  ],
+  "empty_arr": Array [],
+  "empty_obj": Object {},
+  "obj": Object {
+    "val": 3,
+  },
+  "pid": Object {
+    "arr": Array [
+      1,
+    ],
+    "empty_arr": Array [],
+    "empty_obj": Object {},
+    "enabled": true,
+    "file": "/var/run/kibana.pid",
+    "obj": Object {
+      "val": 3,
+    },
+  },
+}
+`;
+
+exports[`returns a deep object 1`] = `
+Object {
+  "pid": Object {
+    "arr": Array [
+      1,
+      2,
+    ],
+    "empty_arr": Array [],
+    "empty_obj": Object {},
+    "enabled": true,
+    "file": "/var/run/kibana.pid",
+    "obj": Object {
+      "val": 3,
+    },
+  },
+}
+`;
+
+exports[`should inject an environment variable value when setting a value with \${ENV_VAR} 1`] = `
+Object {
+  "bar": "pre-val1-mid-val2-post",
+  "elasticsearch": Object {
+    "requestHeadersWhitelist": Array [
+      "val1",
+      "val2",
+    ],
+  },
+  "foo": 1,
+}
+`;
+
+exports[`should throw an exception when referenced environment variable in a config value does not exist 1`] = `"Unknown environment variable referenced in config : KBN_ENV_VAR1"`;

packages/kbn-apm-config-loader/src/utils/apply_config_overrides.test.ts

@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { applyConfigOverrides } from './apply_config_overrides';
+
+describe('applyConfigOverrides', () => {
+  it('overrides `server.uuid` when provided as a command line argument', () => {
+    const config: Record<string, any> = {
+      server: {
+        uuid: 'from-config',
+      },
+    };
+    const argv = ['--server.uuid', 'from-argv'];
+
+    applyConfigOverrides(config, argv);
+
+    expect(config.server.uuid).toEqual('from-argv');
+  });
+
+  it('overrides `path.data` when provided as a command line argument', () => {
+    const config: Record<string, any> = {
+      path: {
+        data: '/from/config',
+      },
+    };
+    const argv = ['--path.data', '/from/argv'];
+
+    applyConfigOverrides(config, argv);
+
+    expect(config.path.data).toEqual('/from/argv');
+  });
+
+  it('properly set the overridden properties even if the parent object is not present in the config', () => {
+    const config: Record<string, any> = {};
+    const argv = ['--server.uuid', 'from-argv', '--path.data', '/data-path'];
+
+    applyConfigOverrides(config, argv);
+
+    expect(config.server.uuid).toEqual('from-argv');
+    expect(config.path.data).toEqual('/data-path');
+  });
+});

packages/kbn-apm-config-loader/src/utils/apply_config_overrides.ts

@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { set } from '@elastic/safer-lodash-set';
+import { getArgValue } from './read_argv';
+
+/**
+ * Manually applies the specific configuration overrides we need to load the APM config.
+ * Currently, only these are needed:
+ *   - server.uuid
+ *   - path.data
+ */
+export const applyConfigOverrides = (config: Record<string, any>, argv: string[]) => {
+  const serverUuid = getArgValue(argv, '--server.uuid');
+  if (serverUuid) {
+    set(config, 'server.uuid', serverUuid);
+  }
+  const dataPath = getArgValue(argv, '--path.data');
+  if (dataPath) {
+    set(config, 'path.data', dataPath);
+  }
+};

packages/kbn-apm-config-loader/src/utils/ensure_deep_object.test.ts

@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ensureDeepObject } from './ensure_deep_object';
+
+test('flat object', () => {
+  const obj = {
+    'foo.a': 1,
+    'foo.b': 2,
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    foo: {
+      a: 1,
+      b: 2,
+    },
+  });
+});
+
+test('deep object', () => {
+  const obj = {
+    foo: {
+      a: 1,
+      b: 2,
+    },
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    foo: {
+      a: 1,
+      b: 2,
+    },
+  });
+});
+
+test('flat within deep object', () => {
+  const obj = {
+    foo: {
+      b: 2,
+      'bar.a': 1,
+    },
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    foo: {
+      b: 2,
+      bar: {
+        a: 1,
+      },
+    },
+  });
+});
+
+test('flat then flat object', () => {
+  const obj = {
+    'foo.bar': {
+      b: 2,
+      'quux.a': 1,
+    },
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    foo: {
+      bar: {
+        b: 2,
+        quux: {
+          a: 1,
+        },
+      },
+    },
+  });
+});
+
+test('full with empty array', () => {
+  const obj = {
+    a: 1,
+    b: [],
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    a: 1,
+    b: [],
+  });
+});
+
+test('full with array of primitive values', () => {
+  const obj = {
+    a: 1,
+    b: [1, 2, 3],
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    a: 1,
+    b: [1, 2, 3],
+  });
+});
+
+test('full with array of full objects', () => {
+  const obj = {
+    a: 1,
+    b: [{ c: 2 }, { d: 3 }],
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    a: 1,
+    b: [{ c: 2 }, { d: 3 }],
+  });
+});
+
+test('full with array of flat objects', () => {
+  const obj = {
+    a: 1,
+    b: [{ 'c.d': 2 }, { 'e.f': 3 }],
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    a: 1,
+    b: [{ c: { d: 2 } }, { e: { f: 3 } }],
+  });
+});
+
+test('flat with flat and array of flat objects', () => {
+  const obj = {
+    a: 1,
+    'b.c': 2,
+    d: [3, { 'e.f': 4 }, { 'g.h': 5 }],
+  };
+
+  expect(ensureDeepObject(obj)).toEqual({
+    a: 1,
+    b: { c: 2 },
+    d: [3, { e: { f: 4 } }, { g: { h: 5 } }],
+  });
+});
+
+test('array composed of flat objects', () => {
+  const arr = [{ 'c.d': 2 }, { 'e.f': 3 }];
+
+  expect(ensureDeepObject(arr)).toEqual([{ c: { d: 2 } }, { e: { f: 3 } }]);
+});

packages/kbn-apm-config-loader/src/utils/ensure_deep_object.ts

@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const separator = '.';
+
+/**
+ * Recursively traverses through the object's properties and expands ones with
+ * dot-separated names into nested objects (eg. { a.b: 'c'} -> { a: { b: 'c' }).
+ * @param obj Object to traverse through.
+ * @returns Same object instance with expanded properties.
+ */
+export function ensureDeepObject(obj: any): any {
+  if (obj == null || typeof obj !== 'object') {
+    return obj;
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) => ensureDeepObject(item));
+  }
+
+  return Object.keys(obj).reduce((fullObject, propertyKey) => {
+    const propertyValue = obj[propertyKey];
+    if (!propertyKey.includes(separator)) {
+      fullObject[propertyKey] = ensureDeepObject(propertyValue);
+    } else {
+      walk(fullObject, propertyKey.split(separator), propertyValue);
+    }
+
+    return fullObject;
+  }, {} as any);
+}
+
+function walk(obj: any, keys: string[], value: any) {
+  const key = keys.shift()!;
+  if (keys.length === 0) {
+    obj[key] = value;
+    return;
+  }
+
+  if (obj[key] === undefined) {
+    obj[key] = {};
+  }
+
+  walk(obj[key], keys, ensureDeepObject(value));
+}

packages/kbn-apm-config-loader/src/utils/get_config_file_paths.test.ts

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { resolve, join } from 'path';
+import { getConfigPath } from '@kbn/utils';
+import { getConfigurationFilePaths } from './get_config_file_paths';
+
+describe('getConfigurationFilePaths', () => {
+  const cwd = process.cwd();
+
+  it('retrieve the config file paths from the command line arguments', () => {
+    const argv = ['--config', './relative-path', '-c', '/absolute-path'];
+
+    expect(getConfigurationFilePaths(argv)).toEqual([
+      resolve(cwd, join('.', 'relative-path')),
+      '/absolute-path',
+    ]);
+  });
+
+  it('fallbacks to `getConfigPath` value', () => {
+    expect(getConfigurationFilePaths([])).toEqual([getConfigPath()]);
+  });
+});

packages/kbn-apm-config-loader/src/utils/get_config_file_paths.ts

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { resolve } from 'path';
+// deep import to avoid loading the whole package
+import { getConfigPath } from '@kbn/utils/target/path';
+import { getArgValues } from './read_argv';
+
+/**
+ * Return the configuration files that needs to be loaded.
+ *
+ * This mimics the behavior of the `src/cli/serve/serve.js` cli script by reading
+ * `-c` and `--config` options from process.argv, and fallbacks to `@kbn/utils`'s `getConfigPath()`
+ */
+export const getConfigurationFilePaths = (argv: string[]): string[] => {
+  const rawPaths = getArgValues(argv, ['-c', '--config']);
+  if (rawPaths.length) {
+    return rawPaths.map((path) => resolve(process.cwd(), path));
+  }
+  return [getConfigPath()];
+};

packages/kbn-apm-config-loader/src/utils/index.ts

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { getConfigFromFiles } from './read_config';
+export { getConfigurationFilePaths } from './get_config_file_paths';
+export { applyConfigOverrides } from './apply_config_overrides';

packages/kbn-apm-config-loader/src/utils/read_argv.test.ts

@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { getArgValue, getArgValues } from './read_argv';
+
+describe('getArgValues', () => {
+  it('retrieve the arg value from the provided argv arguments', () => {
+    const argValues = getArgValues(
+      ['--config', 'my-config', '--foo', '-b', 'bar', '--config', 'other-config', '--baz'],
+      '--config'
+    );
+    expect(argValues).toEqual(['my-config', 'other-config']);
+  });
+
+  it('accept aliases', () => {
+    const argValues = getArgValues(
+      ['--config', 'my-config', '--foo', '-b', 'bar', '-c', 'other-config', '--baz'],
+      ['--config', '-c']
+    );
+    expect(argValues).toEqual(['my-config', 'other-config']);
+  });
+
+  it('returns an empty array when the arg is not found', () => {
+    const argValues = getArgValues(
+      ['--config', 'my-config', '--foo', '-b', 'bar', '-c', 'other-config', '--baz'],
+      '--unicorn'
+    );
+    expect(argValues).toEqual([]);
+  });
+
+  it('ignores the flag when no value is provided', () => {
+    const argValues = getArgValues(
+      ['-c', 'my-config', '--foo', '-b', 'bar', '--config'],
+      ['--config', '-c']
+    );
+    expect(argValues).toEqual(['my-config']);
+  });
+});
+
+describe('getArgValue', () => {
+  it('retrieve the first arg value from the provided argv arguments', () => {
+    const argValues = getArgValue(
+      ['--config', 'my-config', '--foo', '-b', 'bar', '--config', 'other-config', '--baz'],
+      '--config'
+    );
+    expect(argValues).toEqual('my-config');
+  });
+
+  it('accept aliases', () => {
+    const argValues = getArgValue(
+      ['-c', 'my-config', '--foo', '-b', 'bar', '--config', 'other-config', '--baz'],
+      ['--config', '-c']
+    );
+    expect(argValues).toEqual('my-config');
+  });
+
+  it('returns undefined the arg is not found', () => {
+    const argValues = getArgValue(
+      ['--config', 'my-config', '--foo', '-b', 'bar', '-c', 'other-config', '--baz'],
+      '--unicorn'
+    );
+    expect(argValues).toBeUndefined();
+  });
+});

packages/kbn-apm-config-loader/src/utils/read_argv.ts

@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const getArgValues = (argv: string[], flag: string | string[]): string[] => {
+  const flags = typeof flag === 'string' ? [flag] : flag;
+  const values: string[] = [];
+  for (let i = 0; i < argv.length; i++) {
+    if (flags.includes(argv[i]) && argv[i + 1]) {
+      values.push(argv[++i]);
+    }
+  }
+  return values;
+};
+
+export const getArgValue = (argv: string[], flag: string | string[]): string | undefined => {
+  const values = getArgValues(argv, flag);
+  if (values.length) {
+    return values[0];
+  }
+};

packages/kbn-apm-config-loader/src/utils/read_config.test.ts

@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { relative, resolve } from 'path';
+import { getConfigFromFiles } from './read_config';
+
+const fixtureFile = (name: string) => resolve(__dirname, '..', '..', '__fixtures__', name);
+
+test('reads single yaml from file system and parses to json', () => {
+  const config = getConfigFromFiles([fixtureFile('config.yml')]);
+
+  expect(config).toMatchSnapshot();
+});
+
+test('returns a deep object', () => {
+  const config = getConfigFromFiles([fixtureFile('config_flat.yml')]);
+
+  expect(config).toMatchSnapshot();
+});
+
+test('reads and merges multiple yaml files from file system and parses to json', () => {
+  const config = getConfigFromFiles([fixtureFile('one.yml'), fixtureFile('two.yml')]);
+
+  expect(config).toMatchSnapshot();
+});
+
+test('should inject an environment variable value when setting a value with ${ENV_VAR}', () => {
+  process.env.KBN_ENV_VAR1 = 'val1';
+  process.env.KBN_ENV_VAR2 = 'val2';
+
+  const config = getConfigFromFiles([fixtureFile('en_var_ref_config.yml')]);
+
+  delete process.env.KBN_ENV_VAR1;
+  delete process.env.KBN_ENV_VAR2;
+
+  expect(config).toMatchSnapshot();
+});
+
+test('should throw an exception when referenced environment variable in a config value does not exist', () => {
+  expect(() =>
+    getConfigFromFiles([fixtureFile('en_var_ref_config.yml')])
+  ).toThrowErrorMatchingSnapshot();
+});
+
+describe('different cwd()', () => {
+  const originalCwd = process.cwd();
+  const tempCwd = resolve(__dirname);
+
+  beforeAll(() => process.chdir(tempCwd));
+  afterAll(() => process.chdir(originalCwd));
+
+  test('resolves relative files based on the cwd', () => {
+    const relativePath = relative(tempCwd, fixtureFile('one.yml'));
+    const config = getConfigFromFiles([relativePath]);
+
+    expect(config).toMatchSnapshot();
+  });
+
+  test('fails to load relative paths, not found because of the cwd', () => {
+    const relativePath = relative(resolve(__dirname, '..', '..'), fixtureFile('one.yml'));
+    expect(() => getConfigFromFiles([relativePath])).toThrowError(/ENOENT/);
+  });
+});

packages/kbn-apm-config-loader/src/utils/read_config.ts

@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { readFileSync } from 'fs';
+import { safeLoad } from 'js-yaml';
+
+import { set } from '@elastic/safer-lodash-set';
+import { isPlainObject } from 'lodash';
+import { ensureDeepObject } from './ensure_deep_object';
+
+const readYaml = (path: string) => safeLoad(readFileSync(path, 'utf8'));
+
+function replaceEnvVarRefs(val: string) {
+  return val.replace(/\$\{(\w+)\}/g, (match, envVarName) => {
+    const envVarValue = process.env[envVarName];
+    if (envVarValue !== undefined) {
+      return envVarValue;
+    }
+
+    throw new Error(`Unknown environment variable referenced in config : ${envVarName}`);
+  });
+}
+
+function merge(target: Record<string, any>, value: any, key?: string) {
+  if ((isPlainObject(value) || Array.isArray(value)) && Object.keys(value).length > 0) {
+    for (const [subKey, subVal] of Object.entries(value)) {
+      merge(target, subVal, key ? `${key}.${subKey}` : subKey);
+    }
+  } else if (key !== undefined) {
+    set(target, key, typeof value === 'string' ? replaceEnvVarRefs(value) : value);
+  }
+
+  return target;
+}
+
+/** @internal */
+export const getConfigFromFiles = (configFiles: readonly string[]): Record<string, any> => {
+  let mergedYaml: Record<string, any> = {};
+
+  for (const configFile of configFiles) {
+    const yaml = readYaml(configFile);
+    if (yaml !== null) {
+      mergedYaml = merge(mergedYaml, yaml);
+    }
+  }
+
+  return ensureDeepObject(mergedYaml);
+};

packages/kbn-apm-config-loader/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "declaration": true,
+    "outDir": "./target",
+    "stripInternal": false,
+    "declarationMap": true,
+    "types": ["jest", "node"]
+  },
+  "include": ["./src/**/*.ts"],
+  "exclude": ["target"]
+}

packages/kbn-apm-config-loader/yarn.lock

@@ -0,0 +1 @@
+../../yarn.lock

src/apm.js

@@ -18,67 +18,11 @@
  */
 
 const { join } = require('path');
-const { readFileSync } = require('fs');
-const { execSync } = require('child_process');
-const { merge } = require('lodash');
-const { name, version, build } = require('../package.json');
+const { name, build } = require('../package.json');
+const { loadConfiguration } = require('@kbn/apm-config-loader');
 
 const ROOT_DIR = join(__dirname, '..');
-
-function gitRev() {
-  try {
-    return execSync('git rev-parse --short HEAD', {
-      encoding: 'utf-8',
-      stdio: ['ignore', 'pipe', 'ignore'],
-    }).trim();
-  } catch (e) {
-    return null;
-  }
-}
-
-function devConfig() {
-  try {
-    const apmDevConfigPath = join(ROOT_DIR, 'config', 'apm.dev.js');
-    return require(apmDevConfigPath); // eslint-disable-line import/no-dynamic-require
-  } catch (e) {
-    return {};
-  }
-}
-
-const apmConfig = merge(
-  {
-    active: false,
-    serverUrl: 'https://f1542b814f674090afd914960583265f.apm.us-central1.gcp.cloud.es.io:443',
-    // The secretToken below is intended to be hardcoded in this file even though
-    // it makes it public. This is not a security/privacy issue. Normally we'd
-    // instead disable the need for a secretToken in the APM Server config where
-    // the data is transmitted to, but due to how it's being hosted, it's easier,
-    // for now, to simply leave it in.
-    secretToken: 'R0Gjg46pE9K9wGestd',
-    globalLabels: {},
-    breakdownMetrics: true,
-    centralConfig: false,
-    logUncaughtExceptions: true,
-  },
-  devConfig()
-);
-
-try {
-  const filename = join(ROOT_DIR, 'data', 'uuid');
-  apmConfig.globalLabels.kibana_uuid = readFileSync(filename, 'utf-8');
-} catch (e) {} // eslint-disable-line no-empty
-
-const rev = gitRev();
-if (rev !== null) apmConfig.globalLabels.git_rev = rev;
-
-function getConfig(serviceName) {
-  return {
-    ...apmConfig,
-    ...{
-      serviceName: `${serviceName}-${version.replace(/\./g, '_')}`,
-    },
-  };
-}
+let apmConfig;
 
 /**
  * Flag to disable APM RUM support on all kibana builds by default
@@ -86,12 +30,24 @@ function getConfig(serviceName) {
 const isKibanaDistributable = Boolean(build && build.distributable === true);
 
 module.exports = function (serviceName = name) {
-  if (process.env.kbnWorkerType === 'optmzr') return;
-
-  const conf = getConfig(serviceName);
+  if (process.env.kbnWorkerType === 'optmzr') {
+    return;
+  }
 
+  apmConfig = loadConfiguration(process.argv, ROOT_DIR, isKibanaDistributable);
+  const conf = apmConfig.getConfig(serviceName);
   require('elastic-apm-node').start(conf);
 };
 
-module.exports.getConfig = getConfig;
+module.exports.getConfig = (serviceName) => {
+  // integration test runner starts a kibana server that import the module without initializing APM.
+  // so we need to check initialization of the config.
+  // note that we can't just load the configuration during this module's import
+  // because jest IT are ran with `--config path-to-jest-config.js` which conflicts with the CLI's `config` arg
+  // causing the config loader to try to load the jest js config as yaml and throws.
+  if (apmConfig) {
+    return apmConfig.getConfig(serviceName);
+  }
+  return {};
+};
 module.exports.isKibanaDistributable = isKibanaDistributable;