Config loader: remove unecessary properties (#154902)

- Updates the logic of `ensureDeepObject` to remove unnecessary
properties when expanding the object
- We had three versions of this helper, centralized it within `@kbn/std`

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-apm-config-loader/src/utils/ensure_deep_object.ts

@@ -1,50 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-const separator = '.';
-
-/**
- * Recursively traverses through the object's properties and expands ones with
- * dot-separated names into nested objects (eg. { a.b: 'c'} -> { a: { b: 'c' }).
- * @param obj Object to traverse through.
- * @returns Same object instance with expanded properties.
- */
-export function ensureDeepObject(obj: any): any {
-  if (obj == null || typeof obj !== 'object') {
-    return obj;
-  }
-
-  if (Array.isArray(obj)) {
-    return obj.map((item) => ensureDeepObject(item));
-  }
-
-  return Object.keys(obj).reduce((fullObject, propertyKey) => {
-    const propertyValue = obj[propertyKey];
-    if (!propertyKey.includes(separator)) {
-      fullObject[propertyKey] = ensureDeepObject(propertyValue);
-    } else {
-      walk(fullObject, propertyKey.split(separator), propertyValue);
-    }
-
-    return fullObject;
-  }, {} as any);
-}
-
-function walk(obj: any, keys: string[], value: any) {
-  const key = keys.shift()!;
-  if (keys.length === 0) {
-    obj[key] = value;
-    return;
-  }
-
-  if (obj[key] === undefined) {
-    obj[key] = {};
-  }
-
-  walk(obj[key], keys, ensureDeepObject(value));
-}

packages/kbn-apm-config-loader/src/utils/read_config.ts

@@ -10,8 +10,8 @@ import { readFileSync } from 'fs';
 import { safeLoad } from 'js-yaml';
 
 import { set } from '@kbn/safer-lodash-set';
+import { ensureDeepObject } from '@kbn/std';
 import { isPlainObject } from 'lodash';
-import { ensureDeepObject } from './ensure_deep_object';
 
 const readYaml = (path: string) => {
   try {

packages/kbn-apm-config-loader/tsconfig.json

@@ -14,6 +14,7 @@
     "@kbn/safer-lodash-set",
     "@kbn/utils",
     "@kbn/config-schema",
+    "@kbn/std",
   ],
   "exclude": [
     "target/**/*",

packages/kbn-config/src/__fixtures__/forbidden_1.yml

@@ -0,0 +1,5 @@
+test: {
+  "aaa['__proto__.hello']": "Hello",
+  "aaa['__proto__.nested.there']": "There",
+  "aaa['__proto__.nested.here']": "This JS syntax is apparently valid for our parser"
+}

packages/kbn-config/src/__fixtures__/forbidden_2.yml

@@ -0,0 +1,3 @@
+test:
+  hello:
+    '__proto__.dolly': "Well hello there"

packages/kbn-config/src/raw/ensure_deep_object.test.ts

@@ -1,145 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { ensureDeepObject } from './ensure_deep_object';
-
-test('flat object', () => {
-  const obj = {
-    'foo.a': 1,
-    'foo.b': 2,
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    foo: {
-      a: 1,
-      b: 2,
-    },
-  });
-});
-
-test('deep object', () => {
-  const obj = {
-    foo: {
-      a: 1,
-      b: 2,
-    },
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    foo: {
-      a: 1,
-      b: 2,
-    },
-  });
-});
-
-test('flat within deep object', () => {
-  const obj = {
-    foo: {
-      b: 2,
-      'bar.a': 1,
-    },
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    foo: {
-      b: 2,
-      bar: {
-        a: 1,
-      },
-    },
-  });
-});
-
-test('flat then flat object', () => {
-  const obj = {
-    'foo.bar': {
-      b: 2,
-      'quux.a': 1,
-    },
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    foo: {
-      bar: {
-        b: 2,
-        quux: {
-          a: 1,
-        },
-      },
-    },
-  });
-});
-
-test('full with empty array', () => {
-  const obj = {
-    a: 1,
-    b: [],
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    a: 1,
-    b: [],
-  });
-});
-
-test('full with array of primitive values', () => {
-  const obj = {
-    a: 1,
-    b: [1, 2, 3],
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    a: 1,
-    b: [1, 2, 3],
-  });
-});
-
-test('full with array of full objects', () => {
-  const obj = {
-    a: 1,
-    b: [{ c: 2 }, { d: 3 }],
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    a: 1,
-    b: [{ c: 2 }, { d: 3 }],
-  });
-});
-
-test('full with array of flat objects', () => {
-  const obj = {
-    a: 1,
-    b: [{ 'c.d': 2 }, { 'e.f': 3 }],
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    a: 1,
-    b: [{ c: { d: 2 } }, { e: { f: 3 } }],
-  });
-});
-
-test('flat with flat and array of flat objects', () => {
-  const obj = {
-    a: 1,
-    'b.c': 2,
-    d: [3, { 'e.f': 4 }, { 'g.h': 5 }],
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    a: 1,
-    b: { c: 2 },
-    d: [3, { e: { f: 4 } }, { g: { h: 5 } }],
-  });
-});
-
-test('array composed of flat objects', () => {
-  const arr = [{ 'c.d': 2 }, { 'e.f': 3 }];
-
-  expect(ensureDeepObject(arr)).toEqual([{ c: { d: 2 } }, { e: { f: 3 } }]);
-});

packages/kbn-config/src/raw/ensure_deep_object.ts

@@ -1,50 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-const separator = '.';
-
-/**
- * Recursively traverses through the object's properties and expands ones with
- * dot-separated names into nested objects (eg. { a.b: 'c'} -> { a: { b: 'c' }).
- * @param obj Object to traverse through.
- * @returns Same object instance with expanded properties.
- */
-export function ensureDeepObject(obj: any): any {
-  if (obj == null || typeof obj !== 'object') {
-    return obj;
-  }
-
-  if (Array.isArray(obj)) {
-    return obj.map((item) => ensureDeepObject(item));
-  }
-
-  return Object.keys(obj).reduce((fullObject, propertyKey) => {
-    const propertyValue = obj[propertyKey];
-    if (!propertyKey.includes(separator)) {
-      fullObject[propertyKey] = ensureDeepObject(propertyValue);
-    } else {
-      walk(fullObject, propertyKey.split(separator), propertyValue);
-    }
-
-    return fullObject;
-  }, {} as any);
-}
-
-function walk(obj: any, keys: string[], value: any) {
-  const key = keys.shift()!;
-  if (keys.length === 0) {
-    obj[key] = value;
-    return;
-  }
-
-  if (obj[key] === undefined) {
-    obj[key] = {};
-  }
-
-  walk(obj[key], keys, ensureDeepObject(value));
-}

packages/kbn-config/src/raw/read_config.test.ts

@@ -47,6 +47,18 @@ test('should throw an exception when referenced environment variable in a config
   ).toThrowErrorMatchingSnapshot();
 });
 
+test('throws parsing a config with forbidden paths', () => {
+  expect(() =>
+    getConfigFromFiles([fixtureFile('forbidden_1.yml')])
+  ).toThrowErrorMatchingInlineSnapshot(`"Forbidden path detected: test.aaa.__proto__.hello"`);
+});
+
+test('throws parsing another config with forbidden paths', () => {
+  expect(() =>
+    getConfigFromFiles([fixtureFile('forbidden_2.yml')])
+  ).toThrowErrorMatchingInlineSnapshot(`"Forbidden path detected: test.hello.__proto__"`);
+});
+
 describe('different cwd()', () => {
   const originalCwd = process.cwd();
   const tempCwd = resolve(__dirname);

packages/kbn-config/src/raw/read_config.ts

@@ -11,7 +11,7 @@ import { safeLoad } from 'js-yaml';
 
 import { set } from '@kbn/safer-lodash-set';
 import { isPlainObject } from 'lodash';
-import { ensureDeepObject } from './ensure_deep_object';
+import { ensureDeepObject } from '@kbn/std';
 
 const readYaml = (path: string) => safeLoad(readFileSync(path, 'utf8'));
 

packages/kbn-std/index.ts

@@ -27,4 +27,5 @@ export {
   asyncForEach,
   asyncForEachWithLimit,
 } from './src/iteration';
+export { ensureDeepObject } from './src/ensure_deep_object';
 export { Semaphore } from './src/semaphore';

packages/kbn-std/src/ensure_deep_object.test.ts

@@ -143,3 +143,81 @@ test('array composed of flat objects', () => {
 
   expect(ensureDeepObject(arr)).toEqual([{ c: { d: 2 } }, { e: { f: 3 } }]);
 });
+
+describe('forbidden patterns', () => {
+  describe('first pattern', () => {
+    test('throws when finding the first pattern within an object', () => {
+      const obj = {
+        foo: {
+          hello: 'dolly',
+          'bar.__proto__': { yours: 'mine' },
+        },
+      };
+
+      expect(() => ensureDeepObject(obj)).toThrowErrorMatchingInlineSnapshot(
+        `"Forbidden path detected: foo.bar.__proto__"`
+      );
+    });
+
+    test('throws when finding the first pattern within an array', () => {
+      const obj = {
+        array: [
+          'hello',
+          {
+            'bar.__proto__': { their: 'mine' },
+          },
+        ],
+      };
+
+      expect(() => ensureDeepObject(obj)).toThrowErrorMatchingInlineSnapshot(
+        `"Forbidden path detected: array.1.bar.__proto__"`
+      );
+    });
+  });
+
+  describe('second pattern', () => {
+    test('throws when finding the first pattern within an object', () => {
+      const obj = {
+        foo: {
+          hello: 'dolly',
+          'bar.constructor.prototype': { foo: 'bar' },
+        },
+      };
+
+      expect(() => ensureDeepObject(obj)).toThrowErrorMatchingInlineSnapshot(
+        `"Forbidden path detected: foo.bar.constructor.prototype"`
+      );
+    });
+
+    test('throws when finding the first pattern within a nested object', () => {
+      const obj = {
+        foo: {
+          hello: 'dolly',
+          'bar.constructor': {
+            main: 'mine',
+            prototype: 'nope',
+          },
+        },
+      };
+
+      expect(() => ensureDeepObject(obj)).toThrowErrorMatchingInlineSnapshot(
+        `"Forbidden path detected: foo.bar.constructor.prototype"`
+      );
+    });
+
+    test('throws when finding the first pattern within an array', () => {
+      const obj = {
+        array: [
+          'hello',
+          {
+            'bar.constructor.prototype': { foo: 'bar' },
+          },
+        ],
+      };
+
+      expect(() => ensureDeepObject(obj)).toThrowErrorMatchingInlineSnapshot(
+        `"Forbidden path detected: array.1.bar.constructor.prototype"`
+      );
+    });
+  });
+});

packages/kbn-std/src/ensure_deep_object.ts

@@ -6,56 +6,61 @@
  * Side Public License, v 1.
  */
 
-//
-// THIS IS A DIRECT COPY OF
-// 'packages/kbn-config/src/raw/ensure_deep_object.ts'
-// BECAUSE THAT IS BLOCKED FOR IMPORTING BY OUR LINTER.
-//
-// IF THAT IS EXPOSED, WE SHOULD USE IT RATHER THAN CLONE IT.
-//
-
-/* eslint-disable @typescript-eslint/no-explicit-any */
-// ^ Disabling the rule for the entire file because of the complexity to type this
-
+const FORBIDDEN_PATTERNS = ['__proto__', 'constructor.prototype'];
 const separator = '.';
 
 /**
  * Recursively traverses through the object's properties and expands ones with
  * dot-separated names into nested objects (eg. { a.b: 'c'} -> { a: { b: 'c' }).
  * @param obj Object to traverse through.
+ * @param path The current path of the traversal
  * @returns Same object instance with expanded properties.
  */
-export function ensureDeepObject(obj: any): any {
+export function ensureDeepObject(obj: any, path: string[] = []): any {
+  assertValidPath(path);
+
   if (obj == null || typeof obj !== 'object') {
     return obj;
   }
 
   if (Array.isArray(obj)) {
-    return obj.map((item) => ensureDeepObject(item));
+    return obj.map((item, index) => ensureDeepObject(item, [...path, `${index}`]));
   }
 
   return Object.keys(obj).reduce((fullObject, propertyKey) => {
     const propertyValue = obj[propertyKey];
-    if (!propertyKey.includes(separator)) {
-      fullObject[propertyKey] = ensureDeepObject(propertyValue);
+    const propertySplits = propertyKey.split(separator);
+    if (propertySplits.length === 1) {
+      fullObject[propertyKey] = ensureDeepObject(propertyValue, [...path, propertyKey]);
     } else {
-      walk(fullObject, propertyKey.split(separator), propertyValue);
+      walk(fullObject, propertySplits, propertyValue, path);
     }
 
     return fullObject;
   }, {} as any);
 }
 
-function walk(obj: any, keys: string[], value: any) {
+function walk(obj: any, keys: string[], value: any, path: string[]) {
+  assertValidPath([...path, ...keys]);
+
   const key = keys.shift()!;
   if (keys.length === 0) {
     obj[key] = value;
     return;
   }
 
-  if (obj[key] === undefined) {
+  if (!obj.hasOwnProperty(key)) {
     obj[key] = {};
   }
 
-  walk(obj[key], keys, ensureDeepObject(value));
+  walk(obj[key], keys, ensureDeepObject(value, [...path, key, ...keys]), [...path, key]);
 }
+
+const assertValidPath = (path: string[]) => {
+  const flat = path.join('.');
+  FORBIDDEN_PATTERNS.forEach((pattern) => {
+    if (flat.includes(pattern)) {
+      throw new Error(`Forbidden path detected: ${flat}`);
+    }
+  });
+};

src/plugins/telemetry/server/collectors/usage/ensure_deep_object.test.ts

@@ -1,145 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { ensureDeepObject } from './ensure_deep_object';
-
-test('flat object', () => {
-  const obj = {
-    'foo.a': 1,
-    'foo.b': 2,
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    foo: {
-      a: 1,
-      b: 2,
-    },
-  });
-});
-
-test('deep object', () => {
-  const obj = {
-    foo: {
-      a: 1,
-      b: 2,
-    },
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    foo: {
-      a: 1,
-      b: 2,
-    },
-  });
-});
-
-test('flat within deep object', () => {
-  const obj = {
-    foo: {
-      b: 2,
-      'bar.a': 1,
-    },
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    foo: {
-      b: 2,
-      bar: {
-        a: 1,
-      },
-    },
-  });
-});
-
-test('flat then flat object', () => {
-  const obj = {
-    'foo.bar': {
-      b: 2,
-      'quux.a': 1,
-    },
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    foo: {
-      bar: {
-        b: 2,
-        quux: {
-          a: 1,
-        },
-      },
-    },
-  });
-});
-
-test('full with empty array', () => {
-  const obj = {
-    a: 1,
-    b: [],
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    a: 1,
-    b: [],
-  });
-});
-
-test('full with array of primitive values', () => {
-  const obj = {
-    a: 1,
-    b: [1, 2, 3],
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    a: 1,
-    b: [1, 2, 3],
-  });
-});
-
-test('full with array of full objects', () => {
-  const obj = {
-    a: 1,
-    b: [{ c: 2 }, { d: 3 }],
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    a: 1,
-    b: [{ c: 2 }, { d: 3 }],
-  });
-});
-
-test('full with array of flat objects', () => {
-  const obj = {
-    a: 1,
-    b: [{ 'c.d': 2 }, { 'e.f': 3 }],
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    a: 1,
-    b: [{ c: { d: 2 } }, { e: { f: 3 } }],
-  });
-});
-
-test('flat with flat and array of flat objects', () => {
-  const obj = {
-    a: 1,
-    'b.c': 2,
-    d: [3, { 'e.f': 4 }, { 'g.h': 5 }],
-  };
-
-  expect(ensureDeepObject(obj)).toEqual({
-    a: 1,
-    b: { c: 2 },
-    d: [3, { e: { f: 4 } }, { g: { h: 5 } }],
-  });
-});
-
-test('array composed of flat objects', () => {
-  const arr = [{ 'c.d': 2 }, { 'e.f': 3 }];
-
-  expect(ensureDeepObject(arr)).toEqual([{ c: { d: 2 } }, { e: { f: 3 } }]);
-});

src/plugins/telemetry/server/collectors/usage/telemetry_usage_collector.ts

@@ -10,13 +10,12 @@ import { accessSync, constants, readFileSync, statSync } from 'fs';
 import { safeLoad } from 'js-yaml';
 import { dirname, join } from 'path';
 import { Observable, firstValueFrom } from 'rxjs';
-
+import { ensureDeepObject } from '@kbn/std';
 import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
 
 import { TelemetryConfigType } from '../../config';
 
 // look for telemetry.yml in the same places we expect kibana.yml
-import { ensureDeepObject } from './ensure_deep_object';
 import { staticTelemetrySchema } from './schema';
 
 /**

src/plugins/telemetry/tsconfig.json

@@ -31,6 +31,7 @@
     "@kbn/utils",
     "@kbn/core-saved-objects-server",
     "@kbn/core-saved-objects-api-server",
+    "@kbn/std",
   ],
   "exclude": [
     "target/**/*",